1 /* 2 * iSCSI Initiator over TCP/IP Data-Path 3 * 4 * Copyright (C) 2004 Dmitry Yusupov 5 * Copyright (C) 2004 Alex Aizman 6 * Copyright (C) 2005 Mike Christie 7 * maintained by open-iscsi@googlegroups.com 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License as published 11 * by the Free Software Foundation; either version 2 of the License, or 12 * (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, but 15 * WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * General Public License for more details. 18 * 19 * See the file COPYING included with this distribution for more details. 20 * 21 * Credits: 22 * Christoph Hellwig 23 * FUJITA Tomonori 24 * Arne Redlich 25 * Zhenyu Wang 26 */ 27 28 #include <linux/types.h> 29 #include <linux/list.h> 30 #include <linux/inet.h> 31 #include <linux/blkdev.h> 32 #include <linux/crypto.h> 33 #include <linux/delay.h> 34 #include <linux/kfifo.h> 35 #include <linux/scatterlist.h> 36 #include <net/tcp.h> 37 #include <scsi/scsi_cmnd.h> 38 #include <scsi/scsi_device.h> 39 #include <scsi/scsi_eh.h> 40 #include <scsi/scsi_request.h> 41 #include <scsi/scsi_tcq.h> 42 #include <scsi/scsi_host.h> 43 #include <scsi/scsi.h> 44 #include <scsi/scsi_transport_iscsi.h> 45 46 #include "iscsi_tcp.h" 47 48 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, " 49 "Alex Aizman <itn780@yahoo.com>"); 50 MODULE_DESCRIPTION("iSCSI/TCP data-path"); 51 MODULE_LICENSE("GPL"); 52 MODULE_VERSION("0:4.409"); 53 /* #define DEBUG_TCP */ 54 /* #define DEBUG_SCSI */ 55 #define DEBUG_ASSERT 56 57 #ifdef DEBUG_TCP 58 #define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt) 59 #else 60 #define debug_tcp(fmt...) 61 #endif 62 63 #ifdef DEBUG_SCSI 64 #define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt) 65 #else 66 #define debug_scsi(fmt...) 67 #endif 68 69 #ifndef DEBUG_ASSERT 70 #ifdef BUG_ON 71 #undef BUG_ON 72 #endif 73 #define BUG_ON(expr) 74 #endif 75 76 #define INVALID_SN_DELTA 0xffff 77 78 static unsigned int iscsi_max_lun = 512; 79 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO); 80 81 /* global data */ 82 static kmem_cache_t *taskcache; 83 84 static inline void 85 iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size) 86 { 87 sg_init_one(&ibuf->sg, (u8 *)vbuf, size); 88 ibuf->sent = 0; 89 } 90 91 static inline void 92 iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size) 93 { 94 ibuf->sg.page = (void*)vbuf; 95 ibuf->sg.offset = (unsigned int)-1; 96 ibuf->sg.length = size; 97 ibuf->sent = 0; 98 } 99 100 static inline void* 101 iscsi_buf_iov_base(struct iscsi_buf *ibuf) 102 { 103 return (char*)ibuf->sg.page + ibuf->sent; 104 } 105 106 static inline void 107 iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg) 108 { 109 /* 110 * Fastpath: sg element fits into single page 111 */ 112 if (sg->length + sg->offset <= PAGE_SIZE && page_count(sg->page) >= 2) { 113 ibuf->sg.page = sg->page; 114 ibuf->sg.offset = sg->offset; 115 ibuf->sg.length = sg->length; 116 } else 117 iscsi_buf_init_iov(ibuf, page_address(sg->page), sg->length); 118 ibuf->sent = 0; 119 } 120 121 static inline int 122 iscsi_buf_left(struct iscsi_buf *ibuf) 123 { 124 int rc; 125 126 rc = ibuf->sg.length - ibuf->sent; 127 BUG_ON(rc < 0); 128 return rc; 129 } 130 131 static inline void 132 iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf, 133 u8* crc) 134 { 135 crypto_digest_digest(conn->tx_tfm, &buf->sg, 1, crc); 136 buf->sg.length += sizeof(uint32_t); 137 } 138 139 static void 140 iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err) 141 { 142 struct iscsi_session *session = conn->session; 143 unsigned long flags; 144 145 spin_lock_irqsave(&session->lock, flags); 146 if (session->conn_cnt == 1 || session->leadconn == conn) 147 session->state = ISCSI_STATE_FAILED; 148 spin_unlock_irqrestore(&session->lock, flags); 149 set_bit(SUSPEND_BIT, &conn->suspend_tx); 150 set_bit(SUSPEND_BIT, &conn->suspend_rx); 151 iscsi_conn_error(iscsi_handle(conn), err); 152 } 153 154 static inline int 155 iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr) 156 { 157 uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn); 158 uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn); 159 160 if (max_cmdsn < exp_cmdsn -1 && 161 max_cmdsn > exp_cmdsn - INVALID_SN_DELTA) 162 return ISCSI_ERR_MAX_CMDSN; 163 if (max_cmdsn > session->max_cmdsn || 164 max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA) 165 session->max_cmdsn = max_cmdsn; 166 if (exp_cmdsn > session->exp_cmdsn || 167 exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA) 168 session->exp_cmdsn = exp_cmdsn; 169 170 return 0; 171 } 172 173 static inline int 174 iscsi_hdr_extract(struct iscsi_conn *conn) 175 { 176 struct sk_buff *skb = conn->in.skb; 177 178 if (conn->in.copy >= conn->hdr_size && 179 conn->in_progress == IN_PROGRESS_WAIT_HEADER) { 180 /* 181 * Zero-copy PDU Header: using connection context 182 * to store header pointer. 183 */ 184 if (skb_shinfo(skb)->frag_list == NULL && 185 !skb_shinfo(skb)->nr_frags) 186 conn->in.hdr = (struct iscsi_hdr *) 187 ((char*)skb->data + conn->in.offset); 188 else { 189 /* ignoring return code since we checked 190 * in.copy before */ 191 skb_copy_bits(skb, conn->in.offset, 192 &conn->hdr, conn->hdr_size); 193 conn->in.hdr = &conn->hdr; 194 } 195 conn->in.offset += conn->hdr_size; 196 conn->in.copy -= conn->hdr_size; 197 } else { 198 int hdr_remains; 199 int copylen; 200 201 /* 202 * PDU header scattered across SKB's, 203 * copying it... This'll happen quite rarely. 204 */ 205 206 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER) 207 conn->in.hdr_offset = 0; 208 209 hdr_remains = conn->hdr_size - conn->in.hdr_offset; 210 BUG_ON(hdr_remains <= 0); 211 212 copylen = min(conn->in.copy, hdr_remains); 213 skb_copy_bits(skb, conn->in.offset, 214 (char*)&conn->hdr + conn->in.hdr_offset, copylen); 215 216 debug_tcp("PDU gather offset %d bytes %d in.offset %d " 217 "in.copy %d\n", conn->in.hdr_offset, copylen, 218 conn->in.offset, conn->in.copy); 219 220 conn->in.offset += copylen; 221 conn->in.copy -= copylen; 222 if (copylen < hdr_remains) { 223 conn->in_progress = IN_PROGRESS_HEADER_GATHER; 224 conn->in.hdr_offset += copylen; 225 return -EAGAIN; 226 } 227 conn->in.hdr = &conn->hdr; 228 conn->discontiguous_hdr_cnt++; 229 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 230 } 231 232 return 0; 233 } 234 235 static inline void 236 iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 237 { 238 struct scsi_cmnd *sc = ctask->sc; 239 struct iscsi_session *session = conn->session; 240 241 spin_lock(&session->lock); 242 if (unlikely(!sc)) { 243 spin_unlock(&session->lock); 244 return; 245 } 246 if (sc->sc_data_direction == DMA_TO_DEVICE) { 247 struct iscsi_data_task *dtask, *n; 248 /* WRITE: cleanup Data-Out's if any */ 249 spin_lock(&conn->lock); 250 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) { 251 list_del(&dtask->item); 252 mempool_free(dtask, ctask->datapool); 253 } 254 spin_unlock(&conn->lock); 255 } 256 ctask->xmstate = XMSTATE_IDLE; 257 ctask->r2t = NULL; 258 ctask->sc = NULL; 259 __kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*)); 260 spin_unlock(&session->lock); 261 } 262 263 /** 264 * iscsi_cmd_rsp - SCSI Command Response processing 265 * @conn: iscsi connection 266 * @ctask: scsi command task 267 **/ 268 static int 269 iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 270 { 271 int rc; 272 struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr; 273 struct iscsi_session *session = conn->session; 274 struct scsi_cmnd *sc = ctask->sc; 275 276 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr); 277 if (rc) { 278 sc->result = (DID_ERROR << 16); 279 goto out; 280 } 281 282 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1; 283 284 sc->result = (DID_OK << 16) | rhdr->cmd_status; 285 286 if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) { 287 sc->result = (DID_ERROR << 16); 288 goto out; 289 } 290 291 if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) { 292 int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE); 293 294 memcpy(sc->sense_buffer, conn->data + 2, sensecopy); 295 debug_scsi("copied %d bytes of sense\n", sensecopy); 296 } 297 298 if (sc->sc_data_direction == DMA_TO_DEVICE) 299 goto out; 300 301 if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) { 302 int res_count = be32_to_cpu(rhdr->residual_count); 303 304 if (res_count > 0 && res_count <= sc->request_bufflen) 305 sc->resid = res_count; 306 else 307 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status; 308 } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW) 309 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status; 310 else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW) 311 sc->resid = be32_to_cpu(rhdr->residual_count); 312 313 out: 314 debug_scsi("done [sc %lx res %d itt 0x%x]\n", 315 (long)sc, sc->result, ctask->itt); 316 conn->scsirsp_pdus_cnt++; 317 iscsi_ctask_cleanup(conn, ctask); 318 sc->scsi_done(sc); 319 return rc; 320 } 321 322 /** 323 * iscsi_data_rsp - SCSI Data-In Response processing 324 * @conn: iscsi connection 325 * @ctask: scsi command task 326 **/ 327 static int 328 iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 329 { 330 int rc; 331 struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr; 332 struct iscsi_session *session = conn->session; 333 int datasn = be32_to_cpu(rhdr->datasn); 334 335 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr); 336 if (rc) 337 return rc; 338 /* 339 * setup Data-In byte counter (gets decremented..) 340 */ 341 ctask->data_count = conn->in.datalen; 342 343 if (conn->in.datalen == 0) 344 return 0; 345 346 if (ctask->datasn != datasn) 347 return ISCSI_ERR_DATASN; 348 349 ctask->datasn++; 350 351 ctask->data_offset = be32_to_cpu(rhdr->offset); 352 if (ctask->data_offset + conn->in.datalen > ctask->total_length) 353 return ISCSI_ERR_DATA_OFFSET; 354 355 if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) { 356 struct scsi_cmnd *sc = ctask->sc; 357 358 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1; 359 if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) { 360 int res_count = be32_to_cpu(rhdr->residual_count); 361 362 if (res_count > 0 && 363 res_count <= sc->request_bufflen) { 364 sc->resid = res_count; 365 sc->result = (DID_OK << 16) | rhdr->cmd_status; 366 } else 367 sc->result = (DID_BAD_TARGET << 16) | 368 rhdr->cmd_status; 369 } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW) 370 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status; 371 else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW) { 372 sc->resid = be32_to_cpu(rhdr->residual_count); 373 sc->result = (DID_OK << 16) | rhdr->cmd_status; 374 } else 375 sc->result = (DID_OK << 16) | rhdr->cmd_status; 376 } 377 378 conn->datain_pdus_cnt++; 379 return 0; 380 } 381 382 /** 383 * iscsi_solicit_data_init - initialize first Data-Out 384 * @conn: iscsi connection 385 * @ctask: scsi command task 386 * @r2t: R2T info 387 * 388 * Notes: 389 * Initialize first Data-Out within this R2T sequence and finds 390 * proper data_offset within this SCSI command. 391 * 392 * This function is called with connection lock taken. 393 **/ 394 static void 395 iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 396 struct iscsi_r2t_info *r2t) 397 { 398 struct iscsi_data *hdr; 399 struct iscsi_data_task *dtask; 400 struct scsi_cmnd *sc = ctask->sc; 401 402 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 403 BUG_ON(!dtask); 404 hdr = &dtask->hdr; 405 memset(hdr, 0, sizeof(struct iscsi_data)); 406 hdr->ttt = r2t->ttt; 407 hdr->datasn = cpu_to_be32(r2t->solicit_datasn); 408 r2t->solicit_datasn++; 409 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 410 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 411 hdr->itt = ctask->hdr.itt; 412 hdr->exp_statsn = r2t->exp_statsn; 413 hdr->offset = cpu_to_be32(r2t->data_offset); 414 if (r2t->data_length > conn->max_xmit_dlength) { 415 hton24(hdr->dlength, conn->max_xmit_dlength); 416 r2t->data_count = conn->max_xmit_dlength; 417 hdr->flags = 0; 418 } else { 419 hton24(hdr->dlength, r2t->data_length); 420 r2t->data_count = r2t->data_length; 421 hdr->flags = ISCSI_FLAG_CMD_FINAL; 422 } 423 conn->dataout_pdus_cnt++; 424 425 r2t->sent = 0; 426 427 iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr, 428 sizeof(struct iscsi_hdr)); 429 430 r2t->dtask = dtask; 431 432 if (sc->use_sg) { 433 int i, sg_count = 0; 434 struct scatterlist *sg = sc->request_buffer; 435 436 r2t->sg = NULL; 437 for (i = 0; i < sc->use_sg; i++, sg += 1) { 438 /* FIXME: prefetch ? */ 439 if (sg_count + sg->length > r2t->data_offset) { 440 int page_offset; 441 442 /* sg page found! */ 443 444 /* offset within this page */ 445 page_offset = r2t->data_offset - sg_count; 446 447 /* fill in this buffer */ 448 iscsi_buf_init_sg(&r2t->sendbuf, sg); 449 r2t->sendbuf.sg.offset += page_offset; 450 r2t->sendbuf.sg.length -= page_offset; 451 452 /* xmit logic will continue with next one */ 453 r2t->sg = sg + 1; 454 break; 455 } 456 sg_count += sg->length; 457 } 458 BUG_ON(r2t->sg == NULL); 459 } else 460 iscsi_buf_init_iov(&ctask->sendbuf, 461 (char*)sc->request_buffer + r2t->data_offset, 462 r2t->data_count); 463 464 list_add(&dtask->item, &ctask->dataqueue); 465 } 466 467 /** 468 * iscsi_r2t_rsp - iSCSI R2T Response processing 469 * @conn: iscsi connection 470 * @ctask: scsi command task 471 **/ 472 static int 473 iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 474 { 475 struct iscsi_r2t_info *r2t; 476 struct iscsi_session *session = conn->session; 477 struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr; 478 int r2tsn = be32_to_cpu(rhdr->r2tsn); 479 int rc; 480 481 if (conn->in.ahslen) 482 return ISCSI_ERR_AHSLEN; 483 484 if (conn->in.datalen) 485 return ISCSI_ERR_DATALEN; 486 487 if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn) 488 return ISCSI_ERR_R2TSN; 489 490 rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr); 491 if (rc) 492 return rc; 493 494 /* FIXME: use R2TSN to detect missing R2T */ 495 496 /* fill-in new R2T associated with the task */ 497 spin_lock(&session->lock); 498 if (!ctask->sc || ctask->mtask || 499 session->state != ISCSI_STATE_LOGGED_IN) { 500 printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in " 501 "recovery...\n", ctask->itt); 502 spin_unlock(&session->lock); 503 return 0; 504 } 505 rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 506 BUG_ON(!rc); 507 508 r2t->exp_statsn = rhdr->statsn; 509 r2t->data_length = be32_to_cpu(rhdr->data_length); 510 if (r2t->data_length == 0 || 511 r2t->data_length > session->max_burst) { 512 spin_unlock(&session->lock); 513 return ISCSI_ERR_DATALEN; 514 } 515 516 r2t->data_offset = be32_to_cpu(rhdr->data_offset); 517 if (r2t->data_offset + r2t->data_length > ctask->total_length) { 518 spin_unlock(&session->lock); 519 return ISCSI_ERR_DATALEN; 520 } 521 522 r2t->ttt = rhdr->ttt; /* no flip */ 523 r2t->solicit_datasn = 0; 524 525 iscsi_solicit_data_init(conn, ctask, r2t); 526 527 ctask->exp_r2tsn = r2tsn + 1; 528 ctask->xmstate |= XMSTATE_SOL_HDR; 529 __kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*)); 530 __kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*)); 531 532 schedule_work(&conn->xmitwork); 533 conn->r2t_pdus_cnt++; 534 spin_unlock(&session->lock); 535 536 return 0; 537 } 538 539 static int 540 iscsi_hdr_recv(struct iscsi_conn *conn) 541 { 542 int rc = 0; 543 struct iscsi_hdr *hdr; 544 struct iscsi_cmd_task *ctask; 545 struct iscsi_session *session = conn->session; 546 uint32_t cdgst, rdgst = 0; 547 548 hdr = conn->in.hdr; 549 550 /* verify PDU length */ 551 conn->in.datalen = ntoh24(hdr->dlength); 552 if (conn->in.datalen > conn->max_recv_dlength) { 553 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n", 554 conn->in.datalen, conn->max_recv_dlength); 555 return ISCSI_ERR_DATALEN; 556 } 557 conn->data_copied = 0; 558 559 /* read AHS */ 560 conn->in.ahslen = hdr->hlength * 4; 561 conn->in.offset += conn->in.ahslen; 562 conn->in.copy -= conn->in.ahslen; 563 if (conn->in.copy < 0) { 564 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length " 565 "%d bytes\n", conn->in.ahslen); 566 return ISCSI_ERR_AHSLEN; 567 } 568 569 /* calculate read padding */ 570 conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1); 571 if (conn->in.padding) { 572 conn->in.padding = ISCSI_PAD_LEN - conn->in.padding; 573 debug_scsi("read padding %d bytes\n", conn->in.padding); 574 } 575 576 if (conn->hdrdgst_en) { 577 struct scatterlist sg; 578 579 sg_init_one(&sg, (u8 *)hdr, 580 sizeof(struct iscsi_hdr) + conn->in.ahslen); 581 crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst); 582 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) + 583 conn->in.ahslen); 584 } 585 586 /* save opcode for later */ 587 conn->in.opcode = hdr->opcode; 588 589 /* verify itt (itt encoding: age+cid+itt) */ 590 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) { 591 if ((hdr->itt & AGE_MASK) != 592 (session->age << AGE_SHIFT)) { 593 printk(KERN_ERR "iscsi_tcp: received itt %x expected " 594 "session age (%x)\n", hdr->itt, 595 session->age & AGE_MASK); 596 return ISCSI_ERR_BAD_ITT; 597 } 598 599 if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) { 600 printk(KERN_ERR "iscsi_tcp: received itt %x, expected " 601 "CID (%x)\n", hdr->itt, conn->id); 602 return ISCSI_ERR_BAD_ITT; 603 } 604 conn->in.itt = hdr->itt & ITT_MASK; 605 } else 606 conn->in.itt = hdr->itt; 607 608 debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n", 609 hdr->opcode, conn->in.offset, conn->in.copy, 610 conn->in.ahslen, conn->in.datalen); 611 612 if (conn->in.itt < session->cmds_max) { 613 if (conn->hdrdgst_en && cdgst != rdgst) { 614 printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error " 615 "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst, 616 cdgst); 617 return ISCSI_ERR_HDR_DGST; 618 } 619 620 ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt]; 621 622 if (!ctask->sc) { 623 printk(KERN_INFO "iscsi_tcp: dropping ctask with " 624 "itt 0x%x\n", ctask->itt); 625 conn->in.datalen = 0; /* force drop */ 626 return 0; 627 } 628 629 if (ctask->sc->SCp.phase != session->age) { 630 printk(KERN_ERR "iscsi_tcp: ctask's session age %d, " 631 "expected %d\n", ctask->sc->SCp.phase, 632 session->age); 633 return ISCSI_ERR_SESSION_FAILED; 634 } 635 636 conn->in.ctask = ctask; 637 638 debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n", 639 hdr->opcode, conn->id, (long)ctask->sc, 640 ctask->itt, conn->in.datalen); 641 642 switch(conn->in.opcode) { 643 case ISCSI_OP_SCSI_CMD_RSP: 644 BUG_ON((void*)ctask != ctask->sc->SCp.ptr); 645 if (ctask->hdr.flags & ISCSI_FLAG_CMD_WRITE) 646 rc = iscsi_cmd_rsp(conn, ctask); 647 else if (!conn->in.datalen) 648 rc = iscsi_cmd_rsp(conn, ctask); 649 else 650 /* 651 * got sense or response data; copying PDU 652 * Header to the connection's header 653 * placeholder 654 */ 655 memcpy(&conn->hdr, hdr, 656 sizeof(struct iscsi_hdr)); 657 break; 658 case ISCSI_OP_SCSI_DATA_IN: 659 BUG_ON((void*)ctask != ctask->sc->SCp.ptr); 660 /* save flags for non-exceptional status */ 661 conn->in.flags = hdr->flags; 662 /* save cmd_status for sense data */ 663 conn->in.cmd_status = 664 ((struct iscsi_data_rsp*)hdr)->cmd_status; 665 rc = iscsi_data_rsp(conn, ctask); 666 break; 667 case ISCSI_OP_R2T: 668 BUG_ON((void*)ctask != ctask->sc->SCp.ptr); 669 if (ctask->hdr.flags & ISCSI_FLAG_CMD_WRITE && 670 ctask->sc->sc_data_direction == DMA_TO_DEVICE) 671 rc = iscsi_r2t_rsp(conn, ctask); 672 else 673 rc = ISCSI_ERR_PROTO; 674 break; 675 default: 676 rc = ISCSI_ERR_BAD_OPCODE; 677 break; 678 } 679 } else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET && 680 conn->in.itt < ISCSI_MGMT_ITT_OFFSET + 681 session->mgmtpool_max) { 682 struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *) 683 session->mgmt_cmds[conn->in.itt - 684 ISCSI_MGMT_ITT_OFFSET]; 685 686 debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n", 687 conn->in.opcode, conn->id, mtask->itt, 688 conn->in.datalen); 689 690 switch(conn->in.opcode) { 691 case ISCSI_OP_LOGIN_RSP: 692 case ISCSI_OP_TEXT_RSP: 693 case ISCSI_OP_LOGOUT_RSP: 694 rc = iscsi_check_assign_cmdsn(session, 695 (struct iscsi_nopin*)hdr); 696 if (rc) 697 break; 698 699 if (!conn->in.datalen) { 700 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr, 701 NULL, 0); 702 if (conn->login_mtask != mtask) { 703 spin_lock(&session->lock); 704 __kfifo_put(session->mgmtpool.queue, 705 (void*)&mtask, sizeof(void*)); 706 spin_unlock(&session->lock); 707 } 708 } 709 break; 710 case ISCSI_OP_SCSI_TMFUNC_RSP: 711 rc = iscsi_check_assign_cmdsn(session, 712 (struct iscsi_nopin*)hdr); 713 if (rc) 714 break; 715 716 if (conn->in.datalen || conn->in.ahslen) { 717 rc = ISCSI_ERR_PROTO; 718 break; 719 } 720 conn->tmfrsp_pdus_cnt++; 721 spin_lock(&session->lock); 722 if (conn->tmabort_state == TMABORT_INITIAL) { 723 __kfifo_put(session->mgmtpool.queue, 724 (void*)&mtask, sizeof(void*)); 725 conn->tmabort_state = 726 ((struct iscsi_tm_rsp *)hdr)-> 727 response == ISCSI_TMF_RSP_COMPLETE ? 728 TMABORT_SUCCESS:TMABORT_FAILED; 729 /* unblock eh_abort() */ 730 wake_up(&conn->ehwait); 731 } 732 spin_unlock(&session->lock); 733 break; 734 case ISCSI_OP_NOOP_IN: 735 if (hdr->ttt != ISCSI_RESERVED_TAG) { 736 rc = ISCSI_ERR_PROTO; 737 break; 738 } 739 rc = iscsi_check_assign_cmdsn(session, 740 (struct iscsi_nopin*)hdr); 741 if (rc) 742 break; 743 conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1; 744 745 if (!conn->in.datalen) { 746 struct iscsi_mgmt_task *mtask; 747 748 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr, 749 NULL, 0); 750 mtask = (struct iscsi_mgmt_task *) 751 session->mgmt_cmds[conn->in.itt - 752 ISCSI_MGMT_ITT_OFFSET]; 753 if (conn->login_mtask != mtask) { 754 spin_lock(&session->lock); 755 __kfifo_put(session->mgmtpool.queue, 756 (void*)&mtask, sizeof(void*)); 757 spin_unlock(&session->lock); 758 } 759 } 760 break; 761 default: 762 rc = ISCSI_ERR_BAD_OPCODE; 763 break; 764 } 765 } else if (conn->in.itt == ISCSI_RESERVED_TAG) { 766 switch(conn->in.opcode) { 767 case ISCSI_OP_NOOP_IN: 768 if (!conn->in.datalen) { 769 rc = iscsi_check_assign_cmdsn(session, 770 (struct iscsi_nopin*)hdr); 771 if (!rc && hdr->ttt != ISCSI_RESERVED_TAG) 772 rc = iscsi_recv_pdu(iscsi_handle(conn), 773 hdr, NULL, 0); 774 } else 775 rc = ISCSI_ERR_PROTO; 776 break; 777 case ISCSI_OP_REJECT: 778 /* we need sth like iscsi_reject_rsp()*/ 779 case ISCSI_OP_ASYNC_EVENT: 780 /* we need sth like iscsi_async_event_rsp() */ 781 rc = ISCSI_ERR_BAD_OPCODE; 782 break; 783 default: 784 rc = ISCSI_ERR_BAD_OPCODE; 785 break; 786 } 787 } else 788 rc = ISCSI_ERR_BAD_ITT; 789 790 return rc; 791 } 792 793 /** 794 * iscsi_ctask_copy - copy skb bits to the destanation cmd task 795 * @conn: iscsi connection 796 * @ctask: scsi command task 797 * @buf: buffer to copy to 798 * @buf_size: size of buffer 799 * @offset: offset within the buffer 800 * 801 * Notes: 802 * The function calls skb_copy_bits() and updates per-connection and 803 * per-cmd byte counters. 804 * 805 * Read counters (in bytes): 806 * 807 * conn->in.offset offset within in progress SKB 808 * conn->in.copy left to copy from in progress SKB 809 * including padding 810 * conn->in.copied copied already from in progress SKB 811 * conn->data_copied copied already from in progress buffer 812 * ctask->sent total bytes sent up to the MidLayer 813 * ctask->data_count left to copy from in progress Data-In 814 * buf_left left to copy from in progress buffer 815 **/ 816 static inline int 817 iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 818 void *buf, int buf_size, int offset) 819 { 820 int buf_left = buf_size - (conn->data_copied + offset); 821 int size = min(conn->in.copy, buf_left); 822 int rc; 823 824 size = min(size, ctask->data_count); 825 826 debug_tcp("ctask_copy %d bytes at offset %d copied %d\n", 827 size, conn->in.offset, conn->in.copied); 828 829 BUG_ON(size <= 0); 830 BUG_ON(ctask->sent + size > ctask->total_length); 831 832 rc = skb_copy_bits(conn->in.skb, conn->in.offset, 833 (char*)buf + (offset + conn->data_copied), size); 834 /* must fit into skb->len */ 835 BUG_ON(rc); 836 837 conn->in.offset += size; 838 conn->in.copy -= size; 839 conn->in.copied += size; 840 conn->data_copied += size; 841 ctask->sent += size; 842 ctask->data_count -= size; 843 844 BUG_ON(conn->in.copy < 0); 845 BUG_ON(ctask->data_count < 0); 846 847 if (buf_size != (conn->data_copied + offset)) { 848 if (!ctask->data_count) { 849 BUG_ON(buf_size - conn->data_copied < 0); 850 /* done with this PDU */ 851 return buf_size - conn->data_copied; 852 } 853 return -EAGAIN; 854 } 855 856 /* done with this buffer or with both - PDU and buffer */ 857 conn->data_copied = 0; 858 return 0; 859 } 860 861 /** 862 * iscsi_tcp_copy - copy skb bits to the destanation buffer 863 * @conn: iscsi connection 864 * @buf: buffer to copy to 865 * @buf_size: number of bytes to copy 866 * 867 * Notes: 868 * The function calls skb_copy_bits() and updates per-connection 869 * byte counters. 870 **/ 871 static inline int 872 iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size) 873 { 874 int buf_left = buf_size - conn->data_copied; 875 int size = min(conn->in.copy, buf_left); 876 int rc; 877 878 debug_tcp("tcp_copy %d bytes at offset %d copied %d\n", 879 size, conn->in.offset, conn->data_copied); 880 BUG_ON(size <= 0); 881 882 rc = skb_copy_bits(conn->in.skb, conn->in.offset, 883 (char*)buf + conn->data_copied, size); 884 BUG_ON(rc); 885 886 conn->in.offset += size; 887 conn->in.copy -= size; 888 conn->in.copied += size; 889 conn->data_copied += size; 890 891 if (buf_size != conn->data_copied) 892 return -EAGAIN; 893 894 return 0; 895 } 896 897 static inline void 898 partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg, 899 int offset, int length) 900 { 901 struct scatterlist temp; 902 903 memcpy(&temp, sg, sizeof(struct scatterlist)); 904 temp.offset = offset; 905 temp.length = length; 906 crypto_digest_update(conn->data_rx_tfm, &temp, 1); 907 } 908 909 static int iscsi_scsi_data_in(struct iscsi_conn *conn) 910 { 911 struct iscsi_cmd_task *ctask = conn->in.ctask; 912 struct scsi_cmnd *sc = ctask->sc; 913 struct scatterlist tmp, *sg; 914 int i, offset, rc = 0; 915 916 BUG_ON((void*)ctask != sc->SCp.ptr); 917 918 /* 919 * copying Data-In into the Scsi_Cmnd 920 */ 921 if (!sc->use_sg) { 922 i = ctask->data_count; 923 rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer, 924 sc->request_bufflen, ctask->data_offset); 925 if (rc == -EAGAIN) 926 return rc; 927 if (conn->datadgst_en) { 928 sg_init_one(&tmp, sc->request_buffer, i); 929 crypto_digest_update(conn->data_rx_tfm, &tmp, 1); 930 } 931 rc = 0; 932 goto done; 933 } 934 935 offset = ctask->data_offset; 936 sg = sc->request_buffer; 937 938 if (ctask->data_offset) 939 for (i = 0; i < ctask->sg_count; i++) 940 offset -= sg[i].length; 941 /* we've passed through partial sg*/ 942 if (offset < 0) 943 offset = 0; 944 945 for (i = ctask->sg_count; i < sc->use_sg; i++) { 946 char *dest; 947 948 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0); 949 rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset, 950 sg[i].length, offset); 951 kunmap_atomic(dest, KM_SOFTIRQ0); 952 if (rc == -EAGAIN) 953 /* continue with the next SKB/PDU */ 954 return rc; 955 if (!rc) { 956 if (conn->datadgst_en) { 957 if (!offset) 958 crypto_digest_update(conn->data_rx_tfm, 959 &sg[i], 1); 960 else 961 partial_sg_digest_update(conn, &sg[i], 962 sg[i].offset + offset, 963 sg[i].length - offset); 964 } 965 offset = 0; 966 ctask->sg_count++; 967 } 968 969 if (!ctask->data_count) { 970 if (rc && conn->datadgst_en) 971 /* 972 * data-in is complete, but buffer not... 973 */ 974 partial_sg_digest_update(conn, &sg[i], 975 sg[i].offset, sg[i].length-rc); 976 rc = 0; 977 break; 978 } 979 980 if (!conn->in.copy) 981 return -EAGAIN; 982 } 983 BUG_ON(ctask->data_count); 984 985 done: 986 /* check for non-exceptional status */ 987 if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) { 988 debug_scsi("done [sc %lx res %d itt 0x%x]\n", 989 (long)sc, sc->result, ctask->itt); 990 conn->scsirsp_pdus_cnt++; 991 iscsi_ctask_cleanup(conn, ctask); 992 sc->scsi_done(sc); 993 } 994 995 return rc; 996 } 997 998 static int 999 iscsi_data_recv(struct iscsi_conn *conn) 1000 { 1001 struct iscsi_session *session = conn->session; 1002 int rc = 0; 1003 1004 switch(conn->in.opcode) { 1005 case ISCSI_OP_SCSI_DATA_IN: 1006 rc = iscsi_scsi_data_in(conn); 1007 break; 1008 case ISCSI_OP_SCSI_CMD_RSP: { 1009 /* 1010 * SCSI Sense Data: 1011 * copying the entire Data Segment. 1012 */ 1013 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) { 1014 rc = -EAGAIN; 1015 goto exit; 1016 } 1017 1018 /* 1019 * check for sense 1020 */ 1021 conn->in.hdr = &conn->hdr; 1022 conn->senselen = (conn->data[0] << 8) | conn->data[1]; 1023 rc = iscsi_cmd_rsp(conn, conn->in.ctask); 1024 } 1025 break; 1026 case ISCSI_OP_TEXT_RSP: 1027 case ISCSI_OP_LOGIN_RSP: 1028 case ISCSI_OP_NOOP_IN: { 1029 struct iscsi_mgmt_task *mtask = NULL; 1030 1031 if (conn->in.itt != ISCSI_RESERVED_TAG) 1032 mtask = (struct iscsi_mgmt_task *) 1033 session->mgmt_cmds[conn->in.itt - 1034 ISCSI_MGMT_ITT_OFFSET]; 1035 1036 /* 1037 * Collect data segment to the connection's data 1038 * placeholder 1039 */ 1040 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) { 1041 rc = -EAGAIN; 1042 goto exit; 1043 } 1044 1045 rc = iscsi_recv_pdu(iscsi_handle(conn), conn->in.hdr, 1046 conn->data, conn->in.datalen); 1047 1048 if (mtask && conn->login_mtask != mtask) { 1049 spin_lock(&session->lock); 1050 __kfifo_put(session->mgmtpool.queue, (void*)&mtask, 1051 sizeof(void*)); 1052 spin_unlock(&session->lock); 1053 } 1054 } 1055 break; 1056 default: 1057 BUG_ON(1); 1058 } 1059 exit: 1060 return rc; 1061 } 1062 1063 /** 1064 * iscsi_tcp_data_recv - TCP receive in sendfile fashion 1065 * @rd_desc: read descriptor 1066 * @skb: socket buffer 1067 * @offset: offset in skb 1068 * @len: skb->len - offset 1069 **/ 1070 static int 1071 iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 1072 unsigned int offset, size_t len) 1073 { 1074 int rc; 1075 struct iscsi_conn *conn = rd_desc->arg.data; 1076 int processed; 1077 char pad[ISCSI_PAD_LEN]; 1078 struct scatterlist sg; 1079 1080 /* 1081 * Save current SKB and its offset in the corresponding 1082 * connection context. 1083 */ 1084 conn->in.copy = skb->len - offset; 1085 conn->in.offset = offset; 1086 conn->in.skb = skb; 1087 conn->in.len = conn->in.copy; 1088 BUG_ON(conn->in.copy <= 0); 1089 debug_tcp("in %d bytes\n", conn->in.copy); 1090 1091 more: 1092 conn->in.copied = 0; 1093 rc = 0; 1094 1095 if (unlikely(conn->suspend_rx)) { 1096 debug_tcp("conn %d Rx suspended!\n", conn->id); 1097 return 0; 1098 } 1099 1100 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER || 1101 conn->in_progress == IN_PROGRESS_HEADER_GATHER) { 1102 rc = iscsi_hdr_extract(conn); 1103 if (rc) { 1104 if (rc == -EAGAIN) 1105 goto nomore; 1106 else { 1107 iscsi_conn_failure(conn, rc); 1108 return 0; 1109 } 1110 } 1111 1112 /* 1113 * Verify and process incoming PDU header. 1114 */ 1115 rc = iscsi_hdr_recv(conn); 1116 if (!rc && conn->in.datalen) { 1117 if (conn->datadgst_en && 1118 conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) { 1119 BUG_ON(!conn->data_rx_tfm); 1120 crypto_digest_init(conn->data_rx_tfm); 1121 } 1122 conn->in_progress = IN_PROGRESS_DATA_RECV; 1123 } else if (rc) { 1124 iscsi_conn_failure(conn, rc); 1125 return 0; 1126 } 1127 } 1128 1129 if (conn->in_progress == IN_PROGRESS_DDIGEST_RECV) { 1130 debug_tcp("extra data_recv offset %d copy %d\n", 1131 conn->in.offset, conn->in.copy); 1132 if (conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) { 1133 uint32_t recv_digest; 1134 skb_copy_bits(conn->in.skb, conn->in.offset, 1135 &recv_digest, 4); 1136 conn->in.offset += 4; 1137 conn->in.copy -= 4; 1138 if (recv_digest != conn->in.datadgst) { 1139 debug_tcp("iscsi_tcp: data digest error!" 1140 "0x%x != 0x%x\n", recv_digest, 1141 conn->in.datadgst); 1142 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST); 1143 return 0; 1144 } else { 1145 debug_tcp("iscsi_tcp: data digest match!" 1146 "0x%x == 0x%x\n", recv_digest, 1147 conn->in.datadgst); 1148 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 1149 } 1150 } 1151 } 1152 1153 if (conn->in_progress == IN_PROGRESS_DATA_RECV && conn->in.copy) { 1154 1155 debug_tcp("data_recv offset %d copy %d\n", 1156 conn->in.offset, conn->in.copy); 1157 1158 rc = iscsi_data_recv(conn); 1159 if (rc) { 1160 if (rc == -EAGAIN) { 1161 rd_desc->count = conn->in.datalen - 1162 conn->in.ctask->data_count; 1163 goto again; 1164 } 1165 iscsi_conn_failure(conn, rc); 1166 return 0; 1167 } 1168 conn->in.copy -= conn->in.padding; 1169 conn->in.offset += conn->in.padding; 1170 if (conn->datadgst_en && 1171 conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) { 1172 if (conn->in.padding) { 1173 debug_tcp("padding -> %d\n", conn->in.padding); 1174 memset(pad, 0, conn->in.padding); 1175 sg_init_one(&sg, pad, conn->in.padding); 1176 crypto_digest_update(conn->data_rx_tfm, &sg, 1); 1177 } 1178 crypto_digest_final(conn->data_rx_tfm, 1179 (u8 *) & conn->in.datadgst); 1180 debug_tcp("rx digest 0x%x\n", conn->in.datadgst); 1181 conn->in_progress = IN_PROGRESS_DDIGEST_RECV; 1182 } else 1183 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 1184 } 1185 1186 debug_tcp("f, processed %d from out of %d padding %d\n", 1187 conn->in.offset - offset, (int)len, conn->in.padding); 1188 BUG_ON(conn->in.offset - offset > len); 1189 1190 if (conn->in.offset - offset != len) { 1191 debug_tcp("continue to process %d bytes\n", 1192 (int)len - (conn->in.offset - offset)); 1193 goto more; 1194 } 1195 1196 nomore: 1197 processed = conn->in.offset - offset; 1198 BUG_ON(processed == 0); 1199 return processed; 1200 1201 again: 1202 processed = conn->in.offset - offset; 1203 debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n", 1204 processed, (int)len, (int)rd_desc->count); 1205 BUG_ON(processed == 0); 1206 BUG_ON(processed > len); 1207 1208 conn->rxdata_octets += processed; 1209 return processed; 1210 } 1211 1212 static void 1213 iscsi_tcp_data_ready(struct sock *sk, int flag) 1214 { 1215 struct iscsi_conn *conn = sk->sk_user_data; 1216 read_descriptor_t rd_desc; 1217 1218 read_lock(&sk->sk_callback_lock); 1219 1220 /* use rd_desc to pass 'conn' to iscsi_tcp_data_recv */ 1221 rd_desc.arg.data = conn; 1222 rd_desc.count = 0; 1223 tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv); 1224 1225 read_unlock(&sk->sk_callback_lock); 1226 } 1227 1228 static void 1229 iscsi_tcp_state_change(struct sock *sk) 1230 { 1231 struct iscsi_conn *conn; 1232 struct iscsi_session *session; 1233 void (*old_state_change)(struct sock *); 1234 1235 read_lock(&sk->sk_callback_lock); 1236 1237 conn = (struct iscsi_conn*)sk->sk_user_data; 1238 session = conn->session; 1239 1240 if (sk->sk_state == TCP_CLOSE_WAIT || 1241 sk->sk_state == TCP_CLOSE) { 1242 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n"); 1243 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1244 } 1245 1246 old_state_change = conn->old_state_change; 1247 1248 read_unlock(&sk->sk_callback_lock); 1249 1250 old_state_change(sk); 1251 } 1252 1253 /** 1254 * iscsi_write_space - Called when more output buffer space is available 1255 * @sk: socket space is available for 1256 **/ 1257 static void 1258 iscsi_write_space(struct sock *sk) 1259 { 1260 struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data; 1261 conn->old_write_space(sk); 1262 debug_tcp("iscsi_write_space: cid %d\n", conn->id); 1263 clear_bit(SUSPEND_BIT, &conn->suspend_tx); 1264 schedule_work(&conn->xmitwork); 1265 } 1266 1267 static void 1268 iscsi_conn_set_callbacks(struct iscsi_conn *conn) 1269 { 1270 struct sock *sk = conn->sock->sk; 1271 1272 /* assign new callbacks */ 1273 write_lock_bh(&sk->sk_callback_lock); 1274 sk->sk_user_data = conn; 1275 conn->old_data_ready = sk->sk_data_ready; 1276 conn->old_state_change = sk->sk_state_change; 1277 conn->old_write_space = sk->sk_write_space; 1278 sk->sk_data_ready = iscsi_tcp_data_ready; 1279 sk->sk_state_change = iscsi_tcp_state_change; 1280 sk->sk_write_space = iscsi_write_space; 1281 write_unlock_bh(&sk->sk_callback_lock); 1282 } 1283 1284 static void 1285 iscsi_conn_restore_callbacks(struct iscsi_conn *conn) 1286 { 1287 struct sock *sk = conn->sock->sk; 1288 1289 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */ 1290 write_lock_bh(&sk->sk_callback_lock); 1291 sk->sk_user_data = NULL; 1292 sk->sk_data_ready = conn->old_data_ready; 1293 sk->sk_state_change = conn->old_state_change; 1294 sk->sk_write_space = conn->old_write_space; 1295 sk->sk_no_check = 0; 1296 write_unlock_bh(&sk->sk_callback_lock); 1297 } 1298 1299 /** 1300 * iscsi_send - generic send routine 1301 * @sk: kernel's socket 1302 * @buf: buffer to write from 1303 * @size: actual size to write 1304 * @flags: socket's flags 1305 * 1306 * Notes: 1307 * depending on buffer will use tcp_sendpage() or tcp_sendmsg(). 1308 * buf->sg.offset == -1 tells us that buffer is non S/G and forces 1309 * to use tcp_sendmsg(). 1310 */ 1311 static inline int 1312 iscsi_send(struct socket *sk, struct iscsi_buf *buf, int size, int flags) 1313 { 1314 int res; 1315 1316 if ((int)buf->sg.offset >= 0) { 1317 int offset = buf->sg.offset + buf->sent; 1318 1319 /* tcp_sendpage */ 1320 res = sk->ops->sendpage(sk, buf->sg.page, offset, size, flags); 1321 } else { 1322 struct msghdr msg; 1323 1324 buf->iov.iov_base = iscsi_buf_iov_base(buf); 1325 buf->iov.iov_len = size; 1326 1327 memset(&msg, 0, sizeof(struct msghdr)); 1328 1329 /* tcp_sendmsg */ 1330 res = kernel_sendmsg(sk, &msg, &buf->iov, 1, size); 1331 } 1332 1333 return res; 1334 } 1335 1336 /** 1337 * iscsi_sendhdr - send PDU Header via tcp_sendpage() 1338 * @conn: iscsi connection 1339 * @buf: buffer to write from 1340 * @datalen: lenght of data to be sent after the header 1341 * 1342 * Notes: 1343 * (Tx, Fast Path) 1344 **/ 1345 static inline int 1346 iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen) 1347 { 1348 struct socket *sk = conn->sock; 1349 int flags = 0; /* MSG_DONTWAIT; */ 1350 int res, size; 1351 1352 size = buf->sg.length - buf->sent; 1353 BUG_ON(buf->sent + size > buf->sg.length); 1354 if (buf->sent + size != buf->sg.length || datalen) 1355 flags |= MSG_MORE; 1356 1357 res = iscsi_send(sk, buf, size, flags); 1358 debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res); 1359 if (res >= 0) { 1360 conn->txdata_octets += res; 1361 buf->sent += res; 1362 if (size != res) 1363 return -EAGAIN; 1364 return 0; 1365 } else if (res == -EAGAIN) { 1366 conn->sendpage_failures_cnt++; 1367 set_bit(SUSPEND_BIT, &conn->suspend_tx); 1368 } else if (res == -EPIPE) 1369 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1370 1371 return res; 1372 } 1373 1374 /** 1375 * iscsi_sendpage - send one page of iSCSI Data-Out. 1376 * @conn: iscsi connection 1377 * @buf: buffer to write from 1378 * @count: remaining data 1379 * @sent: number of bytes sent 1380 * 1381 * Notes: 1382 * (Tx, Fast Path) 1383 **/ 1384 static inline int 1385 iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf, 1386 int *count, int *sent) 1387 { 1388 struct socket *sk = conn->sock; 1389 int flags = 0; /* MSG_DONTWAIT; */ 1390 int res, size; 1391 1392 size = buf->sg.length - buf->sent; 1393 BUG_ON(buf->sent + size > buf->sg.length); 1394 if (size > *count) 1395 size = *count; 1396 if (buf->sent + size != buf->sg.length || *count != size) 1397 flags |= MSG_MORE; 1398 1399 res = iscsi_send(sk, buf, size, flags); 1400 debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n", 1401 size, buf->sent, *count, *sent, res); 1402 if (res >= 0) { 1403 conn->txdata_octets += res; 1404 buf->sent += res; 1405 *count -= res; 1406 *sent += res; 1407 if (size != res) 1408 return -EAGAIN; 1409 return 0; 1410 } else if (res == -EAGAIN) { 1411 conn->sendpage_failures_cnt++; 1412 set_bit(SUSPEND_BIT, &conn->suspend_tx); 1413 } else if (res == -EPIPE) 1414 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1415 1416 return res; 1417 } 1418 1419 static inline void 1420 iscsi_data_digest_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1421 { 1422 BUG_ON(!conn->data_tx_tfm); 1423 crypto_digest_init(conn->data_tx_tfm); 1424 ctask->digest_count = 4; 1425 } 1426 1427 static inline void 1428 iscsi_buf_data_digest_update(struct iscsi_conn *conn, struct iscsi_buf *buf) 1429 { 1430 struct scatterlist sg; 1431 1432 if (buf->sg.offset != -1) 1433 crypto_digest_update(conn->data_tx_tfm, &buf->sg, 1); 1434 else { 1435 sg_init_one(&sg, (char *)buf->sg.page, buf->sg.length); 1436 crypto_digest_update(conn->data_tx_tfm, &sg, 1); 1437 } 1438 } 1439 1440 static inline int 1441 iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1442 struct iscsi_buf *buf, uint32_t *digest, int final) 1443 { 1444 int rc = 0; 1445 int sent = 0; 1446 1447 if (final) 1448 crypto_digest_final(conn->data_tx_tfm, (u8*)digest); 1449 1450 iscsi_buf_init_virt(buf, (char*)digest, 4); 1451 rc = iscsi_sendpage(conn, buf, &ctask->digest_count, &sent); 1452 if (rc) { 1453 ctask->datadigest = *digest; 1454 ctask->xmstate |= XMSTATE_DATA_DIGEST; 1455 } else 1456 ctask->digest_count = 4; 1457 return rc; 1458 } 1459 1460 /** 1461 * iscsi_solicit_data_cont - initialize next Data-Out 1462 * @conn: iscsi connection 1463 * @ctask: scsi command task 1464 * @r2t: R2T info 1465 * @left: bytes left to transfer 1466 * 1467 * Notes: 1468 * Initialize next Data-Out within this R2T sequence and continue 1469 * to process next Scatter-Gather element(if any) of this SCSI command. 1470 * 1471 * Called under connection lock. 1472 **/ 1473 static void 1474 iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1475 struct iscsi_r2t_info *r2t, int left) 1476 { 1477 struct iscsi_data *hdr; 1478 struct iscsi_data_task *dtask; 1479 struct scsi_cmnd *sc = ctask->sc; 1480 int new_offset; 1481 1482 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 1483 BUG_ON(!dtask); 1484 hdr = &dtask->hdr; 1485 memset(hdr, 0, sizeof(struct iscsi_data)); 1486 hdr->ttt = r2t->ttt; 1487 hdr->datasn = cpu_to_be32(r2t->solicit_datasn); 1488 r2t->solicit_datasn++; 1489 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 1490 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 1491 hdr->itt = ctask->hdr.itt; 1492 hdr->exp_statsn = r2t->exp_statsn; 1493 new_offset = r2t->data_offset + r2t->sent; 1494 hdr->offset = cpu_to_be32(new_offset); 1495 if (left > conn->max_xmit_dlength) { 1496 hton24(hdr->dlength, conn->max_xmit_dlength); 1497 r2t->data_count = conn->max_xmit_dlength; 1498 } else { 1499 hton24(hdr->dlength, left); 1500 r2t->data_count = left; 1501 hdr->flags = ISCSI_FLAG_CMD_FINAL; 1502 } 1503 conn->dataout_pdus_cnt++; 1504 1505 iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr, 1506 sizeof(struct iscsi_hdr)); 1507 1508 r2t->dtask = dtask; 1509 1510 if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) { 1511 BUG_ON(ctask->bad_sg == r2t->sg); 1512 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1513 r2t->sg += 1; 1514 } else 1515 iscsi_buf_init_iov(&ctask->sendbuf, 1516 (char*)sc->request_buffer + new_offset, 1517 r2t->data_count); 1518 1519 list_add(&dtask->item, &ctask->dataqueue); 1520 } 1521 1522 static void 1523 iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1524 { 1525 struct iscsi_data *hdr; 1526 struct iscsi_data_task *dtask; 1527 1528 dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC); 1529 BUG_ON(!dtask); 1530 hdr = &dtask->hdr; 1531 memset(hdr, 0, sizeof(struct iscsi_data)); 1532 hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG); 1533 hdr->datasn = cpu_to_be32(ctask->unsol_datasn); 1534 ctask->unsol_datasn++; 1535 hdr->opcode = ISCSI_OP_SCSI_DATA_OUT; 1536 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 1537 hdr->itt = ctask->hdr.itt; 1538 hdr->exp_statsn = cpu_to_be32(conn->exp_statsn); 1539 hdr->offset = cpu_to_be32(ctask->total_length - 1540 ctask->r2t_data_count - 1541 ctask->unsol_count); 1542 if (ctask->unsol_count > conn->max_xmit_dlength) { 1543 hton24(hdr->dlength, conn->max_xmit_dlength); 1544 ctask->data_count = conn->max_xmit_dlength; 1545 hdr->flags = 0; 1546 } else { 1547 hton24(hdr->dlength, ctask->unsol_count); 1548 ctask->data_count = ctask->unsol_count; 1549 hdr->flags = ISCSI_FLAG_CMD_FINAL; 1550 } 1551 1552 iscsi_buf_init_virt(&ctask->headbuf, (char*)hdr, 1553 sizeof(struct iscsi_hdr)); 1554 1555 list_add(&dtask->item, &ctask->dataqueue); 1556 1557 ctask->dtask = dtask; 1558 } 1559 1560 /** 1561 * iscsi_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands 1562 * @conn: iscsi connection 1563 * @ctask: scsi command task 1564 * @sc: scsi command 1565 **/ 1566 static void 1567 iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask, 1568 struct scsi_cmnd *sc) 1569 { 1570 struct iscsi_session *session = conn->session; 1571 1572 BUG_ON(__kfifo_len(ctask->r2tqueue)); 1573 1574 ctask->sc = sc; 1575 ctask->conn = conn; 1576 ctask->hdr.opcode = ISCSI_OP_SCSI_CMD; 1577 ctask->hdr.flags = ISCSI_ATTR_SIMPLE; 1578 int_to_scsilun(sc->device->lun, (struct scsi_lun *)ctask->hdr.lun); 1579 ctask->hdr.itt = ctask->itt | (conn->id << CID_SHIFT) | 1580 (session->age << AGE_SHIFT); 1581 ctask->hdr.data_length = cpu_to_be32(sc->request_bufflen); 1582 ctask->hdr.cmdsn = cpu_to_be32(session->cmdsn); session->cmdsn++; 1583 ctask->hdr.exp_statsn = cpu_to_be32(conn->exp_statsn); 1584 memcpy(ctask->hdr.cdb, sc->cmnd, sc->cmd_len); 1585 memset(&ctask->hdr.cdb[sc->cmd_len], 0, MAX_COMMAND_SIZE - sc->cmd_len); 1586 1587 ctask->mtask = NULL; 1588 ctask->sent = 0; 1589 ctask->sg_count = 0; 1590 1591 ctask->total_length = sc->request_bufflen; 1592 1593 if (sc->sc_data_direction == DMA_TO_DEVICE) { 1594 ctask->exp_r2tsn = 0; 1595 ctask->hdr.flags |= ISCSI_FLAG_CMD_WRITE; 1596 BUG_ON(ctask->total_length == 0); 1597 if (sc->use_sg) { 1598 struct scatterlist *sg = sc->request_buffer; 1599 1600 iscsi_buf_init_sg(&ctask->sendbuf, 1601 &sg[ctask->sg_count++]); 1602 ctask->sg = sg; 1603 ctask->bad_sg = sg + sc->use_sg; 1604 } else { 1605 iscsi_buf_init_iov(&ctask->sendbuf, sc->request_buffer, 1606 sc->request_bufflen); 1607 } 1608 1609 /* 1610 * Write counters: 1611 * 1612 * imm_count bytes to be sent right after 1613 * SCSI PDU Header 1614 * 1615 * unsol_count bytes(as Data-Out) to be sent 1616 * without R2T ack right after 1617 * immediate data 1618 * 1619 * r2t_data_count bytes to be sent via R2T ack's 1620 * 1621 * pad_count bytes to be sent as zero-padding 1622 */ 1623 ctask->imm_count = 0; 1624 ctask->unsol_count = 0; 1625 ctask->unsol_datasn = 0; 1626 ctask->xmstate = XMSTATE_W_HDR; 1627 /* calculate write padding */ 1628 ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1); 1629 if (ctask->pad_count) { 1630 ctask->pad_count = ISCSI_PAD_LEN - ctask->pad_count; 1631 debug_scsi("write padding %d bytes\n", 1632 ctask->pad_count); 1633 ctask->xmstate |= XMSTATE_W_PAD; 1634 } 1635 if (session->imm_data_en) { 1636 if (ctask->total_length >= session->first_burst) 1637 ctask->imm_count = min(session->first_burst, 1638 conn->max_xmit_dlength); 1639 else 1640 ctask->imm_count = min(ctask->total_length, 1641 conn->max_xmit_dlength); 1642 hton24(ctask->hdr.dlength, ctask->imm_count); 1643 ctask->xmstate |= XMSTATE_IMM_DATA; 1644 } else 1645 zero_data(ctask->hdr.dlength); 1646 1647 if (!session->initial_r2t_en) 1648 ctask->unsol_count = min(session->first_burst, 1649 ctask->total_length) - ctask->imm_count; 1650 if (!ctask->unsol_count) 1651 /* No unsolicit Data-Out's */ 1652 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL; 1653 else 1654 ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT; 1655 1656 ctask->r2t_data_count = ctask->total_length - 1657 ctask->imm_count - 1658 ctask->unsol_count; 1659 1660 debug_scsi("cmd [itt %x total %d imm %d imm_data %d " 1661 "r2t_data %d]\n", 1662 ctask->itt, ctask->total_length, ctask->imm_count, 1663 ctask->unsol_count, ctask->r2t_data_count); 1664 } else { 1665 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL; 1666 if (sc->sc_data_direction == DMA_FROM_DEVICE) 1667 ctask->hdr.flags |= ISCSI_FLAG_CMD_READ; 1668 ctask->datasn = 0; 1669 ctask->xmstate = XMSTATE_R_HDR; 1670 zero_data(ctask->hdr.dlength); 1671 } 1672 1673 iscsi_buf_init_virt(&ctask->headbuf, (char*)&ctask->hdr, 1674 sizeof(struct iscsi_hdr)); 1675 conn->scsicmd_pdus_cnt++; 1676 } 1677 1678 /** 1679 * iscsi_mtask_xmit - xmit management(immediate) task 1680 * @conn: iscsi connection 1681 * @mtask: task management task 1682 * 1683 * Notes: 1684 * The function can return -EAGAIN in which case caller must 1685 * call it again later, or recover. '0' return code means successful 1686 * xmit. 1687 * 1688 * Management xmit state machine consists of two states: 1689 * IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress 1690 * IN_PROGRESS_IMM_DATA - PDU Data xmit in progress 1691 **/ 1692 static int 1693 iscsi_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask) 1694 { 1695 1696 debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n", 1697 conn->id, mtask->xmstate, mtask->itt); 1698 1699 if (mtask->xmstate & XMSTATE_IMM_HDR) { 1700 mtask->xmstate &= ~XMSTATE_IMM_HDR; 1701 if (mtask->data_count) 1702 mtask->xmstate |= XMSTATE_IMM_DATA; 1703 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE && 1704 conn->stop_stage != STOP_CONN_RECOVER && 1705 conn->hdrdgst_en) 1706 iscsi_hdr_digest(conn, &mtask->headbuf, 1707 (u8*)mtask->hdrext); 1708 if (iscsi_sendhdr(conn, &mtask->headbuf, mtask->data_count)) { 1709 mtask->xmstate |= XMSTATE_IMM_HDR; 1710 if (mtask->data_count) 1711 mtask->xmstate &= ~XMSTATE_IMM_DATA; 1712 return -EAGAIN; 1713 } 1714 } 1715 1716 if (mtask->xmstate & XMSTATE_IMM_DATA) { 1717 BUG_ON(!mtask->data_count); 1718 mtask->xmstate &= ~XMSTATE_IMM_DATA; 1719 /* FIXME: implement. 1720 * Virtual buffer could be spreaded across multiple pages... 1721 */ 1722 do { 1723 if (iscsi_sendpage(conn, &mtask->sendbuf, 1724 &mtask->data_count, &mtask->sent)) { 1725 mtask->xmstate |= XMSTATE_IMM_DATA; 1726 return -EAGAIN; 1727 } 1728 } while (mtask->data_count); 1729 } 1730 1731 BUG_ON(mtask->xmstate != XMSTATE_IDLE); 1732 return 0; 1733 } 1734 1735 static inline int 1736 handle_xmstate_r_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1737 { 1738 ctask->xmstate &= ~XMSTATE_R_HDR; 1739 if (conn->hdrdgst_en) 1740 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext); 1741 if (!iscsi_sendhdr(conn, &ctask->headbuf, 0)) { 1742 BUG_ON(ctask->xmstate != XMSTATE_IDLE); 1743 return 0; /* wait for Data-In */ 1744 } 1745 ctask->xmstate |= XMSTATE_R_HDR; 1746 return -EAGAIN; 1747 } 1748 1749 static inline int 1750 handle_xmstate_w_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1751 { 1752 ctask->xmstate &= ~XMSTATE_W_HDR; 1753 if (conn->hdrdgst_en) 1754 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext); 1755 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->imm_count)) { 1756 ctask->xmstate |= XMSTATE_W_HDR; 1757 return -EAGAIN; 1758 } 1759 return 0; 1760 } 1761 1762 static inline int 1763 handle_xmstate_data_digest(struct iscsi_conn *conn, 1764 struct iscsi_cmd_task *ctask) 1765 { 1766 ctask->xmstate &= ~XMSTATE_DATA_DIGEST; 1767 debug_tcp("resent data digest 0x%x\n", ctask->datadigest); 1768 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 1769 &ctask->datadigest, 0)) { 1770 ctask->xmstate |= XMSTATE_DATA_DIGEST; 1771 debug_tcp("resent data digest 0x%x fail!\n", 1772 ctask->datadigest); 1773 return -EAGAIN; 1774 } 1775 return 0; 1776 } 1777 1778 static inline int 1779 handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1780 { 1781 BUG_ON(!ctask->imm_count); 1782 ctask->xmstate &= ~XMSTATE_IMM_DATA; 1783 1784 if (conn->datadgst_en) { 1785 iscsi_data_digest_init(conn, ctask); 1786 ctask->immdigest = 0; 1787 } 1788 1789 for (;;) { 1790 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->imm_count, 1791 &ctask->sent)) { 1792 ctask->xmstate |= XMSTATE_IMM_DATA; 1793 if (conn->datadgst_en) { 1794 crypto_digest_final(conn->data_tx_tfm, 1795 (u8*)&ctask->immdigest); 1796 debug_tcp("tx imm sendpage fail 0x%x\n", 1797 ctask->datadigest); 1798 } 1799 return -EAGAIN; 1800 } 1801 if (conn->datadgst_en) 1802 iscsi_buf_data_digest_update(conn, &ctask->sendbuf); 1803 1804 if (!ctask->imm_count) 1805 break; 1806 iscsi_buf_init_sg(&ctask->sendbuf, 1807 &ctask->sg[ctask->sg_count++]); 1808 } 1809 1810 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) { 1811 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 1812 &ctask->immdigest, 1)) { 1813 debug_tcp("sending imm digest 0x%x fail!\n", 1814 ctask->immdigest); 1815 return -EAGAIN; 1816 } 1817 debug_tcp("sending imm digest 0x%x\n", ctask->immdigest); 1818 } 1819 1820 return 0; 1821 } 1822 1823 static inline int 1824 handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1825 { 1826 struct iscsi_data_task *dtask; 1827 1828 ctask->xmstate |= XMSTATE_UNS_DATA; 1829 if (ctask->xmstate & XMSTATE_UNS_INIT) { 1830 iscsi_unsolicit_data_init(conn, ctask); 1831 BUG_ON(!ctask->dtask); 1832 dtask = ctask->dtask; 1833 if (conn->hdrdgst_en) 1834 iscsi_hdr_digest(conn, &ctask->headbuf, 1835 (u8*)dtask->hdrext); 1836 ctask->xmstate &= ~XMSTATE_UNS_INIT; 1837 } 1838 if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->data_count)) { 1839 ctask->xmstate &= ~XMSTATE_UNS_DATA; 1840 ctask->xmstate |= XMSTATE_UNS_HDR; 1841 return -EAGAIN; 1842 } 1843 1844 debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n", 1845 ctask->itt, ctask->unsol_count, ctask->sent); 1846 return 0; 1847 } 1848 1849 static inline int 1850 handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1851 { 1852 struct iscsi_data_task *dtask = ctask->dtask; 1853 1854 BUG_ON(!ctask->data_count); 1855 ctask->xmstate &= ~XMSTATE_UNS_DATA; 1856 1857 if (conn->datadgst_en) { 1858 iscsi_data_digest_init(conn, ctask); 1859 dtask->digest = 0; 1860 } 1861 1862 for (;;) { 1863 int start = ctask->sent; 1864 1865 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->data_count, 1866 &ctask->sent)) { 1867 ctask->unsol_count -= ctask->sent - start; 1868 ctask->xmstate |= XMSTATE_UNS_DATA; 1869 /* will continue with this ctask later.. */ 1870 if (conn->datadgst_en) { 1871 crypto_digest_final(conn->data_tx_tfm, 1872 (u8 *)&dtask->digest); 1873 debug_tcp("tx uns data fail 0x%x\n", 1874 dtask->digest); 1875 } 1876 return -EAGAIN; 1877 } 1878 1879 BUG_ON(ctask->sent > ctask->total_length); 1880 ctask->unsol_count -= ctask->sent - start; 1881 1882 /* 1883 * XXX:we may run here with un-initial sendbuf. 1884 * so pass it 1885 */ 1886 if (conn->datadgst_en && ctask->sent - start > 0) 1887 iscsi_buf_data_digest_update(conn, &ctask->sendbuf); 1888 1889 if (!ctask->data_count) 1890 break; 1891 iscsi_buf_init_sg(&ctask->sendbuf, 1892 &ctask->sg[ctask->sg_count++]); 1893 } 1894 BUG_ON(ctask->unsol_count < 0); 1895 1896 /* 1897 * Done with the Data-Out. Next, check if we need 1898 * to send another unsolicited Data-Out. 1899 */ 1900 if (ctask->unsol_count) { 1901 if (conn->datadgst_en) { 1902 if (iscsi_digest_final_send(conn, ctask, 1903 &dtask->digestbuf, 1904 &dtask->digest, 1)) { 1905 debug_tcp("send uns digest 0x%x fail\n", 1906 dtask->digest); 1907 return -EAGAIN; 1908 } 1909 debug_tcp("sending uns digest 0x%x, more uns\n", 1910 dtask->digest); 1911 } 1912 ctask->xmstate |= XMSTATE_UNS_INIT; 1913 return 1; 1914 } 1915 1916 if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) { 1917 if (iscsi_digest_final_send(conn, ctask, 1918 &dtask->digestbuf, 1919 &dtask->digest, 1)) { 1920 debug_tcp("send last uns digest 0x%x fail\n", 1921 dtask->digest); 1922 return -EAGAIN; 1923 } 1924 debug_tcp("sending uns digest 0x%x\n",dtask->digest); 1925 } 1926 1927 return 0; 1928 } 1929 1930 static inline int 1931 handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 1932 { 1933 struct iscsi_session *session = conn->session; 1934 struct iscsi_r2t_info *r2t = ctask->r2t; 1935 struct iscsi_data_task *dtask = r2t->dtask; 1936 int left; 1937 1938 ctask->xmstate &= ~XMSTATE_SOL_DATA; 1939 ctask->dtask = dtask; 1940 1941 if (conn->datadgst_en) { 1942 iscsi_data_digest_init(conn, ctask); 1943 dtask->digest = 0; 1944 } 1945 solicit_again: 1946 /* 1947 * send Data-Out whitnin this R2T sequence. 1948 */ 1949 if (!r2t->data_count) 1950 goto data_out_done; 1951 1952 if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) { 1953 ctask->xmstate |= XMSTATE_SOL_DATA; 1954 /* will continue with this ctask later.. */ 1955 if (conn->datadgst_en) { 1956 crypto_digest_final(conn->data_tx_tfm, 1957 (u8 *)&dtask->digest); 1958 debug_tcp("r2t data send fail 0x%x\n", dtask->digest); 1959 } 1960 return -EAGAIN; 1961 } 1962 1963 BUG_ON(r2t->data_count < 0); 1964 if (conn->datadgst_en) 1965 iscsi_buf_data_digest_update(conn, &r2t->sendbuf); 1966 1967 if (r2t->data_count) { 1968 BUG_ON(ctask->sc->use_sg == 0); 1969 if (!iscsi_buf_left(&r2t->sendbuf)) { 1970 BUG_ON(ctask->bad_sg == r2t->sg); 1971 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg); 1972 r2t->sg += 1; 1973 } 1974 goto solicit_again; 1975 } 1976 1977 data_out_done: 1978 /* 1979 * Done with this Data-Out. Next, check if we have 1980 * to send another Data-Out for this R2T. 1981 */ 1982 BUG_ON(r2t->data_length - r2t->sent < 0); 1983 left = r2t->data_length - r2t->sent; 1984 if (left) { 1985 if (conn->datadgst_en) { 1986 if (iscsi_digest_final_send(conn, ctask, 1987 &dtask->digestbuf, 1988 &dtask->digest, 1)) { 1989 debug_tcp("send r2t data digest 0x%x" 1990 "fail\n", dtask->digest); 1991 return -EAGAIN; 1992 } 1993 debug_tcp("r2t data send digest 0x%x\n", 1994 dtask->digest); 1995 } 1996 iscsi_solicit_data_cont(conn, ctask, r2t, left); 1997 ctask->xmstate |= XMSTATE_SOL_DATA; 1998 ctask->xmstate &= ~XMSTATE_SOL_HDR; 1999 return 1; 2000 } 2001 2002 /* 2003 * Done with this R2T. Check if there are more 2004 * outstanding R2Ts ready to be processed. 2005 */ 2006 BUG_ON(ctask->r2t_data_count - r2t->data_length < 0); 2007 if (conn->datadgst_en) { 2008 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf, 2009 &dtask->digest, 1)) { 2010 debug_tcp("send last r2t data digest 0x%x" 2011 "fail\n", dtask->digest); 2012 return -EAGAIN; 2013 } 2014 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest); 2015 } 2016 2017 ctask->r2t_data_count -= r2t->data_length; 2018 ctask->r2t = NULL; 2019 spin_lock_bh(&session->lock); 2020 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*)); 2021 spin_unlock_bh(&session->lock); 2022 if (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) { 2023 ctask->r2t = r2t; 2024 ctask->xmstate |= XMSTATE_SOL_DATA; 2025 ctask->xmstate &= ~XMSTATE_SOL_HDR; 2026 return 1; 2027 } 2028 2029 return 0; 2030 } 2031 2032 static inline int 2033 handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 2034 { 2035 struct iscsi_data_task *dtask = ctask->dtask; 2036 int sent; 2037 2038 ctask->xmstate &= ~XMSTATE_W_PAD; 2039 iscsi_buf_init_virt(&ctask->sendbuf, (char*)&ctask->pad, 2040 ctask->pad_count); 2041 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->pad_count, &sent)) { 2042 ctask->xmstate |= XMSTATE_W_PAD; 2043 return -EAGAIN; 2044 } 2045 2046 if (conn->datadgst_en) { 2047 iscsi_buf_data_digest_update(conn, &ctask->sendbuf); 2048 /* imm data? */ 2049 if (!dtask) { 2050 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf, 2051 &ctask->immdigest, 1)) { 2052 debug_tcp("send padding digest 0x%x" 2053 "fail!\n", ctask->immdigest); 2054 return -EAGAIN; 2055 } 2056 debug_tcp("done with padding, digest 0x%x\n", 2057 ctask->datadigest); 2058 } else { 2059 if (iscsi_digest_final_send(conn, ctask, 2060 &dtask->digestbuf, 2061 &dtask->digest, 1)) { 2062 debug_tcp("send padding digest 0x%x" 2063 "fail\n", dtask->digest); 2064 return -EAGAIN; 2065 } 2066 debug_tcp("done with padding, digest 0x%x\n", 2067 dtask->digest); 2068 } 2069 } 2070 2071 return 0; 2072 } 2073 2074 static int 2075 iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 2076 { 2077 int rc = 0; 2078 2079 debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n", 2080 conn->id, ctask->xmstate, ctask->itt); 2081 2082 /* 2083 * serialize with TMF AbortTask 2084 */ 2085 if (ctask->mtask) 2086 return rc; 2087 2088 if (ctask->xmstate & XMSTATE_R_HDR) { 2089 rc = handle_xmstate_r_hdr(conn, ctask); 2090 return rc; 2091 } 2092 2093 if (ctask->xmstate & XMSTATE_W_HDR) { 2094 rc = handle_xmstate_w_hdr(conn, ctask); 2095 if (rc) 2096 return rc; 2097 } 2098 2099 /* XXX: for data digest xmit recover */ 2100 if (ctask->xmstate & XMSTATE_DATA_DIGEST) { 2101 rc = handle_xmstate_data_digest(conn, ctask); 2102 if (rc) 2103 return rc; 2104 } 2105 2106 if (ctask->xmstate & XMSTATE_IMM_DATA) { 2107 rc = handle_xmstate_imm_data(conn, ctask); 2108 if (rc) 2109 return rc; 2110 } 2111 2112 if (ctask->xmstate & XMSTATE_UNS_HDR) { 2113 BUG_ON(!ctask->unsol_count); 2114 ctask->xmstate &= ~XMSTATE_UNS_HDR; 2115 unsolicit_head_again: 2116 rc = handle_xmstate_uns_hdr(conn, ctask); 2117 if (rc) 2118 return rc; 2119 } 2120 2121 if (ctask->xmstate & XMSTATE_UNS_DATA) { 2122 rc = handle_xmstate_uns_data(conn, ctask); 2123 if (rc == 1) 2124 goto unsolicit_head_again; 2125 else if (rc) 2126 return rc; 2127 goto done; 2128 } 2129 2130 if (ctask->xmstate & XMSTATE_SOL_HDR) { 2131 struct iscsi_r2t_info *r2t; 2132 2133 ctask->xmstate &= ~XMSTATE_SOL_HDR; 2134 ctask->xmstate |= XMSTATE_SOL_DATA; 2135 if (!ctask->r2t) 2136 __kfifo_get(ctask->r2tqueue, (void*)&ctask->r2t, 2137 sizeof(void*)); 2138 solicit_head_again: 2139 r2t = ctask->r2t; 2140 if (conn->hdrdgst_en) 2141 iscsi_hdr_digest(conn, &r2t->headbuf, 2142 (u8*)r2t->dtask->hdrext); 2143 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) { 2144 ctask->xmstate &= ~XMSTATE_SOL_DATA; 2145 ctask->xmstate |= XMSTATE_SOL_HDR; 2146 return -EAGAIN; 2147 } 2148 2149 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n", 2150 r2t->solicit_datasn - 1, ctask->itt, r2t->data_count, 2151 r2t->sent); 2152 } 2153 2154 if (ctask->xmstate & XMSTATE_SOL_DATA) { 2155 rc = handle_xmstate_sol_data(conn, ctask); 2156 if (rc == 1) 2157 goto solicit_head_again; 2158 if (rc) 2159 return rc; 2160 } 2161 2162 done: 2163 /* 2164 * Last thing to check is whether we need to send write 2165 * padding. Note that we check for xmstate equality, not just the bit. 2166 */ 2167 if (ctask->xmstate == XMSTATE_W_PAD) 2168 rc = handle_xmstate_w_pad(conn, ctask); 2169 2170 return rc; 2171 } 2172 2173 /** 2174 * iscsi_data_xmit - xmit any command into the scheduled connection 2175 * @conn: iscsi connection 2176 * 2177 * Notes: 2178 * The function can return -EAGAIN in which case the caller must 2179 * re-schedule it again later or recover. '0' return code means 2180 * successful xmit. 2181 **/ 2182 static int 2183 iscsi_data_xmit(struct iscsi_conn *conn) 2184 { 2185 if (unlikely(conn->suspend_tx)) { 2186 debug_tcp("conn %d Tx suspended!\n", conn->id); 2187 return 0; 2188 } 2189 2190 /* 2191 * Transmit in the following order: 2192 * 2193 * 1) un-finished xmit (ctask or mtask) 2194 * 2) immediate control PDUs 2195 * 3) write data 2196 * 4) SCSI commands 2197 * 5) non-immediate control PDUs 2198 * 2199 * No need to lock around __kfifo_get as long as 2200 * there's one producer and one consumer. 2201 */ 2202 2203 BUG_ON(conn->ctask && conn->mtask); 2204 2205 if (conn->ctask) { 2206 if (iscsi_ctask_xmit(conn, conn->ctask)) 2207 goto again; 2208 /* done with this in-progress ctask */ 2209 conn->ctask = NULL; 2210 } 2211 if (conn->mtask) { 2212 if (iscsi_mtask_xmit(conn, conn->mtask)) 2213 goto again; 2214 /* done with this in-progress mtask */ 2215 conn->mtask = NULL; 2216 } 2217 2218 /* process immediate first */ 2219 if (unlikely(__kfifo_len(conn->immqueue))) { 2220 struct iscsi_session *session = conn->session; 2221 while (__kfifo_get(conn->immqueue, (void*)&conn->mtask, 2222 sizeof(void*))) { 2223 if (iscsi_mtask_xmit(conn, conn->mtask)) 2224 goto again; 2225 2226 if (conn->mtask->hdr.itt == 2227 cpu_to_be32(ISCSI_RESERVED_TAG)) { 2228 spin_lock_bh(&session->lock); 2229 __kfifo_put(session->mgmtpool.queue, 2230 (void*)&conn->mtask, sizeof(void*)); 2231 spin_unlock_bh(&session->lock); 2232 } 2233 } 2234 /* done with this mtask */ 2235 conn->mtask = NULL; 2236 } 2237 2238 /* process write queue */ 2239 while (__kfifo_get(conn->writequeue, (void*)&conn->ctask, 2240 sizeof(void*))) { 2241 if (iscsi_ctask_xmit(conn, conn->ctask)) 2242 goto again; 2243 } 2244 2245 /* process command queue */ 2246 while (__kfifo_get(conn->xmitqueue, (void*)&conn->ctask, 2247 sizeof(void*))) { 2248 if (iscsi_ctask_xmit(conn, conn->ctask)) 2249 goto again; 2250 } 2251 /* done with this ctask */ 2252 conn->ctask = NULL; 2253 2254 /* process the rest control plane PDUs, if any */ 2255 if (unlikely(__kfifo_len(conn->mgmtqueue))) { 2256 struct iscsi_session *session = conn->session; 2257 2258 while (__kfifo_get(conn->mgmtqueue, (void*)&conn->mtask, 2259 sizeof(void*))) { 2260 if (iscsi_mtask_xmit(conn, conn->mtask)) 2261 goto again; 2262 2263 if (conn->mtask->hdr.itt == 2264 cpu_to_be32(ISCSI_RESERVED_TAG)) { 2265 spin_lock_bh(&session->lock); 2266 __kfifo_put(session->mgmtpool.queue, 2267 (void*)&conn->mtask, 2268 sizeof(void*)); 2269 spin_unlock_bh(&session->lock); 2270 } 2271 } 2272 /* done with this mtask */ 2273 conn->mtask = NULL; 2274 } 2275 2276 return 0; 2277 2278 again: 2279 if (unlikely(conn->suspend_tx)) 2280 return 0; 2281 2282 return -EAGAIN; 2283 } 2284 2285 static void 2286 iscsi_xmitworker(void *data) 2287 { 2288 struct iscsi_conn *conn = data; 2289 2290 /* 2291 * serialize Xmit worker on a per-connection basis. 2292 */ 2293 down(&conn->xmitsema); 2294 if (iscsi_data_xmit(conn)) 2295 schedule_work(&conn->xmitwork); 2296 up(&conn->xmitsema); 2297 } 2298 2299 #define FAILURE_BAD_HOST 1 2300 #define FAILURE_SESSION_FAILED 2 2301 #define FAILURE_SESSION_FREED 3 2302 #define FAILURE_WINDOW_CLOSED 4 2303 #define FAILURE_SESSION_TERMINATE 5 2304 2305 static int 2306 iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *)) 2307 { 2308 struct Scsi_Host *host; 2309 int reason = 0; 2310 struct iscsi_session *session; 2311 struct iscsi_conn *conn = NULL; 2312 struct iscsi_cmd_task *ctask = NULL; 2313 2314 sc->scsi_done = done; 2315 sc->result = 0; 2316 2317 host = sc->device->host; 2318 session = iscsi_hostdata(host->hostdata); 2319 BUG_ON(host != session->host); 2320 2321 spin_lock(&session->lock); 2322 2323 if (session->state != ISCSI_STATE_LOGGED_IN) { 2324 if (session->state == ISCSI_STATE_FAILED) { 2325 reason = FAILURE_SESSION_FAILED; 2326 goto reject; 2327 } else if (session->state == ISCSI_STATE_TERMINATE) { 2328 reason = FAILURE_SESSION_TERMINATE; 2329 goto fault; 2330 } 2331 reason = FAILURE_SESSION_FREED; 2332 goto fault; 2333 } 2334 2335 /* 2336 * Check for iSCSI window and take care of CmdSN wrap-around 2337 */ 2338 if ((int)(session->max_cmdsn - session->cmdsn) < 0) { 2339 reason = FAILURE_WINDOW_CLOSED; 2340 goto reject; 2341 } 2342 2343 conn = session->leadconn; 2344 2345 __kfifo_get(session->cmdpool.queue, (void*)&ctask, sizeof(void*)); 2346 BUG_ON(ctask->sc); 2347 2348 sc->SCp.phase = session->age; 2349 sc->SCp.ptr = (char*)ctask; 2350 iscsi_cmd_init(conn, ctask, sc); 2351 2352 __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*)); 2353 debug_scsi( 2354 "ctask enq [%s cid %d sc %lx itt 0x%x len %d cmdsn %d win %d]\n", 2355 sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read", 2356 conn->id, (long)sc, ctask->itt, sc->request_bufflen, 2357 session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1); 2358 spin_unlock(&session->lock); 2359 2360 if (!in_interrupt() && !down_trylock(&conn->xmitsema)) { 2361 spin_unlock_irq(host->host_lock); 2362 if (iscsi_data_xmit(conn)) 2363 schedule_work(&conn->xmitwork); 2364 up(&conn->xmitsema); 2365 spin_lock_irq(host->host_lock); 2366 } else 2367 schedule_work(&conn->xmitwork); 2368 2369 return 0; 2370 2371 reject: 2372 spin_unlock(&session->lock); 2373 debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason); 2374 return SCSI_MLQUEUE_HOST_BUSY; 2375 2376 fault: 2377 spin_unlock(&session->lock); 2378 printk(KERN_ERR "iscsi_tcp: cmd 0x%x is not queued (%d)\n", 2379 sc->cmnd[0], reason); 2380 sc->sense_buffer[0] = 0x70; 2381 sc->sense_buffer[2] = NOT_READY; 2382 sc->sense_buffer[7] = 0x6; 2383 sc->sense_buffer[12] = 0x08; 2384 sc->sense_buffer[13] = 0x00; 2385 sc->result = (DID_NO_CONNECT << 16); 2386 sc->resid = sc->request_bufflen; 2387 sc->scsi_done(sc); 2388 return 0; 2389 } 2390 2391 static int 2392 iscsi_pool_init(struct iscsi_queue *q, int max, void ***items, int item_size) 2393 { 2394 int i; 2395 2396 *items = kmalloc(max * sizeof(void*), GFP_KERNEL); 2397 if (*items == NULL) 2398 return -ENOMEM; 2399 2400 q->max = max; 2401 q->pool = kmalloc(max * sizeof(void*), GFP_KERNEL); 2402 if (q->pool == NULL) { 2403 kfree(*items); 2404 return -ENOMEM; 2405 } 2406 2407 q->queue = kfifo_init((void*)q->pool, max * sizeof(void*), 2408 GFP_KERNEL, NULL); 2409 if (q->queue == ERR_PTR(-ENOMEM)) { 2410 kfree(q->pool); 2411 kfree(*items); 2412 return -ENOMEM; 2413 } 2414 2415 for (i = 0; i < max; i++) { 2416 q->pool[i] = kmalloc(item_size, GFP_KERNEL); 2417 if (q->pool[i] == NULL) { 2418 int j; 2419 2420 for (j = 0; j < i; j++) 2421 kfree(q->pool[j]); 2422 2423 kfifo_free(q->queue); 2424 kfree(q->pool); 2425 kfree(*items); 2426 return -ENOMEM; 2427 } 2428 memset(q->pool[i], 0, item_size); 2429 (*items)[i] = q->pool[i]; 2430 __kfifo_put(q->queue, (void*)&q->pool[i], sizeof(void*)); 2431 } 2432 return 0; 2433 } 2434 2435 static void 2436 iscsi_pool_free(struct iscsi_queue *q, void **items) 2437 { 2438 int i; 2439 2440 for (i = 0; i < q->max; i++) 2441 kfree(items[i]); 2442 kfree(q->pool); 2443 kfree(items); 2444 } 2445 2446 static iscsi_connh_t 2447 iscsi_conn_create(iscsi_sessionh_t sessionh, uint32_t conn_idx) 2448 { 2449 struct iscsi_session *session = iscsi_ptr(sessionh); 2450 struct iscsi_conn *conn = NULL; 2451 2452 conn = kmalloc(sizeof(struct iscsi_conn), GFP_KERNEL); 2453 if (conn == NULL) 2454 goto conn_alloc_fail; 2455 memset(conn, 0, sizeof(struct iscsi_conn)); 2456 2457 conn->c_stage = ISCSI_CONN_INITIAL_STAGE; 2458 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 2459 conn->id = conn_idx; 2460 conn->exp_statsn = 0; 2461 conn->tmabort_state = TMABORT_INITIAL; 2462 2463 /* initial operational parameters */ 2464 conn->hdr_size = sizeof(struct iscsi_hdr); 2465 conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH; 2466 conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH; 2467 2468 spin_lock_init(&conn->lock); 2469 2470 /* initialize general xmit PDU commands queue */ 2471 conn->xmitqueue = kfifo_alloc(session->cmds_max * sizeof(void*), 2472 GFP_KERNEL, NULL); 2473 if (conn->xmitqueue == ERR_PTR(-ENOMEM)) 2474 goto xmitqueue_alloc_fail; 2475 2476 /* initialize write response PDU commands queue */ 2477 conn->writequeue = kfifo_alloc(session->cmds_max * sizeof(void*), 2478 GFP_KERNEL, NULL); 2479 if (conn->writequeue == ERR_PTR(-ENOMEM)) 2480 goto writequeue_alloc_fail; 2481 2482 /* initialize general immediate & non-immediate PDU commands queue */ 2483 conn->immqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*), 2484 GFP_KERNEL, NULL); 2485 if (conn->immqueue == ERR_PTR(-ENOMEM)) 2486 goto immqueue_alloc_fail; 2487 2488 conn->mgmtqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*), 2489 GFP_KERNEL, NULL); 2490 if (conn->mgmtqueue == ERR_PTR(-ENOMEM)) 2491 goto mgmtqueue_alloc_fail; 2492 2493 INIT_WORK(&conn->xmitwork, iscsi_xmitworker, conn); 2494 2495 /* allocate login_mtask used for the login/text sequences */ 2496 spin_lock_bh(&session->lock); 2497 if (!__kfifo_get(session->mgmtpool.queue, 2498 (void*)&conn->login_mtask, 2499 sizeof(void*))) { 2500 spin_unlock_bh(&session->lock); 2501 goto login_mtask_alloc_fail; 2502 } 2503 spin_unlock_bh(&session->lock); 2504 2505 /* allocate initial PDU receive place holder */ 2506 if (conn->data_size <= PAGE_SIZE) 2507 conn->data = kmalloc(conn->data_size, GFP_KERNEL); 2508 else 2509 conn->data = (void*)__get_free_pages(GFP_KERNEL, 2510 get_order(conn->data_size)); 2511 if (!conn->data) 2512 goto max_recv_dlenght_alloc_fail; 2513 2514 init_timer(&conn->tmabort_timer); 2515 init_MUTEX(&conn->xmitsema); 2516 init_waitqueue_head(&conn->ehwait); 2517 2518 return iscsi_handle(conn); 2519 2520 max_recv_dlenght_alloc_fail: 2521 spin_lock_bh(&session->lock); 2522 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask, 2523 sizeof(void*)); 2524 spin_unlock_bh(&session->lock); 2525 login_mtask_alloc_fail: 2526 kfifo_free(conn->mgmtqueue); 2527 mgmtqueue_alloc_fail: 2528 kfifo_free(conn->immqueue); 2529 immqueue_alloc_fail: 2530 kfifo_free(conn->writequeue); 2531 writequeue_alloc_fail: 2532 kfifo_free(conn->xmitqueue); 2533 xmitqueue_alloc_fail: 2534 kfree(conn); 2535 conn_alloc_fail: 2536 return iscsi_handle(NULL); 2537 } 2538 2539 static void 2540 iscsi_conn_destroy(iscsi_connh_t connh) 2541 { 2542 struct iscsi_conn *conn = iscsi_ptr(connh); 2543 struct iscsi_session *session = conn->session; 2544 2545 down(&conn->xmitsema); 2546 set_bit(SUSPEND_BIT, &conn->suspend_tx); 2547 if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE && conn->sock) { 2548 struct sock *sk = conn->sock->sk; 2549 2550 /* 2551 * conn_start() has never been called! 2552 * need to cleanup the socket. 2553 */ 2554 write_lock_bh(&sk->sk_callback_lock); 2555 set_bit(SUSPEND_BIT, &conn->suspend_rx); 2556 write_unlock_bh(&sk->sk_callback_lock); 2557 2558 sock_hold(conn->sock->sk); 2559 iscsi_conn_restore_callbacks(conn); 2560 sock_put(conn->sock->sk); 2561 sock_release(conn->sock); 2562 conn->sock = NULL; 2563 } 2564 2565 spin_lock_bh(&session->lock); 2566 conn->c_stage = ISCSI_CONN_CLEANUP_WAIT; 2567 if (session->leadconn == conn) { 2568 /* 2569 * leading connection? then give up on recovery. 2570 */ 2571 session->state = ISCSI_STATE_TERMINATE; 2572 wake_up(&conn->ehwait); 2573 } 2574 spin_unlock_bh(&session->lock); 2575 2576 up(&conn->xmitsema); 2577 2578 /* 2579 * Block until all in-progress commands for this connection 2580 * time out or fail. 2581 */ 2582 for (;;) { 2583 spin_lock_bh(&conn->lock); 2584 if (!session->host->host_busy) { /* OK for ERL == 0 */ 2585 spin_unlock_bh(&conn->lock); 2586 break; 2587 } 2588 spin_unlock_bh(&conn->lock); 2589 msleep_interruptible(500); 2590 printk("conn_destroy(): host_busy %d host_failed %d\n", 2591 session->host->host_busy, session->host->host_failed); 2592 /* 2593 * force eh_abort() to unblock 2594 */ 2595 wake_up(&conn->ehwait); 2596 } 2597 2598 /* now free crypto */ 2599 if (conn->hdrdgst_en || conn->datadgst_en) { 2600 if (conn->tx_tfm) 2601 crypto_free_tfm(conn->tx_tfm); 2602 if (conn->rx_tfm) 2603 crypto_free_tfm(conn->rx_tfm); 2604 if (conn->data_tx_tfm) 2605 crypto_free_tfm(conn->data_tx_tfm); 2606 if (conn->data_rx_tfm) 2607 crypto_free_tfm(conn->data_rx_tfm); 2608 } 2609 2610 /* free conn->data, size = MaxRecvDataSegmentLength */ 2611 if (conn->data_size <= PAGE_SIZE) 2612 kfree(conn->data); 2613 else 2614 free_pages((unsigned long)conn->data, 2615 get_order(conn->data_size)); 2616 2617 spin_lock_bh(&session->lock); 2618 __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask, 2619 sizeof(void*)); 2620 list_del(&conn->item); 2621 if (list_empty(&session->connections)) 2622 session->leadconn = NULL; 2623 if (session->leadconn && session->leadconn == conn) 2624 session->leadconn = container_of(session->connections.next, 2625 struct iscsi_conn, item); 2626 2627 if (session->leadconn == NULL) 2628 /* none connections exits.. reset sequencing */ 2629 session->cmdsn = session->max_cmdsn = session->exp_cmdsn = 1; 2630 spin_unlock_bh(&session->lock); 2631 2632 kfifo_free(conn->xmitqueue); 2633 kfifo_free(conn->writequeue); 2634 kfifo_free(conn->immqueue); 2635 kfifo_free(conn->mgmtqueue); 2636 kfree(conn); 2637 } 2638 2639 static int 2640 iscsi_conn_bind(iscsi_sessionh_t sessionh, iscsi_connh_t connh, 2641 uint32_t transport_fd, int is_leading) 2642 { 2643 struct iscsi_session *session = iscsi_ptr(sessionh); 2644 struct iscsi_conn *tmp = ERR_PTR(-EEXIST), *conn = iscsi_ptr(connh); 2645 struct sock *sk; 2646 struct socket *sock; 2647 int err; 2648 2649 /* lookup for existing socket */ 2650 sock = sockfd_lookup(transport_fd, &err); 2651 if (!sock) { 2652 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err); 2653 return -EEXIST; 2654 } 2655 2656 /* lookup for existing connection */ 2657 spin_lock_bh(&session->lock); 2658 list_for_each_entry(tmp, &session->connections, item) { 2659 if (tmp == conn) { 2660 if (conn->c_stage != ISCSI_CONN_STOPPED || 2661 conn->stop_stage == STOP_CONN_TERM) { 2662 printk(KERN_ERR "iscsi_tcp: can't bind " 2663 "non-stopped connection (%d:%d)\n", 2664 conn->c_stage, conn->stop_stage); 2665 spin_unlock_bh(&session->lock); 2666 return -EIO; 2667 } 2668 break; 2669 } 2670 } 2671 if (tmp != conn) { 2672 /* bind new iSCSI connection to session */ 2673 conn->session = session; 2674 2675 list_add(&conn->item, &session->connections); 2676 } 2677 spin_unlock_bh(&session->lock); 2678 2679 if (conn->stop_stage != STOP_CONN_SUSPEND) { 2680 /* bind iSCSI connection and socket */ 2681 conn->sock = sock; 2682 2683 /* setup Socket parameters */ 2684 sk = sock->sk; 2685 sk->sk_reuse = 1; 2686 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */ 2687 sk->sk_allocation = GFP_ATOMIC; 2688 2689 /* FIXME: disable Nagle's algorithm */ 2690 2691 /* 2692 * Intercept TCP callbacks for sendfile like receive 2693 * processing. 2694 */ 2695 iscsi_conn_set_callbacks(conn); 2696 2697 /* 2698 * set receive state machine into initial state 2699 */ 2700 conn->in_progress = IN_PROGRESS_WAIT_HEADER; 2701 } 2702 2703 if (is_leading) 2704 session->leadconn = conn; 2705 2706 /* 2707 * Unblock xmitworker(), Login Phase will pass through. 2708 */ 2709 clear_bit(SUSPEND_BIT, &conn->suspend_rx); 2710 clear_bit(SUSPEND_BIT, &conn->suspend_tx); 2711 2712 return 0; 2713 } 2714 2715 static int 2716 iscsi_conn_start(iscsi_connh_t connh) 2717 { 2718 struct iscsi_conn *conn = iscsi_ptr(connh); 2719 struct iscsi_session *session = conn->session; 2720 struct sock *sk; 2721 2722 /* FF phase warming up... */ 2723 2724 if (session == NULL) { 2725 printk(KERN_ERR "iscsi_tcp: can't start unbound connection\n"); 2726 return -EPERM; 2727 } 2728 2729 sk = conn->sock->sk; 2730 2731 write_lock_bh(&sk->sk_callback_lock); 2732 spin_lock_bh(&session->lock); 2733 conn->c_stage = ISCSI_CONN_STARTED; 2734 session->state = ISCSI_STATE_LOGGED_IN; 2735 2736 switch(conn->stop_stage) { 2737 case STOP_CONN_RECOVER: 2738 /* 2739 * unblock eh_abort() if it is blocked. re-try all 2740 * commands after successful recovery 2741 */ 2742 session->conn_cnt++; 2743 conn->stop_stage = 0; 2744 conn->tmabort_state = TMABORT_INITIAL; 2745 session->age++; 2746 wake_up(&conn->ehwait); 2747 break; 2748 case STOP_CONN_TERM: 2749 session->conn_cnt++; 2750 conn->stop_stage = 0; 2751 break; 2752 case STOP_CONN_SUSPEND: 2753 conn->stop_stage = 0; 2754 clear_bit(SUSPEND_BIT, &conn->suspend_rx); 2755 clear_bit(SUSPEND_BIT, &conn->suspend_tx); 2756 break; 2757 default: 2758 break; 2759 } 2760 spin_unlock_bh(&session->lock); 2761 write_unlock_bh(&sk->sk_callback_lock); 2762 2763 return 0; 2764 } 2765 2766 static void 2767 iscsi_conn_stop(iscsi_connh_t connh, int flag) 2768 { 2769 struct iscsi_conn *conn = iscsi_ptr(connh); 2770 struct iscsi_session *session = conn->session; 2771 struct sock *sk; 2772 unsigned long flags; 2773 2774 BUG_ON(!conn->sock); 2775 sk = conn->sock->sk; 2776 write_lock_bh(&sk->sk_callback_lock); 2777 set_bit(SUSPEND_BIT, &conn->suspend_rx); 2778 write_unlock_bh(&sk->sk_callback_lock); 2779 2780 down(&conn->xmitsema); 2781 2782 spin_lock_irqsave(session->host->host_lock, flags); 2783 spin_lock(&session->lock); 2784 conn->stop_stage = flag; 2785 conn->c_stage = ISCSI_CONN_STOPPED; 2786 set_bit(SUSPEND_BIT, &conn->suspend_tx); 2787 2788 if (flag != STOP_CONN_SUSPEND) 2789 session->conn_cnt--; 2790 2791 if (session->conn_cnt == 0 || session->leadconn == conn) 2792 session->state = ISCSI_STATE_FAILED; 2793 2794 spin_unlock(&session->lock); 2795 spin_unlock_irqrestore(session->host->host_lock, flags); 2796 2797 if (flag == STOP_CONN_TERM || flag == STOP_CONN_RECOVER) { 2798 struct iscsi_cmd_task *ctask; 2799 struct iscsi_mgmt_task *mtask; 2800 2801 /* 2802 * Socket must go now. 2803 */ 2804 sock_hold(conn->sock->sk); 2805 iscsi_conn_restore_callbacks(conn); 2806 sock_put(conn->sock->sk); 2807 2808 /* 2809 * flush xmit queues. 2810 */ 2811 spin_lock_bh(&session->lock); 2812 while (__kfifo_get(conn->writequeue, (void*)&ctask, 2813 sizeof(void*)) || 2814 __kfifo_get(conn->xmitqueue, (void*)&ctask, 2815 sizeof(void*))) { 2816 struct iscsi_r2t_info *r2t; 2817 2818 /* 2819 * flush ctask's r2t queues 2820 */ 2821 while (__kfifo_get(ctask->r2tqueue, (void*)&r2t, 2822 sizeof(void*))) 2823 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, 2824 sizeof(void*)); 2825 2826 spin_unlock_bh(&session->lock); 2827 local_bh_disable(); 2828 iscsi_ctask_cleanup(conn, ctask); 2829 local_bh_enable(); 2830 spin_lock_bh(&session->lock); 2831 } 2832 conn->ctask = NULL; 2833 while (__kfifo_get(conn->immqueue, (void*)&mtask, 2834 sizeof(void*)) || 2835 __kfifo_get(conn->mgmtqueue, (void*)&mtask, 2836 sizeof(void*))) { 2837 __kfifo_put(session->mgmtpool.queue, 2838 (void*)&mtask, sizeof(void*)); 2839 } 2840 conn->mtask = NULL; 2841 spin_unlock_bh(&session->lock); 2842 2843 /* 2844 * release socket only after we stopped data_xmit() 2845 * activity and flushed all outstandings 2846 */ 2847 sock_release(conn->sock); 2848 conn->sock = NULL; 2849 2850 /* 2851 * for connection level recovery we should not calculate 2852 * header digest. conn->hdr_size used for optimization 2853 * in hdr_extract() and will be re-negotiated at 2854 * set_param() time. 2855 */ 2856 if (flag == STOP_CONN_RECOVER) 2857 conn->hdr_size = sizeof(struct iscsi_hdr); 2858 } 2859 up(&conn->xmitsema); 2860 } 2861 2862 static int 2863 iscsi_conn_send_generic(struct iscsi_conn *conn, struct iscsi_hdr *hdr, 2864 char *data, uint32_t data_size) 2865 { 2866 struct iscsi_session *session = conn->session; 2867 struct iscsi_nopout *nop = (struct iscsi_nopout *)hdr; 2868 struct iscsi_mgmt_task *mtask; 2869 2870 spin_lock_bh(&session->lock); 2871 if (session->state == ISCSI_STATE_TERMINATE) { 2872 spin_unlock_bh(&session->lock); 2873 return -EPERM; 2874 } 2875 if (hdr->opcode == (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) || 2876 hdr->opcode == (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE)) 2877 /* 2878 * Login and Text are sent serially, in 2879 * request-followed-by-response sequence. 2880 * Same mtask can be used. Same ITT must be used. 2881 * Note that login_mtask is preallocated at conn_create(). 2882 */ 2883 mtask = conn->login_mtask; 2884 else { 2885 BUG_ON(conn->c_stage == ISCSI_CONN_INITIAL_STAGE); 2886 BUG_ON(conn->c_stage == ISCSI_CONN_STOPPED); 2887 2888 if (!__kfifo_get(session->mgmtpool.queue, 2889 (void*)&mtask, sizeof(void*))) { 2890 spin_unlock_bh(&session->lock); 2891 return -ENOSPC; 2892 } 2893 } 2894 2895 /* 2896 * pre-format CmdSN and ExpStatSN for outgoing PDU. 2897 */ 2898 if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) { 2899 hdr->itt = mtask->itt | (conn->id << CID_SHIFT) | 2900 (session->age << AGE_SHIFT); 2901 nop->cmdsn = cpu_to_be32(session->cmdsn); 2902 if (conn->c_stage == ISCSI_CONN_STARTED && 2903 !(hdr->opcode & ISCSI_OP_IMMEDIATE)) 2904 session->cmdsn++; 2905 } else 2906 /* do not advance CmdSN */ 2907 nop->cmdsn = cpu_to_be32(session->cmdsn); 2908 2909 nop->exp_statsn = cpu_to_be32(conn->exp_statsn); 2910 2911 memcpy(&mtask->hdr, hdr, sizeof(struct iscsi_hdr)); 2912 2913 iscsi_buf_init_virt(&mtask->headbuf, (char*)&mtask->hdr, 2914 sizeof(struct iscsi_hdr)); 2915 2916 spin_unlock_bh(&session->lock); 2917 2918 if (data_size) { 2919 memcpy(mtask->data, data, data_size); 2920 mtask->data_count = data_size; 2921 } else 2922 mtask->data_count = 0; 2923 2924 mtask->xmstate = XMSTATE_IMM_HDR; 2925 2926 if (mtask->data_count) { 2927 iscsi_buf_init_iov(&mtask->sendbuf, (char*)mtask->data, 2928 mtask->data_count); 2929 } 2930 2931 debug_scsi("mgmtpdu [op 0x%x hdr->itt 0x%x datalen %d]\n", 2932 hdr->opcode, hdr->itt, data_size); 2933 2934 /* 2935 * since send_pdu() could be called at least from two contexts, 2936 * we need to serialize __kfifo_put, so we don't have to take 2937 * additional lock on fast data-path 2938 */ 2939 if (hdr->opcode & ISCSI_OP_IMMEDIATE) 2940 __kfifo_put(conn->immqueue, (void*)&mtask, sizeof(void*)); 2941 else 2942 __kfifo_put(conn->mgmtqueue, (void*)&mtask, sizeof(void*)); 2943 2944 schedule_work(&conn->xmitwork); 2945 2946 return 0; 2947 } 2948 2949 static int 2950 iscsi_eh_host_reset(struct scsi_cmnd *sc) 2951 { 2952 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr; 2953 struct iscsi_conn *conn = ctask->conn; 2954 struct iscsi_session *session = conn->session; 2955 2956 spin_lock_bh(&session->lock); 2957 if (session->state == ISCSI_STATE_TERMINATE) { 2958 debug_scsi("failing host reset: session terminated " 2959 "[CID %d age %d]", conn->id, session->age); 2960 spin_unlock_bh(&session->lock); 2961 return FAILED; 2962 } 2963 spin_unlock_bh(&session->lock); 2964 2965 debug_scsi("failing connection CID %d due to SCSI host reset " 2966 "[itt 0x%x age %d]", conn->id, ctask->itt, 2967 session->age); 2968 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 2969 2970 return SUCCESS; 2971 } 2972 2973 static void 2974 iscsi_tmabort_timedout(unsigned long data) 2975 { 2976 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)data; 2977 struct iscsi_conn *conn = ctask->conn; 2978 struct iscsi_session *session = conn->session; 2979 2980 spin_lock(&session->lock); 2981 if (conn->tmabort_state == TMABORT_INITIAL) { 2982 __kfifo_put(session->mgmtpool.queue, 2983 (void*)&ctask->mtask, sizeof(void*)); 2984 conn->tmabort_state = TMABORT_TIMEDOUT; 2985 debug_scsi("tmabort timedout [sc %lx itt 0x%x]\n", 2986 (long)ctask->sc, ctask->itt); 2987 /* unblock eh_abort() */ 2988 wake_up(&conn->ehwait); 2989 } 2990 spin_unlock(&session->lock); 2991 } 2992 2993 static int 2994 iscsi_eh_abort(struct scsi_cmnd *sc) 2995 { 2996 int rc; 2997 struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr; 2998 struct iscsi_conn *conn = ctask->conn; 2999 struct iscsi_session *session = conn->session; 3000 3001 conn->eh_abort_cnt++; 3002 debug_scsi("aborting [sc %lx itt 0x%x]\n", (long)sc, ctask->itt); 3003 3004 /* 3005 * two cases for ERL=0 here: 3006 * 3007 * 1) connection-level failure; 3008 * 2) recovery due protocol error; 3009 */ 3010 down(&conn->xmitsema); 3011 spin_lock_bh(&session->lock); 3012 if (session->state != ISCSI_STATE_LOGGED_IN) { 3013 if (session->state == ISCSI_STATE_TERMINATE) { 3014 spin_unlock_bh(&session->lock); 3015 up(&conn->xmitsema); 3016 goto failed; 3017 } 3018 spin_unlock_bh(&session->lock); 3019 } else { 3020 struct iscsi_tm *hdr = &conn->tmhdr; 3021 3022 /* 3023 * Still LOGGED_IN... 3024 */ 3025 3026 if (!ctask->sc || sc->SCp.phase != session->age) { 3027 /* 3028 * 1) ctask completed before time out. But session 3029 * is still ok => Happy Retry. 3030 * 2) session was re-open during time out of ctask. 3031 */ 3032 spin_unlock_bh(&session->lock); 3033 up(&conn->xmitsema); 3034 goto success; 3035 } 3036 conn->tmabort_state = TMABORT_INITIAL; 3037 spin_unlock_bh(&session->lock); 3038 3039 /* 3040 * ctask timed out but session is OK 3041 * ERL=0 requires task mgmt abort to be issued on each 3042 * failed command. requests must be serialized. 3043 */ 3044 memset(hdr, 0, sizeof(struct iscsi_tm)); 3045 hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE; 3046 hdr->flags = ISCSI_TM_FUNC_ABORT_TASK; 3047 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3048 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun)); 3049 hdr->rtt = ctask->hdr.itt; 3050 hdr->refcmdsn = ctask->hdr.cmdsn; 3051 3052 rc = iscsi_conn_send_generic(conn, (struct iscsi_hdr *)hdr, 3053 NULL, 0); 3054 if (rc) { 3055 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 3056 debug_scsi("abort sent failure [itt 0x%x]", ctask->itt); 3057 } else { 3058 struct iscsi_r2t_info *r2t; 3059 3060 /* 3061 * TMF abort vs. TMF response race logic 3062 */ 3063 spin_lock_bh(&session->lock); 3064 ctask->mtask = (struct iscsi_mgmt_task *) 3065 session->mgmt_cmds[(hdr->itt & ITT_MASK) - 3066 ISCSI_MGMT_ITT_OFFSET]; 3067 /* 3068 * have to flush r2tqueue to avoid r2t leaks 3069 */ 3070 while (__kfifo_get(ctask->r2tqueue, (void*)&r2t, 3071 sizeof(void*))) { 3072 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, 3073 sizeof(void*)); 3074 } 3075 if (conn->tmabort_state == TMABORT_INITIAL) { 3076 conn->tmfcmd_pdus_cnt++; 3077 conn->tmabort_timer.expires = 3*HZ + jiffies; 3078 conn->tmabort_timer.function = 3079 iscsi_tmabort_timedout; 3080 conn->tmabort_timer.data = (unsigned long)ctask; 3081 add_timer(&conn->tmabort_timer); 3082 debug_scsi("abort sent [itt 0x%x]", ctask->itt); 3083 } else { 3084 if (!ctask->sc || 3085 conn->tmabort_state == TMABORT_SUCCESS) { 3086 conn->tmabort_state = TMABORT_INITIAL; 3087 spin_unlock_bh(&session->lock); 3088 up(&conn->xmitsema); 3089 goto success; 3090 } 3091 conn->tmabort_state = TMABORT_INITIAL; 3092 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 3093 } 3094 spin_unlock_bh(&session->lock); 3095 } 3096 } 3097 up(&conn->xmitsema); 3098 3099 3100 /* 3101 * block eh thread until: 3102 * 3103 * 1) abort response; 3104 * 2) abort timeout; 3105 * 3) session re-opened; 3106 * 4) session terminated; 3107 */ 3108 for (;;) { 3109 int p_state = session->state; 3110 3111 rc = wait_event_interruptible(conn->ehwait, 3112 (p_state == ISCSI_STATE_LOGGED_IN ? 3113 (session->state == ISCSI_STATE_TERMINATE || 3114 conn->tmabort_state != TMABORT_INITIAL) : 3115 (session->state == ISCSI_STATE_TERMINATE || 3116 session->state == ISCSI_STATE_LOGGED_IN))); 3117 if (rc) { 3118 /* shutdown.. */ 3119 session->state = ISCSI_STATE_TERMINATE; 3120 goto failed; 3121 } 3122 3123 if (signal_pending(current)) 3124 flush_signals(current); 3125 3126 if (session->state == ISCSI_STATE_TERMINATE) 3127 goto failed; 3128 3129 spin_lock_bh(&session->lock); 3130 if (sc->SCp.phase == session->age && 3131 (conn->tmabort_state == TMABORT_TIMEDOUT || 3132 conn->tmabort_state == TMABORT_FAILED)) { 3133 conn->tmabort_state = TMABORT_INITIAL; 3134 if (!ctask->sc) { 3135 /* 3136 * ctask completed before tmf abort response or 3137 * time out. 3138 * But session is still ok => Happy Retry. 3139 */ 3140 spin_unlock_bh(&session->lock); 3141 break; 3142 } 3143 spin_unlock_bh(&session->lock); 3144 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 3145 continue; 3146 } 3147 spin_unlock_bh(&session->lock); 3148 break; 3149 } 3150 3151 success: 3152 debug_scsi("abort success [sc %lx itt 0x%x]\n", (long)sc, ctask->itt); 3153 rc = SUCCESS; 3154 goto exit; 3155 3156 failed: 3157 debug_scsi("abort failed [sc %lx itt 0x%x]\n", (long)sc, ctask->itt); 3158 rc = FAILED; 3159 3160 exit: 3161 del_timer_sync(&conn->tmabort_timer); 3162 3163 down(&conn->xmitsema); 3164 if (conn->sock) { 3165 struct sock *sk = conn->sock->sk; 3166 3167 write_lock_bh(&sk->sk_callback_lock); 3168 iscsi_ctask_cleanup(conn, ctask); 3169 write_unlock_bh(&sk->sk_callback_lock); 3170 } 3171 up(&conn->xmitsema); 3172 return rc; 3173 } 3174 3175 static int 3176 iscsi_r2tpool_alloc(struct iscsi_session *session) 3177 { 3178 int i; 3179 int cmd_i; 3180 3181 /* 3182 * initialize per-task: R2T pool and xmit queue 3183 */ 3184 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { 3185 struct iscsi_cmd_task *ctask = session->cmds[cmd_i]; 3186 3187 /* 3188 * pre-allocated x4 as much r2ts to handle race when 3189 * target acks DataOut faster than we data_xmit() queues 3190 * could replenish r2tqueue. 3191 */ 3192 3193 /* R2T pool */ 3194 if (iscsi_pool_init(&ctask->r2tpool, session->max_r2t * 4, 3195 (void***)&ctask->r2ts, sizeof(struct iscsi_r2t_info))) { 3196 goto r2t_alloc_fail; 3197 } 3198 3199 /* R2T xmit queue */ 3200 ctask->r2tqueue = kfifo_alloc( 3201 session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL); 3202 if (ctask->r2tqueue == ERR_PTR(-ENOMEM)) { 3203 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts); 3204 goto r2t_alloc_fail; 3205 } 3206 3207 /* 3208 * number of 3209 * Data-Out PDU's within R2T-sequence can be quite big; 3210 * using mempool 3211 */ 3212 ctask->datapool = mempool_create(ISCSI_DTASK_DEFAULT_MAX, 3213 mempool_alloc_slab, mempool_free_slab, taskcache); 3214 if (ctask->datapool == NULL) { 3215 kfifo_free(ctask->r2tqueue); 3216 iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts); 3217 goto r2t_alloc_fail; 3218 } 3219 INIT_LIST_HEAD(&ctask->dataqueue); 3220 } 3221 3222 return 0; 3223 3224 r2t_alloc_fail: 3225 for (i = 0; i < cmd_i; i++) { 3226 mempool_destroy(session->cmds[i]->datapool); 3227 kfifo_free(session->cmds[i]->r2tqueue); 3228 iscsi_pool_free(&session->cmds[i]->r2tpool, 3229 (void**)session->cmds[i]->r2ts); 3230 } 3231 return -ENOMEM; 3232 } 3233 3234 static void 3235 iscsi_r2tpool_free(struct iscsi_session *session) 3236 { 3237 int i; 3238 3239 for (i = 0; i < session->cmds_max; i++) { 3240 mempool_destroy(session->cmds[i]->datapool); 3241 kfifo_free(session->cmds[i]->r2tqueue); 3242 iscsi_pool_free(&session->cmds[i]->r2tpool, 3243 (void**)session->cmds[i]->r2ts); 3244 } 3245 } 3246 3247 static struct scsi_host_template iscsi_sht = { 3248 .name = "iSCSI Initiator over TCP/IP, v." 3249 ISCSI_VERSION_STR, 3250 .queuecommand = iscsi_queuecommand, 3251 .can_queue = ISCSI_XMIT_CMDS_MAX - 1, 3252 .sg_tablesize = ISCSI_SG_TABLESIZE, 3253 .cmd_per_lun = ISCSI_CMD_PER_LUN, 3254 .eh_abort_handler = iscsi_eh_abort, 3255 .eh_host_reset_handler = iscsi_eh_host_reset, 3256 .use_clustering = DISABLE_CLUSTERING, 3257 .proc_name = "iscsi_tcp", 3258 .this_id = -1, 3259 }; 3260 3261 static iscsi_sessionh_t 3262 iscsi_session_create(uint32_t initial_cmdsn, struct Scsi_Host *host) 3263 { 3264 int cmd_i; 3265 struct iscsi_session *session; 3266 3267 session = iscsi_hostdata(host->hostdata); 3268 memset(session, 0, sizeof(struct iscsi_session)); 3269 3270 session->host = host; 3271 session->id = host->host_no; 3272 session->state = ISCSI_STATE_LOGGED_IN; 3273 session->mgmtpool_max = ISCSI_MGMT_CMDS_MAX; 3274 session->cmds_max = ISCSI_XMIT_CMDS_MAX; 3275 session->cmdsn = initial_cmdsn; 3276 session->exp_cmdsn = initial_cmdsn + 1; 3277 session->max_cmdsn = initial_cmdsn + 1; 3278 session->max_r2t = 1; 3279 3280 /* initialize SCSI PDU commands pool */ 3281 if (iscsi_pool_init(&session->cmdpool, session->cmds_max, 3282 (void***)&session->cmds, sizeof(struct iscsi_cmd_task))) 3283 goto cmdpool_alloc_fail; 3284 3285 /* pre-format cmds pool with ITT */ 3286 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) 3287 session->cmds[cmd_i]->itt = cmd_i; 3288 3289 spin_lock_init(&session->lock); 3290 INIT_LIST_HEAD(&session->connections); 3291 3292 /* initialize immediate command pool */ 3293 if (iscsi_pool_init(&session->mgmtpool, session->mgmtpool_max, 3294 (void***)&session->mgmt_cmds, sizeof(struct iscsi_mgmt_task))) 3295 goto mgmtpool_alloc_fail; 3296 3297 3298 /* pre-format immediate cmds pool with ITT */ 3299 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) { 3300 session->mgmt_cmds[cmd_i]->itt = ISCSI_MGMT_ITT_OFFSET + cmd_i; 3301 session->mgmt_cmds[cmd_i]->data = kmalloc( 3302 DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH, GFP_KERNEL); 3303 if (!session->mgmt_cmds[cmd_i]->data) { 3304 int j; 3305 3306 for (j = 0; j < cmd_i; j++) 3307 kfree(session->mgmt_cmds[j]->data); 3308 goto immdata_alloc_fail; 3309 } 3310 } 3311 3312 if (iscsi_r2tpool_alloc(session)) 3313 goto r2tpool_alloc_fail; 3314 3315 return iscsi_handle(session); 3316 3317 r2tpool_alloc_fail: 3318 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) 3319 kfree(session->mgmt_cmds[cmd_i]->data); 3320 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds); 3321 immdata_alloc_fail: 3322 mgmtpool_alloc_fail: 3323 iscsi_pool_free(&session->cmdpool, (void**)session->cmds); 3324 cmdpool_alloc_fail: 3325 return iscsi_handle(NULL); 3326 } 3327 3328 static void 3329 iscsi_session_destroy(iscsi_sessionh_t sessionh) 3330 { 3331 int cmd_i; 3332 struct iscsi_data_task *dtask, *n; 3333 struct iscsi_session *session = iscsi_ptr(sessionh); 3334 3335 for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) { 3336 struct iscsi_cmd_task *ctask = session->cmds[cmd_i]; 3337 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) { 3338 list_del(&dtask->item); 3339 mempool_free(dtask, ctask->datapool); 3340 } 3341 } 3342 3343 for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) 3344 kfree(session->mgmt_cmds[cmd_i]->data); 3345 3346 iscsi_r2tpool_free(session); 3347 iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds); 3348 iscsi_pool_free(&session->cmdpool, (void**)session->cmds); 3349 } 3350 3351 static int 3352 iscsi_conn_set_param(iscsi_connh_t connh, enum iscsi_param param, 3353 uint32_t value) 3354 { 3355 struct iscsi_conn *conn = iscsi_ptr(connh); 3356 struct iscsi_session *session = conn->session; 3357 3358 spin_lock_bh(&session->lock); 3359 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE && 3360 conn->stop_stage != STOP_CONN_RECOVER) { 3361 printk(KERN_ERR "iscsi_tcp: can not change parameter [%d]\n", 3362 param); 3363 spin_unlock_bh(&session->lock); 3364 return 0; 3365 } 3366 spin_unlock_bh(&session->lock); 3367 3368 switch(param) { 3369 case ISCSI_PARAM_MAX_RECV_DLENGTH: { 3370 char *saveptr = conn->data; 3371 int flags = GFP_KERNEL; 3372 3373 if (conn->data_size >= value) { 3374 conn->max_recv_dlength = value; 3375 break; 3376 } 3377 3378 spin_lock_bh(&session->lock); 3379 if (conn->stop_stage == STOP_CONN_RECOVER) 3380 flags = GFP_ATOMIC; 3381 spin_unlock_bh(&session->lock); 3382 3383 if (value <= PAGE_SIZE) 3384 conn->data = kmalloc(value, flags); 3385 else 3386 conn->data = (void*)__get_free_pages(flags, 3387 get_order(value)); 3388 if (conn->data == NULL) { 3389 conn->data = saveptr; 3390 return -ENOMEM; 3391 } 3392 if (conn->data_size <= PAGE_SIZE) 3393 kfree(saveptr); 3394 else 3395 free_pages((unsigned long)saveptr, 3396 get_order(conn->data_size)); 3397 conn->max_recv_dlength = value; 3398 conn->data_size = value; 3399 } 3400 break; 3401 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 3402 conn->max_xmit_dlength = value; 3403 break; 3404 case ISCSI_PARAM_HDRDGST_EN: 3405 conn->hdrdgst_en = value; 3406 conn->hdr_size = sizeof(struct iscsi_hdr); 3407 if (conn->hdrdgst_en) { 3408 conn->hdr_size += sizeof(__u32); 3409 if (!conn->tx_tfm) 3410 conn->tx_tfm = crypto_alloc_tfm("crc32c", 0); 3411 if (!conn->tx_tfm) 3412 return -ENOMEM; 3413 if (!conn->rx_tfm) 3414 conn->rx_tfm = crypto_alloc_tfm("crc32c", 0); 3415 if (!conn->rx_tfm) { 3416 crypto_free_tfm(conn->tx_tfm); 3417 return -ENOMEM; 3418 } 3419 } else { 3420 if (conn->tx_tfm) 3421 crypto_free_tfm(conn->tx_tfm); 3422 if (conn->rx_tfm) 3423 crypto_free_tfm(conn->rx_tfm); 3424 } 3425 break; 3426 case ISCSI_PARAM_DATADGST_EN: 3427 conn->datadgst_en = value; 3428 if (conn->datadgst_en) { 3429 if (!conn->data_tx_tfm) 3430 conn->data_tx_tfm = 3431 crypto_alloc_tfm("crc32c", 0); 3432 if (!conn->data_tx_tfm) 3433 return -ENOMEM; 3434 if (!conn->data_rx_tfm) 3435 conn->data_rx_tfm = 3436 crypto_alloc_tfm("crc32c", 0); 3437 if (!conn->data_rx_tfm) { 3438 crypto_free_tfm(conn->data_tx_tfm); 3439 return -ENOMEM; 3440 } 3441 } else { 3442 if (conn->data_tx_tfm) 3443 crypto_free_tfm(conn->data_tx_tfm); 3444 if (conn->data_rx_tfm) 3445 crypto_free_tfm(conn->data_rx_tfm); 3446 } 3447 break; 3448 case ISCSI_PARAM_INITIAL_R2T_EN: 3449 session->initial_r2t_en = value; 3450 break; 3451 case ISCSI_PARAM_MAX_R2T: 3452 if (session->max_r2t == roundup_pow_of_two(value)) 3453 break; 3454 iscsi_r2tpool_free(session); 3455 session->max_r2t = value; 3456 if (session->max_r2t & (session->max_r2t - 1)) 3457 session->max_r2t = roundup_pow_of_two(session->max_r2t); 3458 if (iscsi_r2tpool_alloc(session)) 3459 return -ENOMEM; 3460 break; 3461 case ISCSI_PARAM_IMM_DATA_EN: 3462 session->imm_data_en = value; 3463 break; 3464 case ISCSI_PARAM_FIRST_BURST: 3465 session->first_burst = value; 3466 break; 3467 case ISCSI_PARAM_MAX_BURST: 3468 session->max_burst = value; 3469 break; 3470 case ISCSI_PARAM_PDU_INORDER_EN: 3471 session->pdu_inorder_en = value; 3472 break; 3473 case ISCSI_PARAM_DATASEQ_INORDER_EN: 3474 session->dataseq_inorder_en = value; 3475 break; 3476 case ISCSI_PARAM_ERL: 3477 session->erl = value; 3478 break; 3479 case ISCSI_PARAM_IFMARKER_EN: 3480 BUG_ON(value); 3481 session->ifmarker_en = value; 3482 break; 3483 case ISCSI_PARAM_OFMARKER_EN: 3484 BUG_ON(value); 3485 session->ofmarker_en = value; 3486 break; 3487 default: 3488 break; 3489 } 3490 3491 return 0; 3492 } 3493 3494 static int 3495 iscsi_conn_get_param(iscsi_connh_t connh, enum iscsi_param param, 3496 uint32_t *value) 3497 { 3498 struct iscsi_conn *conn = iscsi_ptr(connh); 3499 struct iscsi_session *session = conn->session; 3500 3501 switch(param) { 3502 case ISCSI_PARAM_MAX_RECV_DLENGTH: 3503 *value = conn->max_recv_dlength; 3504 break; 3505 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 3506 *value = conn->max_xmit_dlength; 3507 break; 3508 case ISCSI_PARAM_HDRDGST_EN: 3509 *value = conn->hdrdgst_en; 3510 break; 3511 case ISCSI_PARAM_DATADGST_EN: 3512 *value = conn->datadgst_en; 3513 break; 3514 case ISCSI_PARAM_INITIAL_R2T_EN: 3515 *value = session->initial_r2t_en; 3516 break; 3517 case ISCSI_PARAM_MAX_R2T: 3518 *value = session->max_r2t; 3519 break; 3520 case ISCSI_PARAM_IMM_DATA_EN: 3521 *value = session->imm_data_en; 3522 break; 3523 case ISCSI_PARAM_FIRST_BURST: 3524 *value = session->first_burst; 3525 break; 3526 case ISCSI_PARAM_MAX_BURST: 3527 *value = session->max_burst; 3528 break; 3529 case ISCSI_PARAM_PDU_INORDER_EN: 3530 *value = session->pdu_inorder_en; 3531 break; 3532 case ISCSI_PARAM_DATASEQ_INORDER_EN: 3533 *value = session->dataseq_inorder_en; 3534 break; 3535 case ISCSI_PARAM_ERL: 3536 *value = session->erl; 3537 break; 3538 case ISCSI_PARAM_IFMARKER_EN: 3539 *value = session->ifmarker_en; 3540 break; 3541 case ISCSI_PARAM_OFMARKER_EN: 3542 *value = session->ofmarker_en; 3543 break; 3544 default: 3545 return ISCSI_ERR_PARAM_NOT_FOUND; 3546 } 3547 3548 return 0; 3549 } 3550 3551 static void 3552 iscsi_conn_get_stats(iscsi_connh_t connh, struct iscsi_stats *stats) 3553 { 3554 struct iscsi_conn *conn = iscsi_ptr(connh); 3555 3556 stats->txdata_octets = conn->txdata_octets; 3557 stats->rxdata_octets = conn->rxdata_octets; 3558 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt; 3559 stats->dataout_pdus = conn->dataout_pdus_cnt; 3560 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt; 3561 stats->datain_pdus = conn->datain_pdus_cnt; 3562 stats->r2t_pdus = conn->r2t_pdus_cnt; 3563 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt; 3564 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; 3565 stats->custom_length = 3; 3566 strcpy(stats->custom[0].desc, "tx_sendpage_failures"); 3567 stats->custom[0].value = conn->sendpage_failures_cnt; 3568 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr"); 3569 stats->custom[1].value = conn->discontiguous_hdr_cnt; 3570 strcpy(stats->custom[2].desc, "eh_abort_cnt"); 3571 stats->custom[2].value = conn->eh_abort_cnt; 3572 } 3573 3574 static int 3575 iscsi_conn_send_pdu(iscsi_connh_t connh, struct iscsi_hdr *hdr, char *data, 3576 uint32_t data_size) 3577 { 3578 struct iscsi_conn *conn = iscsi_ptr(connh); 3579 int rc; 3580 3581 down(&conn->xmitsema); 3582 rc = iscsi_conn_send_generic(conn, hdr, data, data_size); 3583 up(&conn->xmitsema); 3584 3585 return rc; 3586 } 3587 3588 static struct iscsi_transport iscsi_tcp_transport = { 3589 .owner = THIS_MODULE, 3590 .name = "tcp", 3591 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST 3592 | CAP_DATADGST, 3593 .host_template = &iscsi_sht, 3594 .hostdata_size = sizeof(struct iscsi_session), 3595 .max_conn = 1, 3596 .max_cmd_len = ISCSI_TCP_MAX_CMD_LEN, 3597 .create_session = iscsi_session_create, 3598 .destroy_session = iscsi_session_destroy, 3599 .create_conn = iscsi_conn_create, 3600 .bind_conn = iscsi_conn_bind, 3601 .destroy_conn = iscsi_conn_destroy, 3602 .set_param = iscsi_conn_set_param, 3603 .get_param = iscsi_conn_get_param, 3604 .start_conn = iscsi_conn_start, 3605 .stop_conn = iscsi_conn_stop, 3606 .send_pdu = iscsi_conn_send_pdu, 3607 .get_stats = iscsi_conn_get_stats, 3608 }; 3609 3610 static int __init 3611 iscsi_tcp_init(void) 3612 { 3613 int error; 3614 3615 if (iscsi_max_lun < 1) { 3616 printk(KERN_ERR "Invalid max_lun value of %u\n", iscsi_max_lun); 3617 return -EINVAL; 3618 } 3619 iscsi_tcp_transport.max_lun = iscsi_max_lun; 3620 3621 taskcache = kmem_cache_create("iscsi_taskcache", 3622 sizeof(struct iscsi_data_task), 0, 3623 SLAB_HWCACHE_ALIGN | SLAB_NO_REAP, NULL, NULL); 3624 if (!taskcache) 3625 return -ENOMEM; 3626 3627 error = iscsi_register_transport(&iscsi_tcp_transport); 3628 if (error) 3629 kmem_cache_destroy(taskcache); 3630 3631 return error; 3632 } 3633 3634 static void __exit 3635 iscsi_tcp_exit(void) 3636 { 3637 iscsi_unregister_transport(&iscsi_tcp_transport); 3638 kmem_cache_destroy(taskcache); 3639 } 3640 3641 module_init(iscsi_tcp_init); 3642 module_exit(iscsi_tcp_exit); 3643