1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * iSCSI Initiator over TCP/IP Data-Path 4 * 5 * Copyright (C) 2004 Dmitry Yusupov 6 * Copyright (C) 2004 Alex Aizman 7 * Copyright (C) 2005 - 2006 Mike Christie 8 * Copyright (C) 2006 Red Hat, Inc. All rights reserved. 9 * maintained by open-iscsi@googlegroups.com 10 * 11 * See the file COPYING included with this distribution for more details. 12 * 13 * Credits: 14 * Christoph Hellwig 15 * FUJITA Tomonori 16 * Arne Redlich 17 * Zhenyu Wang 18 */ 19 20 #include <crypto/hash.h> 21 #include <linux/types.h> 22 #include <linux/inet.h> 23 #include <linux/slab.h> 24 #include <linux/sched/mm.h> 25 #include <linux/file.h> 26 #include <linux/blkdev.h> 27 #include <linux/delay.h> 28 #include <linux/kfifo.h> 29 #include <linux/scatterlist.h> 30 #include <linux/module.h> 31 #include <linux/backing-dev.h> 32 #include <net/tcp.h> 33 #include <scsi/scsi_cmnd.h> 34 #include <scsi/scsi_device.h> 35 #include <scsi/scsi_host.h> 36 #include <scsi/scsi.h> 37 #include <scsi/scsi_transport_iscsi.h> 38 #include <trace/events/iscsi.h> 39 #include <trace/events/sock.h> 40 41 #include "iscsi_tcp.h" 42 43 MODULE_AUTHOR("Mike Christie <michaelc@cs.wisc.edu>, " 44 "Dmitry Yusupov <dmitry_yus@yahoo.com>, " 45 "Alex Aizman <itn780@yahoo.com>"); 46 MODULE_DESCRIPTION("iSCSI/TCP data-path"); 47 MODULE_LICENSE("GPL"); 48 49 static struct scsi_transport_template *iscsi_sw_tcp_scsi_transport; 50 static const struct scsi_host_template iscsi_sw_tcp_sht; 51 static struct iscsi_transport iscsi_sw_tcp_transport; 52 53 static unsigned int iscsi_max_lun = ~0; 54 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO); 55 56 static bool iscsi_recv_from_iscsi_q; 57 module_param_named(recv_from_iscsi_q, iscsi_recv_from_iscsi_q, bool, 0644); 58 MODULE_PARM_DESC(recv_from_iscsi_q, "Set to true to read iSCSI data/headers from the iscsi_q workqueue. The default is false which will perform reads from the network softirq context."); 59 60 static int iscsi_sw_tcp_dbg; 61 module_param_named(debug_iscsi_tcp, iscsi_sw_tcp_dbg, int, 62 S_IRUGO | S_IWUSR); 63 MODULE_PARM_DESC(debug_iscsi_tcp, "Turn on debugging for iscsi_tcp module " 64 "Set to 1 to turn on, and zero to turn off. Default is off."); 65 66 #define ISCSI_SW_TCP_DBG(_conn, dbg_fmt, arg...) \ 67 do { \ 68 if (iscsi_sw_tcp_dbg) \ 69 iscsi_conn_printk(KERN_INFO, _conn, \ 70 "%s " dbg_fmt, \ 71 __func__, ##arg); \ 72 iscsi_dbg_trace(trace_iscsi_dbg_sw_tcp, \ 73 &(_conn)->cls_conn->dev, \ 74 "%s " dbg_fmt, __func__, ##arg);\ 75 } while (0); 76 77 78 /** 79 * iscsi_sw_tcp_recv - TCP receive in sendfile fashion 80 * @rd_desc: read descriptor 81 * @skb: socket buffer 82 * @offset: offset in skb 83 * @len: skb->len - offset 84 */ 85 static int iscsi_sw_tcp_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 86 unsigned int offset, size_t len) 87 { 88 struct iscsi_conn *conn = rd_desc->arg.data; 89 unsigned int consumed, total_consumed = 0; 90 int status; 91 92 ISCSI_SW_TCP_DBG(conn, "in %d bytes\n", skb->len - offset); 93 94 do { 95 status = 0; 96 consumed = iscsi_tcp_recv_skb(conn, skb, offset, 0, &status); 97 offset += consumed; 98 total_consumed += consumed; 99 } while (consumed != 0 && status != ISCSI_TCP_SKB_DONE); 100 101 ISCSI_SW_TCP_DBG(conn, "read %d bytes status %d\n", 102 skb->len - offset, status); 103 return total_consumed; 104 } 105 106 /** 107 * iscsi_sw_sk_state_check - check socket state 108 * @sk: socket 109 * 110 * If the socket is in CLOSE or CLOSE_WAIT we should 111 * not close the connection if there is still some 112 * data pending. 113 * 114 * Must be called with sk_callback_lock. 115 */ 116 static inline int iscsi_sw_sk_state_check(struct sock *sk) 117 { 118 struct iscsi_conn *conn = sk->sk_user_data; 119 120 if ((sk->sk_state == TCP_CLOSE_WAIT || sk->sk_state == TCP_CLOSE) && 121 (conn->session->state != ISCSI_STATE_LOGGING_OUT) && 122 !atomic_read(&sk->sk_rmem_alloc)) { 123 ISCSI_SW_TCP_DBG(conn, "TCP_CLOSE|TCP_CLOSE_WAIT\n"); 124 iscsi_conn_failure(conn, ISCSI_ERR_TCP_CONN_CLOSE); 125 return -ECONNRESET; 126 } 127 return 0; 128 } 129 130 static void iscsi_sw_tcp_recv_data(struct iscsi_conn *conn) 131 { 132 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 133 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 134 struct sock *sk = tcp_sw_conn->sock->sk; 135 read_descriptor_t rd_desc; 136 137 /* 138 * Use rd_desc to pass 'conn' to iscsi_tcp_recv. 139 * We set count to 1 because we want the network layer to 140 * hand us all the skbs that are available. iscsi_tcp_recv 141 * handled pdus that cross buffers or pdus that still need data. 142 */ 143 rd_desc.arg.data = conn; 144 rd_desc.count = 1; 145 146 tcp_read_sock(sk, &rd_desc, iscsi_sw_tcp_recv); 147 148 /* If we had to (atomically) map a highmem page, 149 * unmap it now. */ 150 iscsi_tcp_segment_unmap(&tcp_conn->in.segment); 151 152 iscsi_sw_sk_state_check(sk); 153 } 154 155 static void iscsi_sw_tcp_recv_data_work(struct work_struct *work) 156 { 157 struct iscsi_conn *conn = container_of(work, struct iscsi_conn, 158 recvwork); 159 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 160 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 161 struct sock *sk = tcp_sw_conn->sock->sk; 162 163 lock_sock(sk); 164 iscsi_sw_tcp_recv_data(conn); 165 release_sock(sk); 166 } 167 168 static void iscsi_sw_tcp_data_ready(struct sock *sk) 169 { 170 struct iscsi_sw_tcp_conn *tcp_sw_conn; 171 struct iscsi_tcp_conn *tcp_conn; 172 struct iscsi_conn *conn; 173 174 trace_sk_data_ready(sk); 175 176 read_lock_bh(&sk->sk_callback_lock); 177 conn = sk->sk_user_data; 178 if (!conn) { 179 read_unlock_bh(&sk->sk_callback_lock); 180 return; 181 } 182 tcp_conn = conn->dd_data; 183 tcp_sw_conn = tcp_conn->dd_data; 184 185 if (tcp_sw_conn->queue_recv) 186 iscsi_conn_queue_recv(conn); 187 else 188 iscsi_sw_tcp_recv_data(conn); 189 read_unlock_bh(&sk->sk_callback_lock); 190 } 191 192 static void iscsi_sw_tcp_state_change(struct sock *sk) 193 { 194 struct iscsi_tcp_conn *tcp_conn; 195 struct iscsi_sw_tcp_conn *tcp_sw_conn; 196 struct iscsi_conn *conn; 197 void (*old_state_change)(struct sock *); 198 199 read_lock_bh(&sk->sk_callback_lock); 200 conn = sk->sk_user_data; 201 if (!conn) { 202 read_unlock_bh(&sk->sk_callback_lock); 203 return; 204 } 205 206 iscsi_sw_sk_state_check(sk); 207 208 tcp_conn = conn->dd_data; 209 tcp_sw_conn = tcp_conn->dd_data; 210 old_state_change = tcp_sw_conn->old_state_change; 211 212 read_unlock_bh(&sk->sk_callback_lock); 213 214 old_state_change(sk); 215 } 216 217 /** 218 * iscsi_sw_tcp_write_space - Called when more output buffer space is available 219 * @sk: socket space is available for 220 **/ 221 static void iscsi_sw_tcp_write_space(struct sock *sk) 222 { 223 struct iscsi_conn *conn; 224 struct iscsi_tcp_conn *tcp_conn; 225 struct iscsi_sw_tcp_conn *tcp_sw_conn; 226 void (*old_write_space)(struct sock *); 227 228 read_lock_bh(&sk->sk_callback_lock); 229 conn = sk->sk_user_data; 230 if (!conn) { 231 read_unlock_bh(&sk->sk_callback_lock); 232 return; 233 } 234 235 tcp_conn = conn->dd_data; 236 tcp_sw_conn = tcp_conn->dd_data; 237 old_write_space = tcp_sw_conn->old_write_space; 238 read_unlock_bh(&sk->sk_callback_lock); 239 240 old_write_space(sk); 241 242 ISCSI_SW_TCP_DBG(conn, "iscsi_write_space\n"); 243 iscsi_conn_queue_xmit(conn); 244 } 245 246 static void iscsi_sw_tcp_conn_set_callbacks(struct iscsi_conn *conn) 247 { 248 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 249 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 250 struct sock *sk = tcp_sw_conn->sock->sk; 251 252 /* assign new callbacks */ 253 write_lock_bh(&sk->sk_callback_lock); 254 sk->sk_user_data = conn; 255 tcp_sw_conn->old_data_ready = sk->sk_data_ready; 256 tcp_sw_conn->old_state_change = sk->sk_state_change; 257 tcp_sw_conn->old_write_space = sk->sk_write_space; 258 sk->sk_data_ready = iscsi_sw_tcp_data_ready; 259 sk->sk_state_change = iscsi_sw_tcp_state_change; 260 sk->sk_write_space = iscsi_sw_tcp_write_space; 261 write_unlock_bh(&sk->sk_callback_lock); 262 } 263 264 static void 265 iscsi_sw_tcp_conn_restore_callbacks(struct iscsi_conn *conn) 266 { 267 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 268 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 269 struct sock *sk = tcp_sw_conn->sock->sk; 270 271 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */ 272 write_lock_bh(&sk->sk_callback_lock); 273 sk->sk_user_data = NULL; 274 sk->sk_data_ready = tcp_sw_conn->old_data_ready; 275 sk->sk_state_change = tcp_sw_conn->old_state_change; 276 sk->sk_write_space = tcp_sw_conn->old_write_space; 277 sk->sk_no_check_tx = 0; 278 write_unlock_bh(&sk->sk_callback_lock); 279 } 280 281 /** 282 * iscsi_sw_tcp_xmit_segment - transmit segment 283 * @tcp_conn: the iSCSI TCP connection 284 * @segment: the buffer to transmnit 285 * 286 * This function transmits as much of the buffer as 287 * the network layer will accept, and returns the number of 288 * bytes transmitted. 289 * 290 * If CRC hashing is enabled, the function will compute the 291 * hash as it goes. When the entire segment has been transmitted, 292 * it will retrieve the hash value and send it as well. 293 */ 294 static int iscsi_sw_tcp_xmit_segment(struct iscsi_tcp_conn *tcp_conn, 295 struct iscsi_segment *segment) 296 { 297 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 298 struct socket *sk = tcp_sw_conn->sock; 299 unsigned int copied = 0; 300 int r = 0; 301 302 while (!iscsi_tcp_segment_done(tcp_conn, segment, 0, r)) { 303 struct scatterlist *sg; 304 struct msghdr msg = {}; 305 struct bio_vec bv; 306 unsigned int offset, copy; 307 308 r = 0; 309 offset = segment->copied; 310 copy = segment->size - offset; 311 312 if (segment->total_copied + segment->size < segment->total_size) 313 msg.msg_flags |= MSG_MORE; 314 315 if (tcp_sw_conn->queue_recv) 316 msg.msg_flags |= MSG_DONTWAIT; 317 318 if (!segment->data) { 319 if (!tcp_conn->iscsi_conn->datadgst_en) 320 msg.msg_flags |= MSG_SPLICE_PAGES; 321 sg = segment->sg; 322 offset += segment->sg_offset + sg->offset; 323 bvec_set_page(&bv, sg_page(sg), copy, offset); 324 } else { 325 bvec_set_virt(&bv, segment->data + offset, copy); 326 } 327 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bv, 1, copy); 328 329 r = sock_sendmsg(sk, &msg); 330 if (r < 0) { 331 iscsi_tcp_segment_unmap(segment); 332 return r; 333 } 334 copied += r; 335 } 336 return copied; 337 } 338 339 /** 340 * iscsi_sw_tcp_xmit - TCP transmit 341 * @conn: iscsi connection 342 **/ 343 static int iscsi_sw_tcp_xmit(struct iscsi_conn *conn) 344 { 345 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 346 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 347 struct iscsi_segment *segment = &tcp_sw_conn->out.segment; 348 unsigned int consumed = 0; 349 int rc = 0; 350 351 while (1) { 352 rc = iscsi_sw_tcp_xmit_segment(tcp_conn, segment); 353 /* 354 * We may not have been able to send data because the conn 355 * is getting stopped. libiscsi will know so propagate err 356 * for it to do the right thing. 357 */ 358 if (rc == -EAGAIN) 359 return rc; 360 else if (rc < 0) { 361 rc = ISCSI_ERR_XMIT_FAILED; 362 goto error; 363 } else if (rc == 0) 364 break; 365 366 consumed += rc; 367 368 if (segment->total_copied >= segment->total_size) { 369 if (segment->done != NULL) { 370 rc = segment->done(tcp_conn, segment); 371 if (rc != 0) 372 goto error; 373 } 374 } 375 } 376 377 ISCSI_SW_TCP_DBG(conn, "xmit %d bytes\n", consumed); 378 379 conn->txdata_octets += consumed; 380 return consumed; 381 382 error: 383 /* Transmit error. We could initiate error recovery 384 * here. */ 385 ISCSI_SW_TCP_DBG(conn, "Error sending PDU, errno=%d\n", rc); 386 iscsi_conn_failure(conn, rc); 387 return -EIO; 388 } 389 390 /** 391 * iscsi_sw_tcp_xmit_qlen - return the number of bytes queued for xmit 392 * @conn: iscsi connection 393 */ 394 static inline int iscsi_sw_tcp_xmit_qlen(struct iscsi_conn *conn) 395 { 396 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 397 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 398 struct iscsi_segment *segment = &tcp_sw_conn->out.segment; 399 400 return segment->total_copied - segment->total_size; 401 } 402 403 static int iscsi_sw_tcp_pdu_xmit(struct iscsi_task *task) 404 { 405 struct iscsi_conn *conn = task->conn; 406 unsigned int noreclaim_flag; 407 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 408 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 409 int rc = 0; 410 411 if (!tcp_sw_conn->sock) { 412 iscsi_conn_printk(KERN_ERR, conn, 413 "Transport not bound to socket!\n"); 414 return -EINVAL; 415 } 416 417 noreclaim_flag = memalloc_noreclaim_save(); 418 419 while (iscsi_sw_tcp_xmit_qlen(conn)) { 420 rc = iscsi_sw_tcp_xmit(conn); 421 if (rc == 0) { 422 rc = -EAGAIN; 423 break; 424 } 425 if (rc < 0) 426 break; 427 rc = 0; 428 } 429 430 memalloc_noreclaim_restore(noreclaim_flag); 431 return rc; 432 } 433 434 /* 435 * This is called when we're done sending the header. 436 * Simply copy the data_segment to the send segment, and return. 437 */ 438 static int iscsi_sw_tcp_send_hdr_done(struct iscsi_tcp_conn *tcp_conn, 439 struct iscsi_segment *segment) 440 { 441 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 442 443 tcp_sw_conn->out.segment = tcp_sw_conn->out.data_segment; 444 ISCSI_SW_TCP_DBG(tcp_conn->iscsi_conn, 445 "Header done. Next segment size %u total_size %u\n", 446 tcp_sw_conn->out.segment.size, 447 tcp_sw_conn->out.segment.total_size); 448 return 0; 449 } 450 451 static void iscsi_sw_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, 452 size_t hdrlen) 453 { 454 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 455 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 456 457 ISCSI_SW_TCP_DBG(conn, "%s\n", conn->hdrdgst_en ? 458 "digest enabled" : "digest disabled"); 459 460 /* Clear the data segment - needs to be filled in by the 461 * caller using iscsi_tcp_send_data_prep() */ 462 memset(&tcp_sw_conn->out.data_segment, 0, 463 sizeof(struct iscsi_segment)); 464 465 /* If header digest is enabled, compute the CRC and 466 * place the digest into the same buffer. We make 467 * sure that both iscsi_tcp_task and mtask have 468 * sufficient room. 469 */ 470 if (conn->hdrdgst_en) { 471 iscsi_tcp_dgst_header(tcp_sw_conn->tx_hash, hdr, hdrlen, 472 hdr + hdrlen); 473 hdrlen += ISCSI_DIGEST_SIZE; 474 } 475 476 /* Remember header pointer for later, when we need 477 * to decide whether there's a payload to go along 478 * with the header. */ 479 tcp_sw_conn->out.hdr = hdr; 480 481 iscsi_segment_init_linear(&tcp_sw_conn->out.segment, hdr, hdrlen, 482 iscsi_sw_tcp_send_hdr_done, NULL); 483 } 484 485 /* 486 * Prepare the send buffer for the payload data. 487 * Padding and checksumming will all be taken care 488 * of by the iscsi_segment routines. 489 */ 490 static int 491 iscsi_sw_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg, 492 unsigned int count, unsigned int offset, 493 unsigned int len) 494 { 495 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 496 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 497 struct ahash_request *tx_hash = NULL; 498 unsigned int hdr_spec_len; 499 500 ISCSI_SW_TCP_DBG(conn, "offset=%d, datalen=%d %s\n", offset, len, 501 conn->datadgst_en ? 502 "digest enabled" : "digest disabled"); 503 504 /* Make sure the datalen matches what the caller 505 said he would send. */ 506 hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength); 507 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len)); 508 509 if (conn->datadgst_en) 510 tx_hash = tcp_sw_conn->tx_hash; 511 512 return iscsi_segment_seek_sg(&tcp_sw_conn->out.data_segment, 513 sg, count, offset, len, 514 NULL, tx_hash); 515 } 516 517 static void 518 iscsi_sw_tcp_send_linear_data_prep(struct iscsi_conn *conn, void *data, 519 size_t len) 520 { 521 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 522 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 523 struct ahash_request *tx_hash = NULL; 524 unsigned int hdr_spec_len; 525 526 ISCSI_SW_TCP_DBG(conn, "datalen=%zd %s\n", len, conn->datadgst_en ? 527 "digest enabled" : "digest disabled"); 528 529 /* Make sure the datalen matches what the caller 530 said he would send. */ 531 hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength); 532 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len)); 533 534 if (conn->datadgst_en) 535 tx_hash = tcp_sw_conn->tx_hash; 536 537 iscsi_segment_init_linear(&tcp_sw_conn->out.data_segment, 538 data, len, NULL, tx_hash); 539 } 540 541 static int iscsi_sw_tcp_pdu_init(struct iscsi_task *task, 542 unsigned int offset, unsigned int count) 543 { 544 struct iscsi_conn *conn = task->conn; 545 int err = 0; 546 547 iscsi_sw_tcp_send_hdr_prep(conn, task->hdr, task->hdr_len); 548 549 if (!count) 550 return 0; 551 552 if (!task->sc) 553 iscsi_sw_tcp_send_linear_data_prep(conn, task->data, count); 554 else { 555 struct scsi_data_buffer *sdb = &task->sc->sdb; 556 557 err = iscsi_sw_tcp_send_data_prep(conn, sdb->table.sgl, 558 sdb->table.nents, offset, 559 count); 560 } 561 562 if (err) { 563 /* got invalid offset/len */ 564 return -EIO; 565 } 566 return 0; 567 } 568 569 static int iscsi_sw_tcp_pdu_alloc(struct iscsi_task *task, uint8_t opcode) 570 { 571 struct iscsi_tcp_task *tcp_task = task->dd_data; 572 573 task->hdr = task->dd_data + sizeof(*tcp_task); 574 task->hdr_max = sizeof(struct iscsi_sw_tcp_hdrbuf) - ISCSI_DIGEST_SIZE; 575 return 0; 576 } 577 578 static struct iscsi_cls_conn * 579 iscsi_sw_tcp_conn_create(struct iscsi_cls_session *cls_session, 580 uint32_t conn_idx) 581 { 582 struct iscsi_conn *conn; 583 struct iscsi_cls_conn *cls_conn; 584 struct iscsi_tcp_conn *tcp_conn; 585 struct iscsi_sw_tcp_conn *tcp_sw_conn; 586 struct crypto_ahash *tfm; 587 588 cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*tcp_sw_conn), 589 conn_idx); 590 if (!cls_conn) 591 return NULL; 592 conn = cls_conn->dd_data; 593 tcp_conn = conn->dd_data; 594 tcp_sw_conn = tcp_conn->dd_data; 595 INIT_WORK(&conn->recvwork, iscsi_sw_tcp_recv_data_work); 596 tcp_sw_conn->queue_recv = iscsi_recv_from_iscsi_q; 597 598 mutex_init(&tcp_sw_conn->sock_lock); 599 600 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC); 601 if (IS_ERR(tfm)) 602 goto free_conn; 603 604 tcp_sw_conn->tx_hash = ahash_request_alloc(tfm, GFP_KERNEL); 605 if (!tcp_sw_conn->tx_hash) 606 goto free_tfm; 607 ahash_request_set_callback(tcp_sw_conn->tx_hash, 0, NULL, NULL); 608 609 tcp_sw_conn->rx_hash = ahash_request_alloc(tfm, GFP_KERNEL); 610 if (!tcp_sw_conn->rx_hash) 611 goto free_tx_hash; 612 ahash_request_set_callback(tcp_sw_conn->rx_hash, 0, NULL, NULL); 613 614 tcp_conn->rx_hash = tcp_sw_conn->rx_hash; 615 616 return cls_conn; 617 618 free_tx_hash: 619 ahash_request_free(tcp_sw_conn->tx_hash); 620 free_tfm: 621 crypto_free_ahash(tfm); 622 free_conn: 623 iscsi_conn_printk(KERN_ERR, conn, 624 "Could not create connection due to crc32c " 625 "loading error. Make sure the crc32c " 626 "module is built as a module or into the " 627 "kernel\n"); 628 iscsi_tcp_conn_teardown(cls_conn); 629 return NULL; 630 } 631 632 static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn) 633 { 634 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 635 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 636 struct socket *sock = tcp_sw_conn->sock; 637 638 /* 639 * The iscsi transport class will make sure we are not called in 640 * parallel with start, stop, bind and destroys. However, this can be 641 * called twice if userspace does a stop then a destroy. 642 */ 643 if (!sock) 644 return; 645 646 /* 647 * Make sure we start socket shutdown now in case userspace is up 648 * but delayed in releasing the socket. 649 */ 650 kernel_sock_shutdown(sock, SHUT_RDWR); 651 652 sock_hold(sock->sk); 653 iscsi_sw_tcp_conn_restore_callbacks(conn); 654 sock_put(sock->sk); 655 656 iscsi_suspend_rx(conn); 657 658 mutex_lock(&tcp_sw_conn->sock_lock); 659 tcp_sw_conn->sock = NULL; 660 mutex_unlock(&tcp_sw_conn->sock_lock); 661 sockfd_put(sock); 662 } 663 664 static void iscsi_sw_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn) 665 { 666 struct iscsi_conn *conn = cls_conn->dd_data; 667 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 668 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 669 670 iscsi_sw_tcp_release_conn(conn); 671 672 ahash_request_free(tcp_sw_conn->rx_hash); 673 if (tcp_sw_conn->tx_hash) { 674 struct crypto_ahash *tfm; 675 676 tfm = crypto_ahash_reqtfm(tcp_sw_conn->tx_hash); 677 ahash_request_free(tcp_sw_conn->tx_hash); 678 crypto_free_ahash(tfm); 679 } 680 681 iscsi_tcp_conn_teardown(cls_conn); 682 } 683 684 static void iscsi_sw_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) 685 { 686 struct iscsi_conn *conn = cls_conn->dd_data; 687 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 688 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 689 struct socket *sock = tcp_sw_conn->sock; 690 691 /* userspace may have goofed up and not bound us */ 692 if (!sock) 693 return; 694 695 sock->sk->sk_err = EIO; 696 wake_up_interruptible(sk_sleep(sock->sk)); 697 698 /* stop xmit side */ 699 iscsi_suspend_tx(conn); 700 701 /* stop recv side and release socket */ 702 iscsi_sw_tcp_release_conn(conn); 703 704 iscsi_conn_stop(cls_conn, flag); 705 } 706 707 static int 708 iscsi_sw_tcp_conn_bind(struct iscsi_cls_session *cls_session, 709 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph, 710 int is_leading) 711 { 712 struct iscsi_conn *conn = cls_conn->dd_data; 713 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 714 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 715 struct sock *sk; 716 struct socket *sock; 717 int err; 718 719 /* lookup for existing socket */ 720 sock = sockfd_lookup((int)transport_eph, &err); 721 if (!sock) { 722 iscsi_conn_printk(KERN_ERR, conn, 723 "sockfd_lookup failed %d\n", err); 724 return -EEXIST; 725 } 726 727 err = iscsi_conn_bind(cls_session, cls_conn, is_leading); 728 if (err) 729 goto free_socket; 730 731 mutex_lock(&tcp_sw_conn->sock_lock); 732 /* bind iSCSI connection and socket */ 733 tcp_sw_conn->sock = sock; 734 mutex_unlock(&tcp_sw_conn->sock_lock); 735 736 /* setup Socket parameters */ 737 sk = sock->sk; 738 sk->sk_reuse = SK_CAN_REUSE; 739 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */ 740 sk->sk_allocation = GFP_ATOMIC; 741 sk->sk_use_task_frag = false; 742 sk_set_memalloc(sk); 743 sock_no_linger(sk); 744 745 iscsi_sw_tcp_conn_set_callbacks(conn); 746 /* 747 * set receive state machine into initial state 748 */ 749 iscsi_tcp_hdr_recv_prep(tcp_conn); 750 return 0; 751 752 free_socket: 753 sockfd_put(sock); 754 return err; 755 } 756 757 static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn, 758 enum iscsi_param param, char *buf, 759 int buflen) 760 { 761 struct iscsi_conn *conn = cls_conn->dd_data; 762 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 763 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 764 765 switch(param) { 766 case ISCSI_PARAM_HDRDGST_EN: 767 iscsi_set_param(cls_conn, param, buf, buflen); 768 break; 769 case ISCSI_PARAM_DATADGST_EN: 770 mutex_lock(&tcp_sw_conn->sock_lock); 771 if (!tcp_sw_conn->sock) { 772 mutex_unlock(&tcp_sw_conn->sock_lock); 773 return -ENOTCONN; 774 } 775 iscsi_set_param(cls_conn, param, buf, buflen); 776 mutex_unlock(&tcp_sw_conn->sock_lock); 777 break; 778 case ISCSI_PARAM_MAX_R2T: 779 return iscsi_tcp_set_max_r2t(conn, buf); 780 default: 781 return iscsi_set_param(cls_conn, param, buf, buflen); 782 } 783 784 return 0; 785 } 786 787 static int iscsi_sw_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn, 788 enum iscsi_param param, char *buf) 789 { 790 struct iscsi_conn *conn = cls_conn->dd_data; 791 struct iscsi_sw_tcp_conn *tcp_sw_conn; 792 struct iscsi_tcp_conn *tcp_conn; 793 struct sockaddr_in6 addr; 794 struct socket *sock; 795 int rc; 796 797 switch(param) { 798 case ISCSI_PARAM_CONN_PORT: 799 case ISCSI_PARAM_CONN_ADDRESS: 800 case ISCSI_PARAM_LOCAL_PORT: 801 spin_lock_bh(&conn->session->frwd_lock); 802 if (!conn->session->leadconn) { 803 spin_unlock_bh(&conn->session->frwd_lock); 804 return -ENOTCONN; 805 } 806 /* 807 * The conn has been setup and bound, so just grab a ref 808 * incase a destroy runs while we are in the net layer. 809 */ 810 iscsi_get_conn(conn->cls_conn); 811 spin_unlock_bh(&conn->session->frwd_lock); 812 813 tcp_conn = conn->dd_data; 814 tcp_sw_conn = tcp_conn->dd_data; 815 816 mutex_lock(&tcp_sw_conn->sock_lock); 817 sock = tcp_sw_conn->sock; 818 if (!sock) { 819 rc = -ENOTCONN; 820 goto sock_unlock; 821 } 822 823 if (param == ISCSI_PARAM_LOCAL_PORT) 824 rc = kernel_getsockname(sock, 825 (struct sockaddr *)&addr); 826 else 827 rc = kernel_getpeername(sock, 828 (struct sockaddr *)&addr); 829 sock_unlock: 830 mutex_unlock(&tcp_sw_conn->sock_lock); 831 iscsi_put_conn(conn->cls_conn); 832 if (rc < 0) 833 return rc; 834 835 return iscsi_conn_get_addr_param((struct sockaddr_storage *) 836 &addr, param, buf); 837 default: 838 return iscsi_conn_get_param(cls_conn, param, buf); 839 } 840 841 return 0; 842 } 843 844 static int iscsi_sw_tcp_host_get_param(struct Scsi_Host *shost, 845 enum iscsi_host_param param, char *buf) 846 { 847 struct iscsi_sw_tcp_host *tcp_sw_host = iscsi_host_priv(shost); 848 struct iscsi_session *session; 849 struct iscsi_conn *conn; 850 struct iscsi_tcp_conn *tcp_conn; 851 struct iscsi_sw_tcp_conn *tcp_sw_conn; 852 struct sockaddr_in6 addr; 853 struct socket *sock; 854 int rc; 855 856 switch (param) { 857 case ISCSI_HOST_PARAM_IPADDRESS: 858 session = tcp_sw_host->session; 859 if (!session) 860 return -ENOTCONN; 861 862 spin_lock_bh(&session->frwd_lock); 863 conn = session->leadconn; 864 if (!conn) { 865 spin_unlock_bh(&session->frwd_lock); 866 return -ENOTCONN; 867 } 868 tcp_conn = conn->dd_data; 869 tcp_sw_conn = tcp_conn->dd_data; 870 /* 871 * The conn has been setup and bound, so just grab a ref 872 * incase a destroy runs while we are in the net layer. 873 */ 874 iscsi_get_conn(conn->cls_conn); 875 spin_unlock_bh(&session->frwd_lock); 876 877 mutex_lock(&tcp_sw_conn->sock_lock); 878 sock = tcp_sw_conn->sock; 879 if (!sock) 880 rc = -ENOTCONN; 881 else 882 rc = kernel_getsockname(sock, (struct sockaddr *)&addr); 883 mutex_unlock(&tcp_sw_conn->sock_lock); 884 iscsi_put_conn(conn->cls_conn); 885 if (rc < 0) 886 return rc; 887 888 return iscsi_conn_get_addr_param((struct sockaddr_storage *) 889 &addr, 890 (enum iscsi_param)param, buf); 891 default: 892 return iscsi_host_get_param(shost, param, buf); 893 } 894 895 return 0; 896 } 897 898 static void 899 iscsi_sw_tcp_conn_get_stats(struct iscsi_cls_conn *cls_conn, 900 struct iscsi_stats *stats) 901 { 902 struct iscsi_conn *conn = cls_conn->dd_data; 903 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 904 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 905 906 stats->custom_length = 3; 907 strcpy(stats->custom[0].desc, "tx_sendpage_failures"); 908 stats->custom[0].value = tcp_sw_conn->sendpage_failures_cnt; 909 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr"); 910 stats->custom[1].value = tcp_sw_conn->discontiguous_hdr_cnt; 911 strcpy(stats->custom[2].desc, "eh_abort_cnt"); 912 stats->custom[2].value = conn->eh_abort_cnt; 913 914 iscsi_tcp_conn_get_stats(cls_conn, stats); 915 } 916 917 static struct iscsi_cls_session * 918 iscsi_sw_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max, 919 uint16_t qdepth, uint32_t initial_cmdsn) 920 { 921 struct iscsi_cls_session *cls_session; 922 struct iscsi_session *session; 923 struct iscsi_sw_tcp_host *tcp_sw_host; 924 struct Scsi_Host *shost; 925 int rc; 926 927 if (ep) { 928 printk(KERN_ERR "iscsi_tcp: invalid ep %p.\n", ep); 929 return NULL; 930 } 931 932 shost = iscsi_host_alloc(&iscsi_sw_tcp_sht, 933 sizeof(struct iscsi_sw_tcp_host), 1); 934 if (!shost) 935 return NULL; 936 shost->transportt = iscsi_sw_tcp_scsi_transport; 937 shost->cmd_per_lun = qdepth; 938 shost->max_lun = iscsi_max_lun; 939 shost->max_id = 0; 940 shost->max_channel = 0; 941 shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE; 942 943 rc = iscsi_host_get_max_scsi_cmds(shost, cmds_max); 944 if (rc < 0) 945 goto free_host; 946 shost->can_queue = rc; 947 948 if (iscsi_host_add(shost, NULL)) 949 goto free_host; 950 951 cls_session = iscsi_session_setup(&iscsi_sw_tcp_transport, shost, 952 cmds_max, 0, 953 sizeof(struct iscsi_tcp_task) + 954 sizeof(struct iscsi_sw_tcp_hdrbuf), 955 initial_cmdsn, 0); 956 if (!cls_session) 957 goto remove_host; 958 session = cls_session->dd_data; 959 960 if (iscsi_tcp_r2tpool_alloc(session)) 961 goto remove_session; 962 963 /* We are now fully setup so expose the session to sysfs. */ 964 tcp_sw_host = iscsi_host_priv(shost); 965 tcp_sw_host->session = session; 966 return cls_session; 967 968 remove_session: 969 iscsi_session_teardown(cls_session); 970 remove_host: 971 iscsi_host_remove(shost, false); 972 free_host: 973 iscsi_host_free(shost); 974 return NULL; 975 } 976 977 static void iscsi_sw_tcp_session_destroy(struct iscsi_cls_session *cls_session) 978 { 979 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 980 struct iscsi_session *session = cls_session->dd_data; 981 982 if (WARN_ON_ONCE(session->leadconn)) 983 return; 984 985 iscsi_session_remove(cls_session); 986 /* 987 * Our get_host_param needs to access the session, so remove the 988 * host from sysfs before freeing the session to make sure userspace 989 * is no longer accessing the callout. 990 */ 991 iscsi_host_remove(shost, false); 992 993 iscsi_tcp_r2tpool_free(cls_session->dd_data); 994 995 iscsi_session_free(cls_session); 996 iscsi_host_free(shost); 997 } 998 999 static umode_t iscsi_sw_tcp_attr_is_visible(int param_type, int param) 1000 { 1001 switch (param_type) { 1002 case ISCSI_HOST_PARAM: 1003 switch (param) { 1004 case ISCSI_HOST_PARAM_NETDEV_NAME: 1005 case ISCSI_HOST_PARAM_HWADDRESS: 1006 case ISCSI_HOST_PARAM_IPADDRESS: 1007 case ISCSI_HOST_PARAM_INITIATOR_NAME: 1008 return S_IRUGO; 1009 default: 1010 return 0; 1011 } 1012 case ISCSI_PARAM: 1013 switch (param) { 1014 case ISCSI_PARAM_MAX_RECV_DLENGTH: 1015 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 1016 case ISCSI_PARAM_HDRDGST_EN: 1017 case ISCSI_PARAM_DATADGST_EN: 1018 case ISCSI_PARAM_CONN_ADDRESS: 1019 case ISCSI_PARAM_CONN_PORT: 1020 case ISCSI_PARAM_LOCAL_PORT: 1021 case ISCSI_PARAM_EXP_STATSN: 1022 case ISCSI_PARAM_PERSISTENT_ADDRESS: 1023 case ISCSI_PARAM_PERSISTENT_PORT: 1024 case ISCSI_PARAM_PING_TMO: 1025 case ISCSI_PARAM_RECV_TMO: 1026 case ISCSI_PARAM_INITIAL_R2T_EN: 1027 case ISCSI_PARAM_MAX_R2T: 1028 case ISCSI_PARAM_IMM_DATA_EN: 1029 case ISCSI_PARAM_FIRST_BURST: 1030 case ISCSI_PARAM_MAX_BURST: 1031 case ISCSI_PARAM_PDU_INORDER_EN: 1032 case ISCSI_PARAM_DATASEQ_INORDER_EN: 1033 case ISCSI_PARAM_ERL: 1034 case ISCSI_PARAM_TARGET_NAME: 1035 case ISCSI_PARAM_TPGT: 1036 case ISCSI_PARAM_USERNAME: 1037 case ISCSI_PARAM_PASSWORD: 1038 case ISCSI_PARAM_USERNAME_IN: 1039 case ISCSI_PARAM_PASSWORD_IN: 1040 case ISCSI_PARAM_FAST_ABORT: 1041 case ISCSI_PARAM_ABORT_TMO: 1042 case ISCSI_PARAM_LU_RESET_TMO: 1043 case ISCSI_PARAM_TGT_RESET_TMO: 1044 case ISCSI_PARAM_IFACE_NAME: 1045 case ISCSI_PARAM_INITIATOR_NAME: 1046 return S_IRUGO; 1047 default: 1048 return 0; 1049 } 1050 } 1051 1052 return 0; 1053 } 1054 1055 static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev) 1056 { 1057 struct iscsi_sw_tcp_host *tcp_sw_host = iscsi_host_priv(sdev->host); 1058 struct iscsi_session *session = tcp_sw_host->session; 1059 struct iscsi_conn *conn = session->leadconn; 1060 1061 if (conn->datadgst_en) 1062 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, 1063 sdev->request_queue); 1064 blk_queue_dma_alignment(sdev->request_queue, 0); 1065 return 0; 1066 } 1067 1068 static const struct scsi_host_template iscsi_sw_tcp_sht = { 1069 .module = THIS_MODULE, 1070 .name = "iSCSI Initiator over TCP/IP", 1071 .queuecommand = iscsi_queuecommand, 1072 .change_queue_depth = scsi_change_queue_depth, 1073 .can_queue = ISCSI_TOTAL_CMDS_MAX, 1074 .sg_tablesize = 4096, 1075 .max_sectors = 0xFFFF, 1076 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, 1077 .eh_timed_out = iscsi_eh_cmd_timed_out, 1078 .eh_abort_handler = iscsi_eh_abort, 1079 .eh_device_reset_handler= iscsi_eh_device_reset, 1080 .eh_target_reset_handler = iscsi_eh_recover_target, 1081 .dma_boundary = PAGE_SIZE - 1, 1082 .slave_configure = iscsi_sw_tcp_slave_configure, 1083 .proc_name = "iscsi_tcp", 1084 .this_id = -1, 1085 .track_queue_depth = 1, 1086 .cmd_size = sizeof(struct iscsi_cmd), 1087 }; 1088 1089 static struct iscsi_transport iscsi_sw_tcp_transport = { 1090 .owner = THIS_MODULE, 1091 .name = "tcp", 1092 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST 1093 | CAP_DATADGST, 1094 /* session management */ 1095 .create_session = iscsi_sw_tcp_session_create, 1096 .destroy_session = iscsi_sw_tcp_session_destroy, 1097 /* connection management */ 1098 .create_conn = iscsi_sw_tcp_conn_create, 1099 .bind_conn = iscsi_sw_tcp_conn_bind, 1100 .destroy_conn = iscsi_sw_tcp_conn_destroy, 1101 .attr_is_visible = iscsi_sw_tcp_attr_is_visible, 1102 .set_param = iscsi_sw_tcp_conn_set_param, 1103 .get_conn_param = iscsi_sw_tcp_conn_get_param, 1104 .get_session_param = iscsi_session_get_param, 1105 .start_conn = iscsi_conn_start, 1106 .stop_conn = iscsi_sw_tcp_conn_stop, 1107 /* iscsi host params */ 1108 .get_host_param = iscsi_sw_tcp_host_get_param, 1109 .set_host_param = iscsi_host_set_param, 1110 /* IO */ 1111 .send_pdu = iscsi_conn_send_pdu, 1112 .get_stats = iscsi_sw_tcp_conn_get_stats, 1113 /* iscsi task/cmd helpers */ 1114 .init_task = iscsi_tcp_task_init, 1115 .xmit_task = iscsi_tcp_task_xmit, 1116 .cleanup_task = iscsi_tcp_cleanup_task, 1117 /* low level pdu helpers */ 1118 .xmit_pdu = iscsi_sw_tcp_pdu_xmit, 1119 .init_pdu = iscsi_sw_tcp_pdu_init, 1120 .alloc_pdu = iscsi_sw_tcp_pdu_alloc, 1121 /* recovery */ 1122 .session_recovery_timedout = iscsi_session_recovery_timedout, 1123 }; 1124 1125 static int __init iscsi_sw_tcp_init(void) 1126 { 1127 if (iscsi_max_lun < 1) { 1128 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n", 1129 iscsi_max_lun); 1130 return -EINVAL; 1131 } 1132 1133 iscsi_sw_tcp_scsi_transport = iscsi_register_transport( 1134 &iscsi_sw_tcp_transport); 1135 if (!iscsi_sw_tcp_scsi_transport) 1136 return -ENODEV; 1137 1138 return 0; 1139 } 1140 1141 static void __exit iscsi_sw_tcp_exit(void) 1142 { 1143 iscsi_unregister_transport(&iscsi_sw_tcp_transport); 1144 } 1145 1146 module_init(iscsi_sw_tcp_init); 1147 module_exit(iscsi_sw_tcp_exit); 1148