1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /******************************************************************************* 3 * This file contains main functions related to the iSCSI Target Core Driver. 4 * 5 * (c) Copyright 2007-2013 Datera, Inc. 6 * 7 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org> 8 * 9 ******************************************************************************/ 10 11 #include <crypto/hash.h> 12 #include <linux/string.h> 13 #include <linux/kthread.h> 14 #include <linux/completion.h> 15 #include <linux/module.h> 16 #include <linux/vmalloc.h> 17 #include <linux/idr.h> 18 #include <linux/delay.h> 19 #include <linux/sched/signal.h> 20 #include <asm/unaligned.h> 21 #include <linux/inet.h> 22 #include <net/ipv6.h> 23 #include <scsi/scsi_proto.h> 24 #include <scsi/iscsi_proto.h> 25 #include <scsi/scsi_tcq.h> 26 #include <target/target_core_base.h> 27 #include <target/target_core_fabric.h> 28 29 #include <target/iscsi/iscsi_target_core.h> 30 #include "iscsi_target_parameters.h" 31 #include "iscsi_target_seq_pdu_list.h" 32 #include "iscsi_target_datain_values.h" 33 #include "iscsi_target_erl0.h" 34 #include "iscsi_target_erl1.h" 35 #include "iscsi_target_erl2.h" 36 #include "iscsi_target_login.h" 37 #include "iscsi_target_tmr.h" 38 #include "iscsi_target_tpg.h" 39 #include "iscsi_target_util.h" 40 #include "iscsi_target.h" 41 #include "iscsi_target_device.h" 42 #include <target/iscsi/iscsi_target_stat.h> 43 44 #include <target/iscsi/iscsi_transport.h> 45 46 static LIST_HEAD(g_tiqn_list); 47 static LIST_HEAD(g_np_list); 48 static DEFINE_SPINLOCK(tiqn_lock); 49 static DEFINE_MUTEX(np_lock); 50 51 static struct idr tiqn_idr; 52 DEFINE_IDA(sess_ida); 53 struct mutex auth_id_lock; 54 55 struct iscsit_global *iscsit_global; 56 57 struct kmem_cache *lio_qr_cache; 58 struct kmem_cache *lio_dr_cache; 59 struct kmem_cache *lio_ooo_cache; 60 struct kmem_cache *lio_r2t_cache; 61 62 static int iscsit_handle_immediate_data(struct iscsi_cmd *, 63 struct iscsi_scsi_req *, u32); 64 65 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf) 66 { 67 struct iscsi_tiqn *tiqn = NULL; 68 69 spin_lock(&tiqn_lock); 70 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) { 71 if (!strcmp(tiqn->tiqn, buf)) { 72 73 spin_lock(&tiqn->tiqn_state_lock); 74 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) { 75 tiqn->tiqn_access_count++; 76 spin_unlock(&tiqn->tiqn_state_lock); 77 spin_unlock(&tiqn_lock); 78 return tiqn; 79 } 80 spin_unlock(&tiqn->tiqn_state_lock); 81 } 82 } 83 spin_unlock(&tiqn_lock); 84 85 return NULL; 86 } 87 88 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn) 89 { 90 spin_lock(&tiqn->tiqn_state_lock); 91 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) { 92 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN; 93 spin_unlock(&tiqn->tiqn_state_lock); 94 return 0; 95 } 96 spin_unlock(&tiqn->tiqn_state_lock); 97 98 return -1; 99 } 100 101 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn) 102 { 103 spin_lock(&tiqn->tiqn_state_lock); 104 tiqn->tiqn_access_count--; 105 spin_unlock(&tiqn->tiqn_state_lock); 106 } 107 108 /* 109 * Note that IQN formatting is expected to be done in userspace, and 110 * no explict IQN format checks are done here. 111 */ 112 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf) 113 { 114 struct iscsi_tiqn *tiqn = NULL; 115 int ret; 116 117 if (strlen(buf) >= ISCSI_IQN_LEN) { 118 pr_err("Target IQN exceeds %d bytes\n", 119 ISCSI_IQN_LEN); 120 return ERR_PTR(-EINVAL); 121 } 122 123 tiqn = kzalloc(sizeof(*tiqn), GFP_KERNEL); 124 if (!tiqn) 125 return ERR_PTR(-ENOMEM); 126 127 sprintf(tiqn->tiqn, "%s", buf); 128 INIT_LIST_HEAD(&tiqn->tiqn_list); 129 INIT_LIST_HEAD(&tiqn->tiqn_tpg_list); 130 spin_lock_init(&tiqn->tiqn_state_lock); 131 spin_lock_init(&tiqn->tiqn_tpg_lock); 132 spin_lock_init(&tiqn->sess_err_stats.lock); 133 spin_lock_init(&tiqn->login_stats.lock); 134 spin_lock_init(&tiqn->logout_stats.lock); 135 136 tiqn->tiqn_state = TIQN_STATE_ACTIVE; 137 138 idr_preload(GFP_KERNEL); 139 spin_lock(&tiqn_lock); 140 141 ret = idr_alloc(&tiqn_idr, NULL, 0, 0, GFP_NOWAIT); 142 if (ret < 0) { 143 pr_err("idr_alloc() failed for tiqn->tiqn_index\n"); 144 spin_unlock(&tiqn_lock); 145 idr_preload_end(); 146 kfree(tiqn); 147 return ERR_PTR(ret); 148 } 149 tiqn->tiqn_index = ret; 150 list_add_tail(&tiqn->tiqn_list, &g_tiqn_list); 151 152 spin_unlock(&tiqn_lock); 153 idr_preload_end(); 154 155 pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn); 156 157 return tiqn; 158 159 } 160 161 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn) 162 { 163 /* 164 * Wait for accesses to said struct iscsi_tiqn to end. 165 */ 166 spin_lock(&tiqn->tiqn_state_lock); 167 while (tiqn->tiqn_access_count != 0) { 168 spin_unlock(&tiqn->tiqn_state_lock); 169 msleep(10); 170 spin_lock(&tiqn->tiqn_state_lock); 171 } 172 spin_unlock(&tiqn->tiqn_state_lock); 173 } 174 175 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn) 176 { 177 /* 178 * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN 179 * while holding tiqn->tiqn_state_lock. This means that all subsequent 180 * attempts to access this struct iscsi_tiqn will fail from both transport 181 * fabric and control code paths. 182 */ 183 if (iscsit_set_tiqn_shutdown(tiqn) < 0) { 184 pr_err("iscsit_set_tiqn_shutdown() failed\n"); 185 return; 186 } 187 188 iscsit_wait_for_tiqn(tiqn); 189 190 spin_lock(&tiqn_lock); 191 list_del(&tiqn->tiqn_list); 192 idr_remove(&tiqn_idr, tiqn->tiqn_index); 193 spin_unlock(&tiqn_lock); 194 195 pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n", 196 tiqn->tiqn); 197 kfree(tiqn); 198 } 199 200 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg) 201 { 202 int ret; 203 /* 204 * Determine if the network portal is accepting storage traffic. 205 */ 206 spin_lock_bh(&np->np_thread_lock); 207 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { 208 spin_unlock_bh(&np->np_thread_lock); 209 return -1; 210 } 211 spin_unlock_bh(&np->np_thread_lock); 212 /* 213 * Determine if the portal group is accepting storage traffic. 214 */ 215 spin_lock_bh(&tpg->tpg_state_lock); 216 if (tpg->tpg_state != TPG_STATE_ACTIVE) { 217 spin_unlock_bh(&tpg->tpg_state_lock); 218 return -1; 219 } 220 spin_unlock_bh(&tpg->tpg_state_lock); 221 222 /* 223 * Here we serialize access across the TIQN+TPG Tuple. 224 */ 225 ret = down_interruptible(&tpg->np_login_sem); 226 if (ret != 0) 227 return -1; 228 229 spin_lock_bh(&tpg->tpg_state_lock); 230 if (tpg->tpg_state != TPG_STATE_ACTIVE) { 231 spin_unlock_bh(&tpg->tpg_state_lock); 232 up(&tpg->np_login_sem); 233 return -1; 234 } 235 spin_unlock_bh(&tpg->tpg_state_lock); 236 237 return 0; 238 } 239 240 void iscsit_login_kref_put(struct kref *kref) 241 { 242 struct iscsi_tpg_np *tpg_np = container_of(kref, 243 struct iscsi_tpg_np, tpg_np_kref); 244 245 complete(&tpg_np->tpg_np_comp); 246 } 247 248 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg, 249 struct iscsi_tpg_np *tpg_np) 250 { 251 struct iscsi_tiqn *tiqn = tpg->tpg_tiqn; 252 253 up(&tpg->np_login_sem); 254 255 if (tpg_np) 256 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put); 257 258 if (tiqn) 259 iscsit_put_tiqn_for_login(tiqn); 260 261 return 0; 262 } 263 264 bool iscsit_check_np_match( 265 struct sockaddr_storage *sockaddr, 266 struct iscsi_np *np, 267 int network_transport) 268 { 269 struct sockaddr_in *sock_in, *sock_in_e; 270 struct sockaddr_in6 *sock_in6, *sock_in6_e; 271 bool ip_match = false; 272 u16 port, port_e; 273 274 if (sockaddr->ss_family == AF_INET6) { 275 sock_in6 = (struct sockaddr_in6 *)sockaddr; 276 sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr; 277 278 if (!memcmp(&sock_in6->sin6_addr.in6_u, 279 &sock_in6_e->sin6_addr.in6_u, 280 sizeof(struct in6_addr))) 281 ip_match = true; 282 283 port = ntohs(sock_in6->sin6_port); 284 port_e = ntohs(sock_in6_e->sin6_port); 285 } else { 286 sock_in = (struct sockaddr_in *)sockaddr; 287 sock_in_e = (struct sockaddr_in *)&np->np_sockaddr; 288 289 if (sock_in->sin_addr.s_addr == sock_in_e->sin_addr.s_addr) 290 ip_match = true; 291 292 port = ntohs(sock_in->sin_port); 293 port_e = ntohs(sock_in_e->sin_port); 294 } 295 296 if (ip_match && (port_e == port) && 297 (np->np_network_transport == network_transport)) 298 return true; 299 300 return false; 301 } 302 303 static struct iscsi_np *iscsit_get_np( 304 struct sockaddr_storage *sockaddr, 305 int network_transport) 306 { 307 struct iscsi_np *np; 308 bool match; 309 310 lockdep_assert_held(&np_lock); 311 312 list_for_each_entry(np, &g_np_list, np_list) { 313 spin_lock_bh(&np->np_thread_lock); 314 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { 315 spin_unlock_bh(&np->np_thread_lock); 316 continue; 317 } 318 319 match = iscsit_check_np_match(sockaddr, np, network_transport); 320 if (match) { 321 /* 322 * Increment the np_exports reference count now to 323 * prevent iscsit_del_np() below from being called 324 * while iscsi_tpg_add_network_portal() is called. 325 */ 326 np->np_exports++; 327 spin_unlock_bh(&np->np_thread_lock); 328 return np; 329 } 330 spin_unlock_bh(&np->np_thread_lock); 331 } 332 333 return NULL; 334 } 335 336 struct iscsi_np *iscsit_add_np( 337 struct sockaddr_storage *sockaddr, 338 int network_transport) 339 { 340 struct iscsi_np *np; 341 int ret; 342 343 mutex_lock(&np_lock); 344 345 /* 346 * Locate the existing struct iscsi_np if already active.. 347 */ 348 np = iscsit_get_np(sockaddr, network_transport); 349 if (np) { 350 mutex_unlock(&np_lock); 351 return np; 352 } 353 354 np = kzalloc(sizeof(*np), GFP_KERNEL); 355 if (!np) { 356 mutex_unlock(&np_lock); 357 return ERR_PTR(-ENOMEM); 358 } 359 360 np->np_flags |= NPF_IP_NETWORK; 361 np->np_network_transport = network_transport; 362 spin_lock_init(&np->np_thread_lock); 363 init_completion(&np->np_restart_comp); 364 INIT_LIST_HEAD(&np->np_list); 365 366 timer_setup(&np->np_login_timer, iscsi_handle_login_thread_timeout, 0); 367 368 ret = iscsi_target_setup_login_socket(np, sockaddr); 369 if (ret != 0) { 370 kfree(np); 371 mutex_unlock(&np_lock); 372 return ERR_PTR(ret); 373 } 374 375 np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np"); 376 if (IS_ERR(np->np_thread)) { 377 pr_err("Unable to create kthread: iscsi_np\n"); 378 ret = PTR_ERR(np->np_thread); 379 kfree(np); 380 mutex_unlock(&np_lock); 381 return ERR_PTR(ret); 382 } 383 /* 384 * Increment the np_exports reference count now to prevent 385 * iscsit_del_np() below from being run while a new call to 386 * iscsi_tpg_add_network_portal() for a matching iscsi_np is 387 * active. We don't need to hold np->np_thread_lock at this 388 * point because iscsi_np has not been added to g_np_list yet. 389 */ 390 np->np_exports = 1; 391 np->np_thread_state = ISCSI_NP_THREAD_ACTIVE; 392 393 list_add_tail(&np->np_list, &g_np_list); 394 mutex_unlock(&np_lock); 395 396 pr_debug("CORE[0] - Added Network Portal: %pISpc on %s\n", 397 &np->np_sockaddr, np->np_transport->name); 398 399 return np; 400 } 401 402 int iscsit_reset_np_thread( 403 struct iscsi_np *np, 404 struct iscsi_tpg_np *tpg_np, 405 struct iscsi_portal_group *tpg, 406 bool shutdown) 407 { 408 spin_lock_bh(&np->np_thread_lock); 409 if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) { 410 spin_unlock_bh(&np->np_thread_lock); 411 return 0; 412 } 413 np->np_thread_state = ISCSI_NP_THREAD_RESET; 414 atomic_inc(&np->np_reset_count); 415 416 if (np->np_thread) { 417 spin_unlock_bh(&np->np_thread_lock); 418 send_sig(SIGINT, np->np_thread, 1); 419 wait_for_completion(&np->np_restart_comp); 420 spin_lock_bh(&np->np_thread_lock); 421 } 422 spin_unlock_bh(&np->np_thread_lock); 423 424 if (tpg_np && shutdown) { 425 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put); 426 427 wait_for_completion(&tpg_np->tpg_np_comp); 428 } 429 430 return 0; 431 } 432 433 static void iscsit_free_np(struct iscsi_np *np) 434 { 435 if (np->np_socket) 436 sock_release(np->np_socket); 437 } 438 439 int iscsit_del_np(struct iscsi_np *np) 440 { 441 spin_lock_bh(&np->np_thread_lock); 442 np->np_exports--; 443 if (np->np_exports) { 444 np->enabled = true; 445 spin_unlock_bh(&np->np_thread_lock); 446 return 0; 447 } 448 np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN; 449 spin_unlock_bh(&np->np_thread_lock); 450 451 if (np->np_thread) { 452 /* 453 * We need to send the signal to wakeup Linux/Net 454 * which may be sleeping in sock_accept().. 455 */ 456 send_sig(SIGINT, np->np_thread, 1); 457 kthread_stop(np->np_thread); 458 np->np_thread = NULL; 459 } 460 461 np->np_transport->iscsit_free_np(np); 462 463 mutex_lock(&np_lock); 464 list_del(&np->np_list); 465 mutex_unlock(&np_lock); 466 467 pr_debug("CORE[0] - Removed Network Portal: %pISpc on %s\n", 468 &np->np_sockaddr, np->np_transport->name); 469 470 iscsit_put_transport(np->np_transport); 471 kfree(np); 472 return 0; 473 } 474 475 static void iscsit_get_rx_pdu(struct iscsi_conn *); 476 477 int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 478 { 479 return iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state); 480 } 481 EXPORT_SYMBOL(iscsit_queue_rsp); 482 483 void iscsit_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 484 { 485 spin_lock_bh(&conn->cmd_lock); 486 if (!list_empty(&cmd->i_conn_node)) 487 list_del_init(&cmd->i_conn_node); 488 spin_unlock_bh(&conn->cmd_lock); 489 490 __iscsit_free_cmd(cmd, true); 491 } 492 EXPORT_SYMBOL(iscsit_aborted_task); 493 494 static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *, 495 u32, u32, const void *, void *); 496 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *); 497 498 static int 499 iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 500 const void *data_buf, u32 data_buf_len) 501 { 502 struct iscsi_hdr *hdr = (struct iscsi_hdr *)cmd->pdu; 503 struct kvec *iov; 504 u32 niov = 0, tx_size = ISCSI_HDR_LEN; 505 int ret; 506 507 iov = &cmd->iov_misc[0]; 508 iov[niov].iov_base = cmd->pdu; 509 iov[niov++].iov_len = ISCSI_HDR_LEN; 510 511 if (conn->conn_ops->HeaderDigest) { 512 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 513 514 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, 515 ISCSI_HDR_LEN, 0, NULL, 516 header_digest); 517 518 iov[0].iov_len += ISCSI_CRC_LEN; 519 tx_size += ISCSI_CRC_LEN; 520 pr_debug("Attaching CRC32C HeaderDigest" 521 " to opcode 0x%x 0x%08x\n", 522 hdr->opcode, *header_digest); 523 } 524 525 if (data_buf_len) { 526 u32 padding = ((-data_buf_len) & 3); 527 528 iov[niov].iov_base = (void *)data_buf; 529 iov[niov++].iov_len = data_buf_len; 530 tx_size += data_buf_len; 531 532 if (padding != 0) { 533 iov[niov].iov_base = &cmd->pad_bytes; 534 iov[niov++].iov_len = padding; 535 tx_size += padding; 536 pr_debug("Attaching %u additional" 537 " padding bytes.\n", padding); 538 } 539 540 if (conn->conn_ops->DataDigest) { 541 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, 542 data_buf, data_buf_len, 543 padding, &cmd->pad_bytes, 544 &cmd->data_crc); 545 546 iov[niov].iov_base = &cmd->data_crc; 547 iov[niov++].iov_len = ISCSI_CRC_LEN; 548 tx_size += ISCSI_CRC_LEN; 549 pr_debug("Attached DataDigest for %u" 550 " bytes opcode 0x%x, CRC 0x%08x\n", 551 data_buf_len, hdr->opcode, cmd->data_crc); 552 } 553 } 554 555 cmd->iov_misc_count = niov; 556 cmd->tx_size = tx_size; 557 558 ret = iscsit_send_tx_data(cmd, conn, 1); 559 if (ret < 0) { 560 iscsit_tx_thread_wait_for_tcp(conn); 561 return ret; 562 } 563 564 return 0; 565 } 566 567 static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec, 568 u32 data_offset, u32 data_length); 569 static void iscsit_unmap_iovec(struct iscsi_cmd *); 570 static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsi_cmd *, 571 u32, u32, u32, u8 *); 572 static int 573 iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 574 const struct iscsi_datain *datain) 575 { 576 struct kvec *iov; 577 u32 iov_count = 0, tx_size = 0; 578 int ret, iov_ret; 579 580 iov = &cmd->iov_data[0]; 581 iov[iov_count].iov_base = cmd->pdu; 582 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 583 tx_size += ISCSI_HDR_LEN; 584 585 if (conn->conn_ops->HeaderDigest) { 586 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 587 588 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu, 589 ISCSI_HDR_LEN, 0, NULL, 590 header_digest); 591 592 iov[0].iov_len += ISCSI_CRC_LEN; 593 tx_size += ISCSI_CRC_LEN; 594 595 pr_debug("Attaching CRC32 HeaderDigest for DataIN PDU 0x%08x\n", 596 *header_digest); 597 } 598 599 iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[iov_count], 600 cmd->orig_iov_data_count - (iov_count + 2), 601 datain->offset, datain->length); 602 if (iov_ret < 0) 603 return -1; 604 605 iov_count += iov_ret; 606 tx_size += datain->length; 607 608 cmd->padding = ((-datain->length) & 3); 609 if (cmd->padding) { 610 iov[iov_count].iov_base = cmd->pad_bytes; 611 iov[iov_count++].iov_len = cmd->padding; 612 tx_size += cmd->padding; 613 614 pr_debug("Attaching %u padding bytes\n", cmd->padding); 615 } 616 617 if (conn->conn_ops->DataDigest) { 618 cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash, 619 cmd, datain->offset, 620 datain->length, 621 cmd->padding, 622 cmd->pad_bytes); 623 624 iov[iov_count].iov_base = &cmd->data_crc; 625 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 626 tx_size += ISCSI_CRC_LEN; 627 628 pr_debug("Attached CRC32C DataDigest %d bytes, crc 0x%08x\n", 629 datain->length + cmd->padding, cmd->data_crc); 630 } 631 632 cmd->iov_data_count = iov_count; 633 cmd->tx_size = tx_size; 634 635 ret = iscsit_fe_sendpage_sg(cmd, conn); 636 637 iscsit_unmap_iovec(cmd); 638 639 if (ret < 0) { 640 iscsit_tx_thread_wait_for_tcp(conn); 641 return ret; 642 } 643 644 return 0; 645 } 646 647 static int iscsit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 648 struct iscsi_datain_req *dr, const void *buf, 649 u32 buf_len) 650 { 651 if (dr) 652 return iscsit_xmit_datain_pdu(conn, cmd, buf); 653 else 654 return iscsit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 655 } 656 657 static enum target_prot_op iscsit_get_sup_prot_ops(struct iscsi_conn *conn) 658 { 659 return TARGET_PROT_NORMAL; 660 } 661 662 static struct iscsit_transport iscsi_target_transport = { 663 .name = "iSCSI/TCP", 664 .transport_type = ISCSI_TCP, 665 .rdma_shutdown = false, 666 .owner = NULL, 667 .iscsit_setup_np = iscsit_setup_np, 668 .iscsit_accept_np = iscsit_accept_np, 669 .iscsit_free_np = iscsit_free_np, 670 .iscsit_get_login_rx = iscsit_get_login_rx, 671 .iscsit_put_login_tx = iscsit_put_login_tx, 672 .iscsit_get_dataout = iscsit_build_r2ts_for_cmd, 673 .iscsit_immediate_queue = iscsit_immediate_queue, 674 .iscsit_response_queue = iscsit_response_queue, 675 .iscsit_queue_data_in = iscsit_queue_rsp, 676 .iscsit_queue_status = iscsit_queue_rsp, 677 .iscsit_aborted_task = iscsit_aborted_task, 678 .iscsit_xmit_pdu = iscsit_xmit_pdu, 679 .iscsit_get_rx_pdu = iscsit_get_rx_pdu, 680 .iscsit_get_sup_prot_ops = iscsit_get_sup_prot_ops, 681 }; 682 683 static int __init iscsi_target_init_module(void) 684 { 685 int ret = 0, size; 686 687 pr_debug("iSCSI-Target "ISCSIT_VERSION"\n"); 688 iscsit_global = kzalloc(sizeof(*iscsit_global), GFP_KERNEL); 689 if (!iscsit_global) 690 return -1; 691 692 spin_lock_init(&iscsit_global->ts_bitmap_lock); 693 mutex_init(&auth_id_lock); 694 idr_init(&tiqn_idr); 695 696 ret = target_register_template(&iscsi_ops); 697 if (ret) 698 goto out; 699 700 size = BITS_TO_LONGS(ISCSIT_BITMAP_BITS) * sizeof(long); 701 iscsit_global->ts_bitmap = vzalloc(size); 702 if (!iscsit_global->ts_bitmap) 703 goto configfs_out; 704 705 lio_qr_cache = kmem_cache_create("lio_qr_cache", 706 sizeof(struct iscsi_queue_req), 707 __alignof__(struct iscsi_queue_req), 0, NULL); 708 if (!lio_qr_cache) { 709 pr_err("Unable to kmem_cache_create() for" 710 " lio_qr_cache\n"); 711 goto bitmap_out; 712 } 713 714 lio_dr_cache = kmem_cache_create("lio_dr_cache", 715 sizeof(struct iscsi_datain_req), 716 __alignof__(struct iscsi_datain_req), 0, NULL); 717 if (!lio_dr_cache) { 718 pr_err("Unable to kmem_cache_create() for" 719 " lio_dr_cache\n"); 720 goto qr_out; 721 } 722 723 lio_ooo_cache = kmem_cache_create("lio_ooo_cache", 724 sizeof(struct iscsi_ooo_cmdsn), 725 __alignof__(struct iscsi_ooo_cmdsn), 0, NULL); 726 if (!lio_ooo_cache) { 727 pr_err("Unable to kmem_cache_create() for" 728 " lio_ooo_cache\n"); 729 goto dr_out; 730 } 731 732 lio_r2t_cache = kmem_cache_create("lio_r2t_cache", 733 sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t), 734 0, NULL); 735 if (!lio_r2t_cache) { 736 pr_err("Unable to kmem_cache_create() for" 737 " lio_r2t_cache\n"); 738 goto ooo_out; 739 } 740 741 iscsit_register_transport(&iscsi_target_transport); 742 743 if (iscsit_load_discovery_tpg() < 0) 744 goto r2t_out; 745 746 return ret; 747 r2t_out: 748 iscsit_unregister_transport(&iscsi_target_transport); 749 kmem_cache_destroy(lio_r2t_cache); 750 ooo_out: 751 kmem_cache_destroy(lio_ooo_cache); 752 dr_out: 753 kmem_cache_destroy(lio_dr_cache); 754 qr_out: 755 kmem_cache_destroy(lio_qr_cache); 756 bitmap_out: 757 vfree(iscsit_global->ts_bitmap); 758 configfs_out: 759 /* XXX: this probably wants it to be it's own unwind step.. */ 760 if (iscsit_global->discovery_tpg) 761 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1); 762 target_unregister_template(&iscsi_ops); 763 out: 764 kfree(iscsit_global); 765 return -ENOMEM; 766 } 767 768 static void __exit iscsi_target_cleanup_module(void) 769 { 770 iscsit_release_discovery_tpg(); 771 iscsit_unregister_transport(&iscsi_target_transport); 772 kmem_cache_destroy(lio_qr_cache); 773 kmem_cache_destroy(lio_dr_cache); 774 kmem_cache_destroy(lio_ooo_cache); 775 kmem_cache_destroy(lio_r2t_cache); 776 777 /* 778 * Shutdown discovery sessions and disable discovery TPG 779 */ 780 if (iscsit_global->discovery_tpg) 781 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1); 782 783 target_unregister_template(&iscsi_ops); 784 785 vfree(iscsit_global->ts_bitmap); 786 kfree(iscsit_global); 787 } 788 789 int iscsit_add_reject( 790 struct iscsi_conn *conn, 791 u8 reason, 792 unsigned char *buf) 793 { 794 struct iscsi_cmd *cmd; 795 796 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 797 if (!cmd) 798 return -1; 799 800 cmd->iscsi_opcode = ISCSI_OP_REJECT; 801 cmd->reject_reason = reason; 802 803 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL); 804 if (!cmd->buf_ptr) { 805 pr_err("Unable to allocate memory for cmd->buf_ptr\n"); 806 iscsit_free_cmd(cmd, false); 807 return -1; 808 } 809 810 spin_lock_bh(&conn->cmd_lock); 811 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 812 spin_unlock_bh(&conn->cmd_lock); 813 814 cmd->i_state = ISTATE_SEND_REJECT; 815 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 816 817 return -1; 818 } 819 EXPORT_SYMBOL(iscsit_add_reject); 820 821 static int iscsit_add_reject_from_cmd( 822 struct iscsi_cmd *cmd, 823 u8 reason, 824 bool add_to_conn, 825 unsigned char *buf) 826 { 827 struct iscsi_conn *conn; 828 const bool do_put = cmd->se_cmd.se_tfo != NULL; 829 830 if (!cmd->conn) { 831 pr_err("cmd->conn is NULL for ITT: 0x%08x\n", 832 cmd->init_task_tag); 833 return -1; 834 } 835 conn = cmd->conn; 836 837 cmd->iscsi_opcode = ISCSI_OP_REJECT; 838 cmd->reject_reason = reason; 839 840 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL); 841 if (!cmd->buf_ptr) { 842 pr_err("Unable to allocate memory for cmd->buf_ptr\n"); 843 iscsit_free_cmd(cmd, false); 844 return -1; 845 } 846 847 if (add_to_conn) { 848 spin_lock_bh(&conn->cmd_lock); 849 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 850 spin_unlock_bh(&conn->cmd_lock); 851 } 852 853 cmd->i_state = ISTATE_SEND_REJECT; 854 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 855 /* 856 * Perform the kref_put now if se_cmd has already been setup by 857 * scsit_setup_scsi_cmd() 858 */ 859 if (do_put) { 860 pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n"); 861 target_put_sess_cmd(&cmd->se_cmd); 862 } 863 return -1; 864 } 865 866 static int iscsit_add_reject_cmd(struct iscsi_cmd *cmd, u8 reason, 867 unsigned char *buf) 868 { 869 return iscsit_add_reject_from_cmd(cmd, reason, true, buf); 870 } 871 872 int iscsit_reject_cmd(struct iscsi_cmd *cmd, u8 reason, unsigned char *buf) 873 { 874 return iscsit_add_reject_from_cmd(cmd, reason, false, buf); 875 } 876 EXPORT_SYMBOL(iscsit_reject_cmd); 877 878 /* 879 * Map some portion of the allocated scatterlist to an iovec, suitable for 880 * kernel sockets to copy data in/out. 881 */ 882 static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec, 883 u32 data_offset, u32 data_length) 884 { 885 u32 i = 0, orig_data_length = data_length; 886 struct scatterlist *sg; 887 unsigned int page_off; 888 889 /* 890 * We know each entry in t_data_sg contains a page. 891 */ 892 u32 ent = data_offset / PAGE_SIZE; 893 894 if (!data_length) 895 return 0; 896 897 if (ent >= cmd->se_cmd.t_data_nents) { 898 pr_err("Initial page entry out-of-bounds\n"); 899 goto overflow; 900 } 901 902 sg = &cmd->se_cmd.t_data_sg[ent]; 903 page_off = (data_offset % PAGE_SIZE); 904 905 cmd->first_data_sg = sg; 906 cmd->first_data_sg_off = page_off; 907 908 while (data_length) { 909 u32 cur_len; 910 911 if (WARN_ON_ONCE(!sg || i >= nvec)) 912 goto overflow; 913 914 cur_len = min_t(u32, data_length, sg->length - page_off); 915 916 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off; 917 iov[i].iov_len = cur_len; 918 919 data_length -= cur_len; 920 page_off = 0; 921 sg = sg_next(sg); 922 i++; 923 } 924 925 cmd->kmapped_nents = i; 926 927 return i; 928 929 overflow: 930 pr_err("offset %d + length %d overflow; %d/%d; sg-list:\n", 931 data_offset, orig_data_length, i, nvec); 932 for_each_sg(cmd->se_cmd.t_data_sg, sg, 933 cmd->se_cmd.t_data_nents, i) { 934 pr_err("[%d] off %d len %d\n", 935 i, sg->offset, sg->length); 936 } 937 return -1; 938 } 939 940 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd) 941 { 942 u32 i; 943 struct scatterlist *sg; 944 945 sg = cmd->first_data_sg; 946 947 for (i = 0; i < cmd->kmapped_nents; i++) 948 kunmap(sg_page(&sg[i])); 949 } 950 951 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn) 952 { 953 LIST_HEAD(ack_list); 954 struct iscsi_cmd *cmd, *cmd_p; 955 956 conn->exp_statsn = exp_statsn; 957 958 if (conn->sess->sess_ops->RDMAExtensions) 959 return; 960 961 spin_lock_bh(&conn->cmd_lock); 962 list_for_each_entry_safe(cmd, cmd_p, &conn->conn_cmd_list, i_conn_node) { 963 spin_lock(&cmd->istate_lock); 964 if ((cmd->i_state == ISTATE_SENT_STATUS) && 965 iscsi_sna_lt(cmd->stat_sn, exp_statsn)) { 966 cmd->i_state = ISTATE_REMOVE; 967 spin_unlock(&cmd->istate_lock); 968 list_move_tail(&cmd->i_conn_node, &ack_list); 969 continue; 970 } 971 spin_unlock(&cmd->istate_lock); 972 } 973 spin_unlock_bh(&conn->cmd_lock); 974 975 list_for_each_entry_safe(cmd, cmd_p, &ack_list, i_conn_node) { 976 list_del_init(&cmd->i_conn_node); 977 iscsit_free_cmd(cmd, false); 978 } 979 } 980 981 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd) 982 { 983 u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE)); 984 985 iov_count += ISCSI_IOV_DATA_BUFFER; 986 cmd->iov_data = kcalloc(iov_count, sizeof(*cmd->iov_data), GFP_KERNEL); 987 if (!cmd->iov_data) 988 return -ENOMEM; 989 990 cmd->orig_iov_data_count = iov_count; 991 return 0; 992 } 993 994 int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 995 unsigned char *buf) 996 { 997 int data_direction, payload_length; 998 struct iscsi_scsi_req *hdr; 999 int iscsi_task_attr; 1000 int sam_task_attr; 1001 1002 atomic_long_inc(&conn->sess->cmd_pdus); 1003 1004 hdr = (struct iscsi_scsi_req *) buf; 1005 payload_length = ntoh24(hdr->dlength); 1006 1007 /* FIXME; Add checks for AdditionalHeaderSegment */ 1008 1009 if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) && 1010 !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) { 1011 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL" 1012 " not set. Bad iSCSI Initiator.\n"); 1013 return iscsit_add_reject_cmd(cmd, 1014 ISCSI_REASON_BOOKMARK_INVALID, buf); 1015 } 1016 1017 if (((hdr->flags & ISCSI_FLAG_CMD_READ) || 1018 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) { 1019 /* 1020 * From RFC-3720 Section 10.3.1: 1021 * 1022 * "Either or both of R and W MAY be 1 when either the 1023 * Expected Data Transfer Length and/or Bidirectional Read 1024 * Expected Data Transfer Length are 0" 1025 * 1026 * For this case, go ahead and clear the unnecssary bits 1027 * to avoid any confusion with ->data_direction. 1028 */ 1029 hdr->flags &= ~ISCSI_FLAG_CMD_READ; 1030 hdr->flags &= ~ISCSI_FLAG_CMD_WRITE; 1031 1032 pr_warn("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE" 1033 " set when Expected Data Transfer Length is 0 for" 1034 " CDB: 0x%02x, Fixing up flags\n", hdr->cdb[0]); 1035 } 1036 1037 if (!(hdr->flags & ISCSI_FLAG_CMD_READ) && 1038 !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) { 1039 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE" 1040 " MUST be set if Expected Data Transfer Length is not 0." 1041 " Bad iSCSI Initiator\n"); 1042 return iscsit_add_reject_cmd(cmd, 1043 ISCSI_REASON_BOOKMARK_INVALID, buf); 1044 } 1045 1046 if ((hdr->flags & ISCSI_FLAG_CMD_READ) && 1047 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) { 1048 pr_err("Bidirectional operations not supported!\n"); 1049 return iscsit_add_reject_cmd(cmd, 1050 ISCSI_REASON_BOOKMARK_INVALID, buf); 1051 } 1052 1053 if (hdr->opcode & ISCSI_OP_IMMEDIATE) { 1054 pr_err("Illegally set Immediate Bit in iSCSI Initiator" 1055 " Scsi Command PDU.\n"); 1056 return iscsit_add_reject_cmd(cmd, 1057 ISCSI_REASON_BOOKMARK_INVALID, buf); 1058 } 1059 1060 if (payload_length && !conn->sess->sess_ops->ImmediateData) { 1061 pr_err("ImmediateData=No but DataSegmentLength=%u," 1062 " protocol error.\n", payload_length); 1063 return iscsit_add_reject_cmd(cmd, 1064 ISCSI_REASON_PROTOCOL_ERROR, buf); 1065 } 1066 1067 if ((be32_to_cpu(hdr->data_length) == payload_length) && 1068 (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) { 1069 pr_err("Expected Data Transfer Length and Length of" 1070 " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL" 1071 " bit is not set protocol error\n"); 1072 return iscsit_add_reject_cmd(cmd, 1073 ISCSI_REASON_PROTOCOL_ERROR, buf); 1074 } 1075 1076 if (payload_length > be32_to_cpu(hdr->data_length)) { 1077 pr_err("DataSegmentLength: %u is greater than" 1078 " EDTL: %u, protocol error.\n", payload_length, 1079 hdr->data_length); 1080 return iscsit_add_reject_cmd(cmd, 1081 ISCSI_REASON_PROTOCOL_ERROR, buf); 1082 } 1083 1084 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 1085 pr_err("DataSegmentLength: %u is greater than" 1086 " MaxXmitDataSegmentLength: %u, protocol error.\n", 1087 payload_length, conn->conn_ops->MaxXmitDataSegmentLength); 1088 return iscsit_add_reject_cmd(cmd, 1089 ISCSI_REASON_PROTOCOL_ERROR, buf); 1090 } 1091 1092 if (payload_length > conn->sess->sess_ops->FirstBurstLength) { 1093 pr_err("DataSegmentLength: %u is greater than" 1094 " FirstBurstLength: %u, protocol error.\n", 1095 payload_length, conn->sess->sess_ops->FirstBurstLength); 1096 return iscsit_add_reject_cmd(cmd, 1097 ISCSI_REASON_BOOKMARK_INVALID, buf); 1098 } 1099 1100 data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE : 1101 (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE : 1102 DMA_NONE; 1103 1104 cmd->data_direction = data_direction; 1105 iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK; 1106 /* 1107 * Figure out the SAM Task Attribute for the incoming SCSI CDB 1108 */ 1109 if ((iscsi_task_attr == ISCSI_ATTR_UNTAGGED) || 1110 (iscsi_task_attr == ISCSI_ATTR_SIMPLE)) 1111 sam_task_attr = TCM_SIMPLE_TAG; 1112 else if (iscsi_task_attr == ISCSI_ATTR_ORDERED) 1113 sam_task_attr = TCM_ORDERED_TAG; 1114 else if (iscsi_task_attr == ISCSI_ATTR_HEAD_OF_QUEUE) 1115 sam_task_attr = TCM_HEAD_TAG; 1116 else if (iscsi_task_attr == ISCSI_ATTR_ACA) 1117 sam_task_attr = TCM_ACA_TAG; 1118 else { 1119 pr_debug("Unknown iSCSI Task Attribute: 0x%02x, using" 1120 " TCM_SIMPLE_TAG\n", iscsi_task_attr); 1121 sam_task_attr = TCM_SIMPLE_TAG; 1122 } 1123 1124 cmd->iscsi_opcode = ISCSI_OP_SCSI_CMD; 1125 cmd->i_state = ISTATE_NEW_CMD; 1126 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 1127 cmd->immediate_data = (payload_length) ? 1 : 0; 1128 cmd->unsolicited_data = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) && 1129 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0); 1130 if (cmd->unsolicited_data) 1131 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA; 1132 1133 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 1134 if (hdr->flags & ISCSI_FLAG_CMD_READ) 1135 cmd->targ_xfer_tag = session_get_next_ttt(conn->sess); 1136 else 1137 cmd->targ_xfer_tag = 0xFFFFFFFF; 1138 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 1139 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 1140 cmd->first_burst_len = payload_length; 1141 1142 if (!conn->sess->sess_ops->RDMAExtensions && 1143 cmd->data_direction == DMA_FROM_DEVICE) { 1144 struct iscsi_datain_req *dr; 1145 1146 dr = iscsit_allocate_datain_req(); 1147 if (!dr) 1148 return iscsit_add_reject_cmd(cmd, 1149 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1150 1151 iscsit_attach_datain_req(cmd, dr); 1152 } 1153 1154 /* 1155 * Initialize struct se_cmd descriptor from target_core_mod infrastructure 1156 */ 1157 __target_init_cmd(&cmd->se_cmd, &iscsi_ops, 1158 conn->sess->se_sess, be32_to_cpu(hdr->data_length), 1159 cmd->data_direction, sam_task_attr, 1160 cmd->sense_buffer + 2, scsilun_to_int(&hdr->lun)); 1161 1162 pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x," 1163 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt, 1164 hdr->cmdsn, be32_to_cpu(hdr->data_length), payload_length, 1165 conn->cid); 1166 1167 target_get_sess_cmd(&cmd->se_cmd, true); 1168 1169 cmd->se_cmd.tag = (__force u32)cmd->init_task_tag; 1170 cmd->sense_reason = target_cmd_init_cdb(&cmd->se_cmd, hdr->cdb, 1171 GFP_KERNEL); 1172 1173 if (cmd->sense_reason) { 1174 if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) { 1175 return iscsit_add_reject_cmd(cmd, 1176 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1177 } 1178 1179 goto attach_cmd; 1180 } 1181 1182 cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd); 1183 if (cmd->sense_reason) 1184 goto attach_cmd; 1185 1186 cmd->sense_reason = target_cmd_parse_cdb(&cmd->se_cmd); 1187 if (cmd->sense_reason) 1188 goto attach_cmd; 1189 1190 if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) { 1191 return iscsit_add_reject_cmd(cmd, 1192 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1193 } 1194 1195 attach_cmd: 1196 spin_lock_bh(&conn->cmd_lock); 1197 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 1198 spin_unlock_bh(&conn->cmd_lock); 1199 /* 1200 * Check if we need to delay processing because of ALUA 1201 * Active/NonOptimized primary access state.. 1202 */ 1203 core_alua_check_nonop_delay(&cmd->se_cmd); 1204 1205 return 0; 1206 } 1207 EXPORT_SYMBOL(iscsit_setup_scsi_cmd); 1208 1209 void iscsit_set_unsolicited_dataout(struct iscsi_cmd *cmd) 1210 { 1211 iscsit_set_dataout_sequence_values(cmd); 1212 1213 spin_lock_bh(&cmd->dataout_timeout_lock); 1214 iscsit_start_dataout_timer(cmd, cmd->conn); 1215 spin_unlock_bh(&cmd->dataout_timeout_lock); 1216 } 1217 EXPORT_SYMBOL(iscsit_set_unsolicited_dataout); 1218 1219 int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1220 struct iscsi_scsi_req *hdr) 1221 { 1222 int cmdsn_ret = 0; 1223 /* 1224 * Check the CmdSN against ExpCmdSN/MaxCmdSN here if 1225 * the Immediate Bit is not set, and no Immediate 1226 * Data is attached. 1227 * 1228 * A PDU/CmdSN carrying Immediate Data can only 1229 * be processed after the DataCRC has passed. 1230 * If the DataCRC fails, the CmdSN MUST NOT 1231 * be acknowledged. (See below) 1232 */ 1233 if (!cmd->immediate_data) { 1234 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 1235 (unsigned char *)hdr, hdr->cmdsn); 1236 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1237 return -1; 1238 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 1239 target_put_sess_cmd(&cmd->se_cmd); 1240 return 0; 1241 } 1242 } 1243 1244 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 1245 1246 /* 1247 * If no Immediate Data is attached, it's OK to return now. 1248 */ 1249 if (!cmd->immediate_data) { 1250 if (!cmd->sense_reason && cmd->unsolicited_data) 1251 iscsit_set_unsolicited_dataout(cmd); 1252 if (!cmd->sense_reason) 1253 return 0; 1254 1255 target_put_sess_cmd(&cmd->se_cmd); 1256 return 0; 1257 } 1258 1259 /* 1260 * Early CHECK_CONDITIONs with ImmediateData never make it to command 1261 * execution. These exceptions are processed in CmdSN order using 1262 * iscsit_check_received_cmdsn() in iscsit_get_immediate_data() below. 1263 */ 1264 if (cmd->sense_reason) 1265 return 1; 1266 /* 1267 * Call directly into transport_generic_new_cmd() to perform 1268 * the backend memory allocation. 1269 */ 1270 cmd->sense_reason = transport_generic_new_cmd(&cmd->se_cmd); 1271 if (cmd->sense_reason) 1272 return 1; 1273 1274 return 0; 1275 } 1276 EXPORT_SYMBOL(iscsit_process_scsi_cmd); 1277 1278 static int 1279 iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 1280 bool dump_payload) 1281 { 1282 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 1283 int rc; 1284 1285 /* 1286 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. 1287 */ 1288 if (dump_payload) { 1289 u32 length = min(cmd->se_cmd.data_length - cmd->write_data_done, 1290 cmd->first_burst_len); 1291 1292 pr_debug("Dumping min(%d - %d, %d) = %d bytes of immediate data\n", 1293 cmd->se_cmd.data_length, cmd->write_data_done, 1294 cmd->first_burst_len, length); 1295 rc = iscsit_dump_data_payload(cmd->conn, length, 1); 1296 pr_debug("Finished dumping immediate data\n"); 1297 if (rc < 0) 1298 immed_ret = IMMEDIATE_DATA_CANNOT_RECOVER; 1299 } else { 1300 immed_ret = iscsit_handle_immediate_data(cmd, hdr, 1301 cmd->first_burst_len); 1302 } 1303 1304 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 1305 /* 1306 * A PDU/CmdSN carrying Immediate Data passed 1307 * DataCRC, check against ExpCmdSN/MaxCmdSN if 1308 * Immediate Bit is not set. 1309 */ 1310 cmdsn_ret = iscsit_sequence_cmd(cmd->conn, cmd, 1311 (unsigned char *)hdr, hdr->cmdsn); 1312 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1313 return -1; 1314 1315 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 1316 target_put_sess_cmd(&cmd->se_cmd); 1317 1318 return 0; 1319 } else if (cmd->unsolicited_data) 1320 iscsit_set_unsolicited_dataout(cmd); 1321 1322 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 1323 /* 1324 * Immediate Data failed DataCRC and ERL>=1, 1325 * silently drop this PDU and let the initiator 1326 * plug the CmdSN gap. 1327 * 1328 * FIXME: Send Unsolicited NOPIN with reserved 1329 * TTT here to help the initiator figure out 1330 * the missing CmdSN, although they should be 1331 * intelligent enough to determine the missing 1332 * CmdSN and issue a retry to plug the sequence. 1333 */ 1334 cmd->i_state = ISTATE_REMOVE; 1335 iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, cmd->i_state); 1336 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 1337 return -1; 1338 1339 return 0; 1340 } 1341 1342 static int 1343 iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1344 unsigned char *buf) 1345 { 1346 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1347 int rc, immed_data; 1348 bool dump_payload = false; 1349 1350 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1351 if (rc < 0) 1352 return 0; 1353 /* 1354 * Allocation iovecs needed for struct socket operations for 1355 * traditional iSCSI block I/O. 1356 */ 1357 if (iscsit_allocate_iovecs(cmd) < 0) { 1358 return iscsit_reject_cmd(cmd, 1359 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 1360 } 1361 immed_data = cmd->immediate_data; 1362 1363 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1364 if (rc < 0) 1365 return rc; 1366 else if (rc > 0) 1367 dump_payload = true; 1368 1369 if (!immed_data) 1370 return 0; 1371 1372 return iscsit_get_immediate_data(cmd, hdr, dump_payload); 1373 } 1374 1375 static u32 iscsit_do_crypto_hash_sg( 1376 struct ahash_request *hash, 1377 struct iscsi_cmd *cmd, 1378 u32 data_offset, 1379 u32 data_length, 1380 u32 padding, 1381 u8 *pad_bytes) 1382 { 1383 u32 data_crc; 1384 struct scatterlist *sg; 1385 unsigned int page_off; 1386 1387 crypto_ahash_init(hash); 1388 1389 sg = cmd->first_data_sg; 1390 page_off = cmd->first_data_sg_off; 1391 1392 if (data_length && page_off) { 1393 struct scatterlist first_sg; 1394 u32 len = min_t(u32, data_length, sg->length - page_off); 1395 1396 sg_init_table(&first_sg, 1); 1397 sg_set_page(&first_sg, sg_page(sg), len, sg->offset + page_off); 1398 1399 ahash_request_set_crypt(hash, &first_sg, NULL, len); 1400 crypto_ahash_update(hash); 1401 1402 data_length -= len; 1403 sg = sg_next(sg); 1404 } 1405 1406 while (data_length) { 1407 u32 cur_len = min_t(u32, data_length, sg->length); 1408 1409 ahash_request_set_crypt(hash, sg, NULL, cur_len); 1410 crypto_ahash_update(hash); 1411 1412 data_length -= cur_len; 1413 /* iscsit_map_iovec has already checked for invalid sg pointers */ 1414 sg = sg_next(sg); 1415 } 1416 1417 if (padding) { 1418 struct scatterlist pad_sg; 1419 1420 sg_init_one(&pad_sg, pad_bytes, padding); 1421 ahash_request_set_crypt(hash, &pad_sg, (u8 *)&data_crc, 1422 padding); 1423 crypto_ahash_finup(hash); 1424 } else { 1425 ahash_request_set_crypt(hash, NULL, (u8 *)&data_crc, 0); 1426 crypto_ahash_final(hash); 1427 } 1428 1429 return data_crc; 1430 } 1431 1432 static void iscsit_do_crypto_hash_buf(struct ahash_request *hash, 1433 const void *buf, u32 payload_length, u32 padding, 1434 const void *pad_bytes, void *data_crc) 1435 { 1436 struct scatterlist sg[2]; 1437 1438 sg_init_table(sg, ARRAY_SIZE(sg)); 1439 sg_set_buf(sg, buf, payload_length); 1440 if (padding) 1441 sg_set_buf(sg + 1, pad_bytes, padding); 1442 1443 ahash_request_set_crypt(hash, sg, data_crc, payload_length + padding); 1444 1445 crypto_ahash_digest(hash); 1446 } 1447 1448 int 1449 __iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf, 1450 struct iscsi_cmd *cmd, u32 payload_length, 1451 bool *success) 1452 { 1453 struct iscsi_data *hdr = buf; 1454 struct se_cmd *se_cmd; 1455 int rc; 1456 1457 /* iSCSI write */ 1458 atomic_long_add(payload_length, &conn->sess->rx_data_octets); 1459 1460 pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x," 1461 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", 1462 hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset), 1463 payload_length, conn->cid); 1464 1465 if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) { 1466 pr_err("Command ITT: 0x%08x received DataOUT after" 1467 " last DataOUT received, dumping payload\n", 1468 cmd->init_task_tag); 1469 return iscsit_dump_data_payload(conn, payload_length, 1); 1470 } 1471 1472 if (cmd->data_direction != DMA_TO_DEVICE) { 1473 pr_err("Command ITT: 0x%08x received DataOUT for a" 1474 " NON-WRITE command.\n", cmd->init_task_tag); 1475 return iscsit_dump_data_payload(conn, payload_length, 1); 1476 } 1477 se_cmd = &cmd->se_cmd; 1478 iscsit_mod_dataout_timer(cmd); 1479 1480 if ((be32_to_cpu(hdr->offset) + payload_length) > cmd->se_cmd.data_length) { 1481 pr_err("DataOut Offset: %u, Length %u greater than iSCSI Command EDTL %u, protocol error.\n", 1482 be32_to_cpu(hdr->offset), payload_length, 1483 cmd->se_cmd.data_length); 1484 return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID, buf); 1485 } 1486 1487 if (cmd->unsolicited_data) { 1488 int dump_unsolicited_data = 0; 1489 1490 if (conn->sess->sess_ops->InitialR2T) { 1491 pr_err("Received unexpected unsolicited data" 1492 " while InitialR2T=Yes, protocol error.\n"); 1493 transport_send_check_condition_and_sense(&cmd->se_cmd, 1494 TCM_UNEXPECTED_UNSOLICITED_DATA, 0); 1495 return -1; 1496 } 1497 /* 1498 * Special case for dealing with Unsolicited DataOUT 1499 * and Unsupported SAM WRITE Opcodes and SE resource allocation 1500 * failures; 1501 */ 1502 1503 /* Something's amiss if we're not in WRITE_PENDING state... */ 1504 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING); 1505 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE)) 1506 dump_unsolicited_data = 1; 1507 1508 if (dump_unsolicited_data) { 1509 /* 1510 * Check if a delayed TASK_ABORTED status needs to 1511 * be sent now if the ISCSI_FLAG_CMD_FINAL has been 1512 * received with the unsolicited data out. 1513 */ 1514 if (hdr->flags & ISCSI_FLAG_CMD_FINAL) 1515 iscsit_stop_dataout_timer(cmd); 1516 1517 return iscsit_dump_data_payload(conn, payload_length, 1); 1518 } 1519 } else { 1520 /* 1521 * For the normal solicited data path: 1522 * 1523 * Check for a delayed TASK_ABORTED status and dump any 1524 * incoming data out payload if one exists. Also, when the 1525 * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current 1526 * data out sequence, we decrement outstanding_r2ts. Once 1527 * outstanding_r2ts reaches zero, go ahead and send the delayed 1528 * TASK_ABORTED status. 1529 */ 1530 if (se_cmd->transport_state & CMD_T_ABORTED) { 1531 if (hdr->flags & ISCSI_FLAG_CMD_FINAL && 1532 --cmd->outstanding_r2ts < 1) 1533 iscsit_stop_dataout_timer(cmd); 1534 1535 return iscsit_dump_data_payload(conn, payload_length, 1); 1536 } 1537 } 1538 /* 1539 * Perform DataSN, DataSequenceInOrder, DataPDUInOrder, and 1540 * within-command recovery checks before receiving the payload. 1541 */ 1542 rc = iscsit_check_pre_dataout(cmd, buf); 1543 if (rc == DATAOUT_WITHIN_COMMAND_RECOVERY) 1544 return 0; 1545 else if (rc == DATAOUT_CANNOT_RECOVER) 1546 return -1; 1547 *success = true; 1548 return 0; 1549 } 1550 EXPORT_SYMBOL(__iscsit_check_dataout_hdr); 1551 1552 int 1553 iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf, 1554 struct iscsi_cmd **out_cmd) 1555 { 1556 struct iscsi_data *hdr = buf; 1557 struct iscsi_cmd *cmd; 1558 u32 payload_length = ntoh24(hdr->dlength); 1559 int rc; 1560 bool success = false; 1561 1562 if (!payload_length) { 1563 pr_warn_ratelimited("DataOUT payload is ZERO, ignoring.\n"); 1564 return 0; 1565 } 1566 1567 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 1568 pr_err_ratelimited("DataSegmentLength: %u is greater than" 1569 " MaxXmitDataSegmentLength: %u\n", payload_length, 1570 conn->conn_ops->MaxXmitDataSegmentLength); 1571 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, buf); 1572 } 1573 1574 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, payload_length); 1575 if (!cmd) 1576 return 0; 1577 1578 rc = __iscsit_check_dataout_hdr(conn, buf, cmd, payload_length, &success); 1579 1580 if (success) 1581 *out_cmd = cmd; 1582 1583 return rc; 1584 } 1585 EXPORT_SYMBOL(iscsit_check_dataout_hdr); 1586 1587 static int 1588 iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1589 struct iscsi_data *hdr) 1590 { 1591 struct kvec *iov; 1592 u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0; 1593 u32 payload_length; 1594 int iov_ret, data_crc_failed = 0; 1595 1596 payload_length = min_t(u32, cmd->se_cmd.data_length, 1597 ntoh24(hdr->dlength)); 1598 rx_size += payload_length; 1599 iov = &cmd->iov_data[0]; 1600 1601 iov_ret = iscsit_map_iovec(cmd, iov, cmd->orig_iov_data_count - 2, 1602 be32_to_cpu(hdr->offset), payload_length); 1603 if (iov_ret < 0) 1604 return -1; 1605 1606 iov_count += iov_ret; 1607 1608 padding = ((-payload_length) & 3); 1609 if (padding != 0) { 1610 iov[iov_count].iov_base = cmd->pad_bytes; 1611 iov[iov_count++].iov_len = padding; 1612 rx_size += padding; 1613 pr_debug("Receiving %u padding bytes.\n", padding); 1614 } 1615 1616 if (conn->conn_ops->DataDigest) { 1617 iov[iov_count].iov_base = &checksum; 1618 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 1619 rx_size += ISCSI_CRC_LEN; 1620 } 1621 1622 WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count); 1623 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); 1624 1625 iscsit_unmap_iovec(cmd); 1626 1627 if (rx_got != rx_size) 1628 return -1; 1629 1630 if (conn->conn_ops->DataDigest) { 1631 u32 data_crc; 1632 1633 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd, 1634 be32_to_cpu(hdr->offset), 1635 payload_length, padding, 1636 cmd->pad_bytes); 1637 1638 if (checksum != data_crc) { 1639 pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 1640 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x" 1641 " does not match computed 0x%08x\n", 1642 hdr->itt, hdr->offset, payload_length, 1643 hdr->datasn, checksum, data_crc); 1644 data_crc_failed = 1; 1645 } else { 1646 pr_debug("Got CRC32C DataDigest 0x%08x for" 1647 " %u bytes of Data Out\n", checksum, 1648 payload_length); 1649 } 1650 } 1651 1652 return data_crc_failed; 1653 } 1654 1655 int 1656 iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr, 1657 bool data_crc_failed) 1658 { 1659 struct iscsi_conn *conn = cmd->conn; 1660 int rc, ooo_cmdsn; 1661 /* 1662 * Increment post receive data and CRC values or perform 1663 * within-command recovery. 1664 */ 1665 rc = iscsit_check_post_dataout(cmd, (unsigned char *)hdr, data_crc_failed); 1666 if ((rc == DATAOUT_NORMAL) || (rc == DATAOUT_WITHIN_COMMAND_RECOVERY)) 1667 return 0; 1668 else if (rc == DATAOUT_SEND_R2T) { 1669 iscsit_set_dataout_sequence_values(cmd); 1670 conn->conn_transport->iscsit_get_dataout(conn, cmd, false); 1671 } else if (rc == DATAOUT_SEND_TO_TRANSPORT) { 1672 /* 1673 * Handle extra special case for out of order 1674 * Unsolicited Data Out. 1675 */ 1676 spin_lock_bh(&cmd->istate_lock); 1677 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN); 1678 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1679 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1680 spin_unlock_bh(&cmd->istate_lock); 1681 1682 iscsit_stop_dataout_timer(cmd); 1683 if (ooo_cmdsn) 1684 return 0; 1685 target_execute_cmd(&cmd->se_cmd); 1686 return 0; 1687 } else /* DATAOUT_CANNOT_RECOVER */ 1688 return -1; 1689 1690 return 0; 1691 } 1692 EXPORT_SYMBOL(iscsit_check_dataout_payload); 1693 1694 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) 1695 { 1696 struct iscsi_cmd *cmd = NULL; 1697 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1698 int rc; 1699 bool data_crc_failed = false; 1700 1701 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1702 if (rc < 0) 1703 return 0; 1704 else if (!cmd) 1705 return 0; 1706 1707 rc = iscsit_get_dataout(conn, cmd, hdr); 1708 if (rc < 0) 1709 return rc; 1710 else if (rc > 0) 1711 data_crc_failed = true; 1712 1713 return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed); 1714 } 1715 1716 int iscsit_setup_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1717 struct iscsi_nopout *hdr) 1718 { 1719 u32 payload_length = ntoh24(hdr->dlength); 1720 1721 if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL)) { 1722 pr_err("NopOUT Flag's, Left Most Bit not set, protocol error.\n"); 1723 if (!cmd) 1724 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1725 (unsigned char *)hdr); 1726 1727 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1728 (unsigned char *)hdr); 1729 } 1730 1731 if (hdr->itt == RESERVED_ITT && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1732 pr_err("NOPOUT ITT is reserved, but Immediate Bit is" 1733 " not set, protocol error.\n"); 1734 if (!cmd) 1735 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1736 (unsigned char *)hdr); 1737 1738 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1739 (unsigned char *)hdr); 1740 } 1741 1742 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 1743 pr_err("NOPOUT Ping Data DataSegmentLength: %u is" 1744 " greater than MaxXmitDataSegmentLength: %u, protocol" 1745 " error.\n", payload_length, 1746 conn->conn_ops->MaxXmitDataSegmentLength); 1747 if (!cmd) 1748 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1749 (unsigned char *)hdr); 1750 1751 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1752 (unsigned char *)hdr); 1753 } 1754 1755 pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%08x," 1756 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n", 1757 hdr->itt == RESERVED_ITT ? "Response" : "Request", 1758 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn, 1759 payload_length); 1760 /* 1761 * This is not a response to a Unsolicited NopIN, which means 1762 * it can either be a NOPOUT ping request (with a valid ITT), 1763 * or a NOPOUT not requesting a NOPIN (with a reserved ITT). 1764 * Either way, make sure we allocate an struct iscsi_cmd, as both 1765 * can contain ping data. 1766 */ 1767 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1768 cmd->iscsi_opcode = ISCSI_OP_NOOP_OUT; 1769 cmd->i_state = ISTATE_SEND_NOPIN; 1770 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1771 1 : 0); 1772 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 1773 cmd->targ_xfer_tag = 0xFFFFFFFF; 1774 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 1775 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 1776 cmd->data_direction = DMA_NONE; 1777 } 1778 1779 return 0; 1780 } 1781 EXPORT_SYMBOL(iscsit_setup_nop_out); 1782 1783 int iscsit_process_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1784 struct iscsi_nopout *hdr) 1785 { 1786 struct iscsi_cmd *cmd_p = NULL; 1787 int cmdsn_ret = 0; 1788 /* 1789 * Initiator is expecting a NopIN ping reply.. 1790 */ 1791 if (hdr->itt != RESERVED_ITT) { 1792 if (!cmd) 1793 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1794 (unsigned char *)hdr); 1795 1796 spin_lock_bh(&conn->cmd_lock); 1797 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 1798 spin_unlock_bh(&conn->cmd_lock); 1799 1800 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 1801 1802 if (hdr->opcode & ISCSI_OP_IMMEDIATE) { 1803 iscsit_add_cmd_to_response_queue(cmd, conn, 1804 cmd->i_state); 1805 return 0; 1806 } 1807 1808 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 1809 (unsigned char *)hdr, hdr->cmdsn); 1810 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) 1811 return 0; 1812 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1813 return -1; 1814 1815 return 0; 1816 } 1817 /* 1818 * This was a response to a unsolicited NOPIN ping. 1819 */ 1820 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 1821 cmd_p = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt)); 1822 if (!cmd_p) 1823 return -EINVAL; 1824 1825 iscsit_stop_nopin_response_timer(conn); 1826 1827 cmd_p->i_state = ISTATE_REMOVE; 1828 iscsit_add_cmd_to_immediate_queue(cmd_p, conn, cmd_p->i_state); 1829 1830 iscsit_start_nopin_timer(conn); 1831 return 0; 1832 } 1833 /* 1834 * Otherwise, initiator is not expecting a NOPIN is response. 1835 * Just ignore for now. 1836 */ 1837 1838 if (cmd) 1839 iscsit_free_cmd(cmd, false); 1840 1841 return 0; 1842 } 1843 EXPORT_SYMBOL(iscsit_process_nop_out); 1844 1845 static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1846 unsigned char *buf) 1847 { 1848 unsigned char *ping_data = NULL; 1849 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1850 struct kvec *iov = NULL; 1851 u32 payload_length = ntoh24(hdr->dlength); 1852 int ret; 1853 1854 ret = iscsit_setup_nop_out(conn, cmd, hdr); 1855 if (ret < 0) 1856 return 0; 1857 /* 1858 * Handle NOP-OUT payload for traditional iSCSI sockets 1859 */ 1860 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1861 u32 checksum, data_crc, padding = 0; 1862 int niov = 0, rx_got, rx_size = payload_length; 1863 1864 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1865 if (!ping_data) { 1866 ret = -1; 1867 goto out; 1868 } 1869 1870 iov = &cmd->iov_misc[0]; 1871 iov[niov].iov_base = ping_data; 1872 iov[niov++].iov_len = payload_length; 1873 1874 padding = ((-payload_length) & 3); 1875 if (padding != 0) { 1876 pr_debug("Receiving %u additional bytes" 1877 " for padding.\n", padding); 1878 iov[niov].iov_base = &cmd->pad_bytes; 1879 iov[niov++].iov_len = padding; 1880 rx_size += padding; 1881 } 1882 if (conn->conn_ops->DataDigest) { 1883 iov[niov].iov_base = &checksum; 1884 iov[niov++].iov_len = ISCSI_CRC_LEN; 1885 rx_size += ISCSI_CRC_LEN; 1886 } 1887 1888 WARN_ON_ONCE(niov > ARRAY_SIZE(cmd->iov_misc)); 1889 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size); 1890 if (rx_got != rx_size) { 1891 ret = -1; 1892 goto out; 1893 } 1894 1895 if (conn->conn_ops->DataDigest) { 1896 iscsit_do_crypto_hash_buf(conn->conn_rx_hash, ping_data, 1897 payload_length, padding, 1898 cmd->pad_bytes, &data_crc); 1899 1900 if (checksum != data_crc) { 1901 pr_err("Ping data CRC32C DataDigest" 1902 " 0x%08x does not match computed 0x%08x\n", 1903 checksum, data_crc); 1904 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1905 pr_err("Unable to recover from" 1906 " NOPOUT Ping DataCRC failure while in" 1907 " ERL=0.\n"); 1908 ret = -1; 1909 goto out; 1910 } else { 1911 /* 1912 * Silently drop this PDU and let the 1913 * initiator plug the CmdSN gap. 1914 */ 1915 pr_debug("Dropping NOPOUT" 1916 " Command CmdSN: 0x%08x due to" 1917 " DataCRC error.\n", hdr->cmdsn); 1918 ret = 0; 1919 goto out; 1920 } 1921 } else { 1922 pr_debug("Got CRC32C DataDigest" 1923 " 0x%08x for %u bytes of ping data.\n", 1924 checksum, payload_length); 1925 } 1926 } 1927 1928 ping_data[payload_length] = '\0'; 1929 /* 1930 * Attach ping data to struct iscsi_cmd->buf_ptr. 1931 */ 1932 cmd->buf_ptr = ping_data; 1933 cmd->buf_ptr_size = payload_length; 1934 1935 pr_debug("Got %u bytes of NOPOUT ping" 1936 " data.\n", payload_length); 1937 pr_debug("Ping Data: \"%s\"\n", ping_data); 1938 } 1939 1940 return iscsit_process_nop_out(conn, cmd, hdr); 1941 out: 1942 if (cmd) 1943 iscsit_free_cmd(cmd, false); 1944 1945 kfree(ping_data); 1946 return ret; 1947 } 1948 1949 static enum tcm_tmreq_table iscsit_convert_tmf(u8 iscsi_tmf) 1950 { 1951 switch (iscsi_tmf) { 1952 case ISCSI_TM_FUNC_ABORT_TASK: 1953 return TMR_ABORT_TASK; 1954 case ISCSI_TM_FUNC_ABORT_TASK_SET: 1955 return TMR_ABORT_TASK_SET; 1956 case ISCSI_TM_FUNC_CLEAR_ACA: 1957 return TMR_CLEAR_ACA; 1958 case ISCSI_TM_FUNC_CLEAR_TASK_SET: 1959 return TMR_CLEAR_TASK_SET; 1960 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET: 1961 return TMR_LUN_RESET; 1962 case ISCSI_TM_FUNC_TARGET_WARM_RESET: 1963 return TMR_TARGET_WARM_RESET; 1964 case ISCSI_TM_FUNC_TARGET_COLD_RESET: 1965 return TMR_TARGET_COLD_RESET; 1966 default: 1967 return TMR_UNKNOWN; 1968 } 1969 } 1970 1971 int 1972 iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 1973 unsigned char *buf) 1974 { 1975 struct se_tmr_req *se_tmr; 1976 struct iscsi_tmr_req *tmr_req; 1977 struct iscsi_tm *hdr; 1978 int out_of_order_cmdsn = 0, ret; 1979 u8 function, tcm_function = TMR_UNKNOWN; 1980 1981 hdr = (struct iscsi_tm *) buf; 1982 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL; 1983 function = hdr->flags; 1984 1985 pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:" 1986 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:" 1987 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function, 1988 hdr->rtt, hdr->refcmdsn, conn->cid); 1989 1990 if ((function != ISCSI_TM_FUNC_ABORT_TASK) && 1991 ((function != ISCSI_TM_FUNC_TASK_REASSIGN) && 1992 hdr->rtt != RESERVED_ITT)) { 1993 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n"); 1994 hdr->rtt = RESERVED_ITT; 1995 } 1996 1997 if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) && 1998 !(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1999 pr_err("Task Management Request TASK_REASSIGN not" 2000 " issued as immediate command, bad iSCSI Initiator" 2001 "implementation\n"); 2002 return iscsit_add_reject_cmd(cmd, 2003 ISCSI_REASON_PROTOCOL_ERROR, buf); 2004 } 2005 if ((function != ISCSI_TM_FUNC_ABORT_TASK) && 2006 be32_to_cpu(hdr->refcmdsn) != ISCSI_RESERVED_TAG) 2007 hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG); 2008 2009 cmd->data_direction = DMA_NONE; 2010 cmd->tmr_req = kzalloc(sizeof(*cmd->tmr_req), GFP_KERNEL); 2011 if (!cmd->tmr_req) { 2012 return iscsit_add_reject_cmd(cmd, 2013 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 2014 buf); 2015 } 2016 2017 __target_init_cmd(&cmd->se_cmd, &iscsi_ops, 2018 conn->sess->se_sess, 0, DMA_NONE, 2019 TCM_SIMPLE_TAG, cmd->sense_buffer + 2, 2020 scsilun_to_int(&hdr->lun)); 2021 2022 target_get_sess_cmd(&cmd->se_cmd, true); 2023 2024 /* 2025 * TASK_REASSIGN for ERL=2 / connection stays inside of 2026 * LIO-Target $FABRIC_MOD 2027 */ 2028 if (function != ISCSI_TM_FUNC_TASK_REASSIGN) { 2029 tcm_function = iscsit_convert_tmf(function); 2030 if (tcm_function == TMR_UNKNOWN) { 2031 pr_err("Unknown iSCSI TMR Function:" 2032 " 0x%02x\n", function); 2033 return iscsit_add_reject_cmd(cmd, 2034 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 2035 } 2036 } 2037 ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req, tcm_function, 2038 GFP_KERNEL); 2039 if (ret < 0) 2040 return iscsit_add_reject_cmd(cmd, 2041 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 2042 2043 cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req; 2044 2045 cmd->iscsi_opcode = ISCSI_OP_SCSI_TMFUNC; 2046 cmd->i_state = ISTATE_SEND_TASKMGTRSP; 2047 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 2048 cmd->init_task_tag = hdr->itt; 2049 cmd->targ_xfer_tag = 0xFFFFFFFF; 2050 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 2051 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 2052 se_tmr = cmd->se_cmd.se_tmr_req; 2053 tmr_req = cmd->tmr_req; 2054 /* 2055 * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN 2056 */ 2057 if (function != ISCSI_TM_FUNC_TASK_REASSIGN) { 2058 ret = transport_lookup_tmr_lun(&cmd->se_cmd); 2059 if (ret < 0) { 2060 se_tmr->response = ISCSI_TMF_RSP_NO_LUN; 2061 goto attach; 2062 } 2063 } 2064 2065 switch (function) { 2066 case ISCSI_TM_FUNC_ABORT_TASK: 2067 se_tmr->response = iscsit_tmr_abort_task(cmd, buf); 2068 if (se_tmr->response) 2069 goto attach; 2070 break; 2071 case ISCSI_TM_FUNC_ABORT_TASK_SET: 2072 case ISCSI_TM_FUNC_CLEAR_ACA: 2073 case ISCSI_TM_FUNC_CLEAR_TASK_SET: 2074 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET: 2075 break; 2076 case ISCSI_TM_FUNC_TARGET_WARM_RESET: 2077 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) { 2078 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED; 2079 goto attach; 2080 } 2081 break; 2082 case ISCSI_TM_FUNC_TARGET_COLD_RESET: 2083 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) { 2084 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED; 2085 goto attach; 2086 } 2087 break; 2088 case ISCSI_TM_FUNC_TASK_REASSIGN: 2089 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf); 2090 /* 2091 * Perform sanity checks on the ExpDataSN only if the 2092 * TASK_REASSIGN was successful. 2093 */ 2094 if (se_tmr->response) 2095 break; 2096 2097 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0) 2098 return iscsit_add_reject_cmd(cmd, 2099 ISCSI_REASON_BOOKMARK_INVALID, buf); 2100 break; 2101 default: 2102 pr_err("Unknown TMR function: 0x%02x, protocol" 2103 " error.\n", function); 2104 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED; 2105 goto attach; 2106 } 2107 2108 if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) && 2109 (se_tmr->response == ISCSI_TMF_RSP_COMPLETE)) 2110 se_tmr->call_transport = 1; 2111 attach: 2112 spin_lock_bh(&conn->cmd_lock); 2113 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 2114 spin_unlock_bh(&conn->cmd_lock); 2115 2116 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 2117 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 2118 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP) { 2119 out_of_order_cmdsn = 1; 2120 } else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 2121 target_put_sess_cmd(&cmd->se_cmd); 2122 return 0; 2123 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) { 2124 return -1; 2125 } 2126 } 2127 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 2128 2129 if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE)) 2130 return 0; 2131 /* 2132 * Found the referenced task, send to transport for processing. 2133 */ 2134 if (se_tmr->call_transport) 2135 return transport_generic_handle_tmr(&cmd->se_cmd); 2136 2137 /* 2138 * Could not find the referenced LUN, task, or Task Management 2139 * command not authorized or supported. Change state and 2140 * let the tx_thread send the response. 2141 * 2142 * For connection recovery, this is also the default action for 2143 * TMR TASK_REASSIGN. 2144 */ 2145 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2146 target_put_sess_cmd(&cmd->se_cmd); 2147 return 0; 2148 } 2149 EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd); 2150 2151 /* #warning FIXME: Support Text Command parameters besides SendTargets */ 2152 int 2153 iscsit_setup_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2154 struct iscsi_text *hdr) 2155 { 2156 u32 payload_length = ntoh24(hdr->dlength); 2157 2158 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { 2159 pr_err("Unable to accept text parameter length: %u" 2160 "greater than MaxXmitDataSegmentLength %u.\n", 2161 payload_length, conn->conn_ops->MaxXmitDataSegmentLength); 2162 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 2163 (unsigned char *)hdr); 2164 } 2165 2166 if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL) || 2167 (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)) { 2168 pr_err("Multi sequence text commands currently not supported\n"); 2169 return iscsit_reject_cmd(cmd, ISCSI_REASON_CMD_NOT_SUPPORTED, 2170 (unsigned char *)hdr); 2171 } 2172 2173 pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x," 2174 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn, 2175 hdr->exp_statsn, payload_length); 2176 2177 cmd->iscsi_opcode = ISCSI_OP_TEXT; 2178 cmd->i_state = ISTATE_SEND_TEXTRSP; 2179 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 2180 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 2181 cmd->targ_xfer_tag = 0xFFFFFFFF; 2182 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 2183 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 2184 cmd->data_direction = DMA_NONE; 2185 kfree(cmd->text_in_ptr); 2186 cmd->text_in_ptr = NULL; 2187 2188 return 0; 2189 } 2190 EXPORT_SYMBOL(iscsit_setup_text_cmd); 2191 2192 int 2193 iscsit_process_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2194 struct iscsi_text *hdr) 2195 { 2196 unsigned char *text_in = cmd->text_in_ptr, *text_ptr; 2197 int cmdsn_ret; 2198 2199 if (!text_in) { 2200 cmd->targ_xfer_tag = be32_to_cpu(hdr->ttt); 2201 if (cmd->targ_xfer_tag == 0xFFFFFFFF) { 2202 pr_err("Unable to locate text_in buffer for sendtargets" 2203 " discovery\n"); 2204 goto reject; 2205 } 2206 goto empty_sendtargets; 2207 } 2208 if (strncmp("SendTargets=", text_in, 12) != 0) { 2209 pr_err("Received Text Data that is not" 2210 " SendTargets, cannot continue.\n"); 2211 goto reject; 2212 } 2213 /* '=' confirmed in strncmp */ 2214 text_ptr = strchr(text_in, '='); 2215 BUG_ON(!text_ptr); 2216 if (!strncmp("=All", text_ptr, 5)) { 2217 cmd->cmd_flags |= ICF_SENDTARGETS_ALL; 2218 } else if (!strncmp("=iqn.", text_ptr, 5) || 2219 !strncmp("=eui.", text_ptr, 5)) { 2220 cmd->cmd_flags |= ICF_SENDTARGETS_SINGLE; 2221 } else { 2222 pr_err("Unable to locate valid SendTargets%s value\n", 2223 text_ptr); 2224 goto reject; 2225 } 2226 2227 spin_lock_bh(&conn->cmd_lock); 2228 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 2229 spin_unlock_bh(&conn->cmd_lock); 2230 2231 empty_sendtargets: 2232 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 2233 2234 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 2235 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 2236 (unsigned char *)hdr, hdr->cmdsn); 2237 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 2238 return -1; 2239 2240 return 0; 2241 } 2242 2243 return iscsit_execute_cmd(cmd, 0); 2244 2245 reject: 2246 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 2247 (unsigned char *)hdr); 2248 } 2249 EXPORT_SYMBOL(iscsit_process_text_cmd); 2250 2251 static int 2252 iscsit_handle_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2253 unsigned char *buf) 2254 { 2255 struct iscsi_text *hdr = (struct iscsi_text *)buf; 2256 char *text_in = NULL; 2257 u32 payload_length = ntoh24(hdr->dlength); 2258 int rx_size, rc; 2259 2260 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 2261 if (rc < 0) 2262 return 0; 2263 2264 rx_size = payload_length; 2265 if (payload_length) { 2266 u32 checksum = 0, data_crc = 0; 2267 u32 padding = 0; 2268 int niov = 0, rx_got; 2269 struct kvec iov[2]; 2270 2271 rx_size = ALIGN(payload_length, 4); 2272 text_in = kzalloc(rx_size, GFP_KERNEL); 2273 if (!text_in) 2274 goto reject; 2275 2276 cmd->text_in_ptr = text_in; 2277 2278 memset(iov, 0, sizeof(iov)); 2279 iov[niov].iov_base = text_in; 2280 iov[niov++].iov_len = rx_size; 2281 2282 padding = rx_size - payload_length; 2283 if (padding) 2284 pr_debug("Receiving %u additional bytes" 2285 " for padding.\n", padding); 2286 if (conn->conn_ops->DataDigest) { 2287 iov[niov].iov_base = &checksum; 2288 iov[niov++].iov_len = ISCSI_CRC_LEN; 2289 rx_size += ISCSI_CRC_LEN; 2290 } 2291 2292 WARN_ON_ONCE(niov > ARRAY_SIZE(iov)); 2293 rx_got = rx_data(conn, &iov[0], niov, rx_size); 2294 if (rx_got != rx_size) 2295 goto reject; 2296 2297 if (conn->conn_ops->DataDigest) { 2298 iscsit_do_crypto_hash_buf(conn->conn_rx_hash, 2299 text_in, rx_size, 0, NULL, 2300 &data_crc); 2301 2302 if (checksum != data_crc) { 2303 pr_err("Text data CRC32C DataDigest" 2304 " 0x%08x does not match computed" 2305 " 0x%08x\n", checksum, data_crc); 2306 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2307 pr_err("Unable to recover from" 2308 " Text Data digest failure while in" 2309 " ERL=0.\n"); 2310 goto reject; 2311 } else { 2312 /* 2313 * Silently drop this PDU and let the 2314 * initiator plug the CmdSN gap. 2315 */ 2316 pr_debug("Dropping Text" 2317 " Command CmdSN: 0x%08x due to" 2318 " DataCRC error.\n", hdr->cmdsn); 2319 kfree(text_in); 2320 return 0; 2321 } 2322 } else { 2323 pr_debug("Got CRC32C DataDigest" 2324 " 0x%08x for %u bytes of text data.\n", 2325 checksum, payload_length); 2326 } 2327 } 2328 text_in[payload_length - 1] = '\0'; 2329 pr_debug("Successfully read %d bytes of text" 2330 " data.\n", payload_length); 2331 } 2332 2333 return iscsit_process_text_cmd(conn, cmd, hdr); 2334 2335 reject: 2336 kfree(cmd->text_in_ptr); 2337 cmd->text_in_ptr = NULL; 2338 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf); 2339 } 2340 2341 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2342 { 2343 struct iscsi_conn *conn_p; 2344 struct iscsi_session *sess = conn->sess; 2345 2346 pr_debug("Received logout request CLOSESESSION on CID: %hu" 2347 " for SID: %u.\n", conn->cid, conn->sess->sid); 2348 2349 atomic_set(&sess->session_logout, 1); 2350 atomic_set(&conn->conn_logout_remove, 1); 2351 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION; 2352 2353 iscsit_inc_conn_usage_count(conn); 2354 iscsit_inc_session_usage_count(sess); 2355 2356 spin_lock_bh(&sess->conn_lock); 2357 list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) { 2358 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN) 2359 continue; 2360 2361 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 2362 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT; 2363 } 2364 spin_unlock_bh(&sess->conn_lock); 2365 2366 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2367 2368 return 0; 2369 } 2370 2371 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2372 { 2373 struct iscsi_conn *l_conn; 2374 struct iscsi_session *sess = conn->sess; 2375 2376 pr_debug("Received logout request CLOSECONNECTION for CID:" 2377 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); 2378 2379 /* 2380 * A Logout Request with a CLOSECONNECTION reason code for a CID 2381 * can arrive on a connection with a differing CID. 2382 */ 2383 if (conn->cid == cmd->logout_cid) { 2384 spin_lock_bh(&conn->state_lock); 2385 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 2386 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT; 2387 2388 atomic_set(&conn->conn_logout_remove, 1); 2389 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION; 2390 iscsit_inc_conn_usage_count(conn); 2391 2392 spin_unlock_bh(&conn->state_lock); 2393 } else { 2394 /* 2395 * Handle all different cid CLOSECONNECTION requests in 2396 * iscsit_logout_post_handler_diffcid() as to give enough 2397 * time for any non immediate command's CmdSN to be 2398 * acknowledged on the connection in question. 2399 * 2400 * Here we simply make sure the CID is still around. 2401 */ 2402 l_conn = iscsit_get_conn_from_cid(sess, 2403 cmd->logout_cid); 2404 if (!l_conn) { 2405 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND; 2406 iscsit_add_cmd_to_response_queue(cmd, conn, 2407 cmd->i_state); 2408 return 0; 2409 } 2410 2411 iscsit_dec_conn_usage_count(l_conn); 2412 } 2413 2414 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2415 2416 return 0; 2417 } 2418 2419 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2420 { 2421 struct iscsi_session *sess = conn->sess; 2422 2423 pr_debug("Received explicit REMOVECONNFORRECOVERY logout for" 2424 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); 2425 2426 if (sess->sess_ops->ErrorRecoveryLevel != 2) { 2427 pr_err("Received Logout Request REMOVECONNFORRECOVERY" 2428 " while ERL!=2.\n"); 2429 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED; 2430 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2431 return 0; 2432 } 2433 2434 if (conn->cid == cmd->logout_cid) { 2435 pr_err("Received Logout Request REMOVECONNFORRECOVERY" 2436 " with CID: %hu on CID: %hu, implementation error.\n", 2437 cmd->logout_cid, conn->cid); 2438 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED; 2439 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2440 return 0; 2441 } 2442 2443 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2444 2445 return 0; 2446 } 2447 2448 int 2449 iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2450 unsigned char *buf) 2451 { 2452 int cmdsn_ret, logout_remove = 0; 2453 u8 reason_code = 0; 2454 struct iscsi_logout *hdr; 2455 struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn); 2456 2457 hdr = (struct iscsi_logout *) buf; 2458 reason_code = (hdr->flags & 0x7f); 2459 2460 if (tiqn) { 2461 spin_lock(&tiqn->logout_stats.lock); 2462 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) 2463 tiqn->logout_stats.normal_logouts++; 2464 else 2465 tiqn->logout_stats.abnormal_logouts++; 2466 spin_unlock(&tiqn->logout_stats.lock); 2467 } 2468 2469 pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x" 2470 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n", 2471 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code, 2472 hdr->cid, conn->cid); 2473 2474 if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) { 2475 pr_err("Received logout request on connection that" 2476 " is not in logged in state, ignoring request.\n"); 2477 iscsit_free_cmd(cmd, false); 2478 return 0; 2479 } 2480 2481 cmd->iscsi_opcode = ISCSI_OP_LOGOUT; 2482 cmd->i_state = ISTATE_SEND_LOGOUTRSP; 2483 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 2484 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 2485 cmd->targ_xfer_tag = 0xFFFFFFFF; 2486 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 2487 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); 2488 cmd->logout_cid = be16_to_cpu(hdr->cid); 2489 cmd->logout_reason = reason_code; 2490 cmd->data_direction = DMA_NONE; 2491 2492 /* 2493 * We need to sleep in these cases (by returning 1) until the Logout 2494 * Response gets sent in the tx thread. 2495 */ 2496 if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) || 2497 ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) && 2498 be16_to_cpu(hdr->cid) == conn->cid)) 2499 logout_remove = 1; 2500 2501 spin_lock_bh(&conn->cmd_lock); 2502 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 2503 spin_unlock_bh(&conn->cmd_lock); 2504 2505 if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY) 2506 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); 2507 2508 /* 2509 * Immediate commands are executed, well, immediately. 2510 * Non-Immediate Logout Commands are executed in CmdSN order. 2511 */ 2512 if (cmd->immediate_cmd) { 2513 int ret = iscsit_execute_cmd(cmd, 0); 2514 2515 if (ret < 0) 2516 return ret; 2517 } else { 2518 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 2519 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) 2520 logout_remove = 0; 2521 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 2522 return -1; 2523 } 2524 2525 return logout_remove; 2526 } 2527 EXPORT_SYMBOL(iscsit_handle_logout_cmd); 2528 2529 int iscsit_handle_snack( 2530 struct iscsi_conn *conn, 2531 unsigned char *buf) 2532 { 2533 struct iscsi_snack *hdr; 2534 2535 hdr = (struct iscsi_snack *) buf; 2536 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL; 2537 2538 pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:" 2539 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x," 2540 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags, 2541 hdr->begrun, hdr->runlength, conn->cid); 2542 2543 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2544 pr_err("Initiator sent SNACK request while in" 2545 " ErrorRecoveryLevel=0.\n"); 2546 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 2547 buf); 2548 } 2549 /* 2550 * SNACK_DATA and SNACK_R2T are both 0, so check which function to 2551 * call from inside iscsi_send_recovery_datain_or_r2t(). 2552 */ 2553 switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) { 2554 case 0: 2555 return iscsit_handle_recovery_datain_or_r2t(conn, buf, 2556 hdr->itt, 2557 be32_to_cpu(hdr->ttt), 2558 be32_to_cpu(hdr->begrun), 2559 be32_to_cpu(hdr->runlength)); 2560 case ISCSI_FLAG_SNACK_TYPE_STATUS: 2561 return iscsit_handle_status_snack(conn, hdr->itt, 2562 be32_to_cpu(hdr->ttt), 2563 be32_to_cpu(hdr->begrun), be32_to_cpu(hdr->runlength)); 2564 case ISCSI_FLAG_SNACK_TYPE_DATA_ACK: 2565 return iscsit_handle_data_ack(conn, be32_to_cpu(hdr->ttt), 2566 be32_to_cpu(hdr->begrun), 2567 be32_to_cpu(hdr->runlength)); 2568 case ISCSI_FLAG_SNACK_TYPE_RDATA: 2569 /* FIXME: Support R-Data SNACK */ 2570 pr_err("R-Data SNACK Not Supported.\n"); 2571 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 2572 buf); 2573 default: 2574 pr_err("Unknown SNACK type 0x%02x, protocol" 2575 " error.\n", hdr->flags & 0x0f); 2576 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 2577 buf); 2578 } 2579 2580 return 0; 2581 } 2582 EXPORT_SYMBOL(iscsit_handle_snack); 2583 2584 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn) 2585 { 2586 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || 2587 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { 2588 wait_for_completion_interruptible_timeout( 2589 &conn->rx_half_close_comp, 2590 ISCSI_RX_THREAD_TCP_TIMEOUT * HZ); 2591 } 2592 } 2593 2594 static int iscsit_handle_immediate_data( 2595 struct iscsi_cmd *cmd, 2596 struct iscsi_scsi_req *hdr, 2597 u32 length) 2598 { 2599 int iov_ret, rx_got = 0, rx_size = 0; 2600 u32 checksum, iov_count = 0, padding = 0; 2601 struct iscsi_conn *conn = cmd->conn; 2602 struct kvec *iov; 2603 void *overflow_buf = NULL; 2604 2605 BUG_ON(cmd->write_data_done > cmd->se_cmd.data_length); 2606 rx_size = min(cmd->se_cmd.data_length - cmd->write_data_done, length); 2607 iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, 2608 cmd->orig_iov_data_count - 2, 2609 cmd->write_data_done, rx_size); 2610 if (iov_ret < 0) 2611 return IMMEDIATE_DATA_CANNOT_RECOVER; 2612 2613 iov_count = iov_ret; 2614 iov = &cmd->iov_data[0]; 2615 if (rx_size < length) { 2616 /* 2617 * Special case: length of immediate data exceeds the data 2618 * buffer size derived from the CDB. 2619 */ 2620 overflow_buf = kmalloc(length - rx_size, GFP_KERNEL); 2621 if (!overflow_buf) { 2622 iscsit_unmap_iovec(cmd); 2623 return IMMEDIATE_DATA_CANNOT_RECOVER; 2624 } 2625 cmd->overflow_buf = overflow_buf; 2626 iov[iov_count].iov_base = overflow_buf; 2627 iov[iov_count].iov_len = length - rx_size; 2628 iov_count++; 2629 rx_size = length; 2630 } 2631 2632 padding = ((-length) & 3); 2633 if (padding != 0) { 2634 iov[iov_count].iov_base = cmd->pad_bytes; 2635 iov[iov_count++].iov_len = padding; 2636 rx_size += padding; 2637 } 2638 2639 if (conn->conn_ops->DataDigest) { 2640 iov[iov_count].iov_base = &checksum; 2641 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 2642 rx_size += ISCSI_CRC_LEN; 2643 } 2644 2645 WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count); 2646 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); 2647 2648 iscsit_unmap_iovec(cmd); 2649 2650 if (rx_got != rx_size) { 2651 iscsit_rx_thread_wait_for_tcp(conn); 2652 return IMMEDIATE_DATA_CANNOT_RECOVER; 2653 } 2654 2655 if (conn->conn_ops->DataDigest) { 2656 u32 data_crc; 2657 2658 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd, 2659 cmd->write_data_done, length, padding, 2660 cmd->pad_bytes); 2661 2662 if (checksum != data_crc) { 2663 pr_err("ImmediateData CRC32C DataDigest 0x%08x" 2664 " does not match computed 0x%08x\n", checksum, 2665 data_crc); 2666 2667 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2668 pr_err("Unable to recover from" 2669 " Immediate Data digest failure while" 2670 " in ERL=0.\n"); 2671 iscsit_reject_cmd(cmd, 2672 ISCSI_REASON_DATA_DIGEST_ERROR, 2673 (unsigned char *)hdr); 2674 return IMMEDIATE_DATA_CANNOT_RECOVER; 2675 } else { 2676 iscsit_reject_cmd(cmd, 2677 ISCSI_REASON_DATA_DIGEST_ERROR, 2678 (unsigned char *)hdr); 2679 return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 2680 } 2681 } else { 2682 pr_debug("Got CRC32C DataDigest 0x%08x for" 2683 " %u bytes of Immediate Data\n", checksum, 2684 length); 2685 } 2686 } 2687 2688 cmd->write_data_done += length; 2689 2690 if (cmd->write_data_done == cmd->se_cmd.data_length) { 2691 spin_lock_bh(&cmd->istate_lock); 2692 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 2693 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 2694 spin_unlock_bh(&cmd->istate_lock); 2695 } 2696 2697 return IMMEDIATE_DATA_NORMAL_OPERATION; 2698 } 2699 2700 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections 2701 with active network interface */ 2702 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn) 2703 { 2704 struct iscsi_cmd *cmd; 2705 struct iscsi_conn *conn_p; 2706 bool found = false; 2707 2708 lockdep_assert_held(&conn->sess->conn_lock); 2709 2710 /* 2711 * Only send a Asynchronous Message on connections whos network 2712 * interface is still functional. 2713 */ 2714 list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) { 2715 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) { 2716 iscsit_inc_conn_usage_count(conn_p); 2717 found = true; 2718 break; 2719 } 2720 } 2721 2722 if (!found) 2723 return; 2724 2725 cmd = iscsit_allocate_cmd(conn_p, TASK_RUNNING); 2726 if (!cmd) { 2727 iscsit_dec_conn_usage_count(conn_p); 2728 return; 2729 } 2730 2731 cmd->logout_cid = conn->cid; 2732 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT; 2733 cmd->i_state = ISTATE_SEND_ASYNCMSG; 2734 2735 spin_lock_bh(&conn_p->cmd_lock); 2736 list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list); 2737 spin_unlock_bh(&conn_p->cmd_lock); 2738 2739 iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state); 2740 iscsit_dec_conn_usage_count(conn_p); 2741 } 2742 2743 static int iscsit_send_conn_drop_async_message( 2744 struct iscsi_cmd *cmd, 2745 struct iscsi_conn *conn) 2746 { 2747 struct iscsi_async *hdr; 2748 2749 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT; 2750 2751 hdr = (struct iscsi_async *) cmd->pdu; 2752 hdr->opcode = ISCSI_OP_ASYNC_EVENT; 2753 hdr->flags = ISCSI_FLAG_CMD_FINAL; 2754 cmd->init_task_tag = RESERVED_ITT; 2755 cmd->targ_xfer_tag = 0xFFFFFFFF; 2756 put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]); 2757 cmd->stat_sn = conn->stat_sn++; 2758 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2759 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2760 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 2761 hdr->async_event = ISCSI_ASYNC_MSG_DROPPING_CONNECTION; 2762 hdr->param1 = cpu_to_be16(cmd->logout_cid); 2763 hdr->param2 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait); 2764 hdr->param3 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain); 2765 2766 pr_debug("Sending Connection Dropped Async Message StatSN:" 2767 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn, 2768 cmd->logout_cid, conn->cid); 2769 2770 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 2771 } 2772 2773 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn) 2774 { 2775 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || 2776 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { 2777 wait_for_completion_interruptible_timeout( 2778 &conn->tx_half_close_comp, 2779 ISCSI_TX_THREAD_TCP_TIMEOUT * HZ); 2780 } 2781 } 2782 2783 void 2784 iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2785 struct iscsi_datain *datain, struct iscsi_data_rsp *hdr, 2786 bool set_statsn) 2787 { 2788 hdr->opcode = ISCSI_OP_SCSI_DATA_IN; 2789 hdr->flags = datain->flags; 2790 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) { 2791 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { 2792 hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW; 2793 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 2794 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { 2795 hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW; 2796 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 2797 } 2798 } 2799 hton24(hdr->dlength, datain->length); 2800 if (hdr->flags & ISCSI_FLAG_DATA_ACK) 2801 int_to_scsilun(cmd->se_cmd.orig_fe_lun, 2802 (struct scsi_lun *)&hdr->lun); 2803 else 2804 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); 2805 2806 hdr->itt = cmd->init_task_tag; 2807 2808 if (hdr->flags & ISCSI_FLAG_DATA_ACK) 2809 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 2810 else 2811 hdr->ttt = cpu_to_be32(0xFFFFFFFF); 2812 if (set_statsn) 2813 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2814 else 2815 hdr->statsn = cpu_to_be32(0xFFFFFFFF); 2816 2817 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2818 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 2819 hdr->datasn = cpu_to_be32(datain->data_sn); 2820 hdr->offset = cpu_to_be32(datain->offset); 2821 2822 pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x," 2823 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", 2824 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn), 2825 ntohl(hdr->offset), datain->length, conn->cid); 2826 } 2827 EXPORT_SYMBOL(iscsit_build_datain_pdu); 2828 2829 static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2830 { 2831 struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0]; 2832 struct iscsi_datain datain; 2833 struct iscsi_datain_req *dr; 2834 int eodr = 0, ret; 2835 bool set_statsn = false; 2836 2837 memset(&datain, 0, sizeof(struct iscsi_datain)); 2838 dr = iscsit_get_datain_values(cmd, &datain); 2839 if (!dr) { 2840 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n", 2841 cmd->init_task_tag); 2842 return -1; 2843 } 2844 /* 2845 * Be paranoid and double check the logic for now. 2846 */ 2847 if ((datain.offset + datain.length) > cmd->se_cmd.data_length) { 2848 pr_err("Command ITT: 0x%08x, datain.offset: %u and" 2849 " datain.length: %u exceeds cmd->data_length: %u\n", 2850 cmd->init_task_tag, datain.offset, datain.length, 2851 cmd->se_cmd.data_length); 2852 return -1; 2853 } 2854 2855 atomic_long_add(datain.length, &conn->sess->tx_data_octets); 2856 /* 2857 * Special case for successfully execution w/ both DATAIN 2858 * and Sense Data. 2859 */ 2860 if ((datain.flags & ISCSI_FLAG_DATA_STATUS) && 2861 (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)) 2862 datain.flags &= ~ISCSI_FLAG_DATA_STATUS; 2863 else { 2864 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) || 2865 (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) { 2866 iscsit_increment_maxcmdsn(cmd, conn->sess); 2867 cmd->stat_sn = conn->stat_sn++; 2868 set_statsn = true; 2869 } else if (dr->dr_complete == 2870 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY) 2871 set_statsn = true; 2872 } 2873 2874 iscsit_build_datain_pdu(cmd, conn, &datain, hdr, set_statsn); 2875 2876 ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, dr, &datain, 0); 2877 if (ret < 0) 2878 return ret; 2879 2880 if (dr->dr_complete) { 2881 eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ? 2882 2 : 1; 2883 iscsit_free_datain_req(cmd, dr); 2884 } 2885 2886 return eodr; 2887 } 2888 2889 int 2890 iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2891 struct iscsi_logout_rsp *hdr) 2892 { 2893 struct iscsi_conn *logout_conn = NULL; 2894 struct iscsi_conn_recovery *cr = NULL; 2895 struct iscsi_session *sess = conn->sess; 2896 /* 2897 * The actual shutting down of Sessions and/or Connections 2898 * for CLOSESESSION and CLOSECONNECTION Logout Requests 2899 * is done in scsi_logout_post_handler(). 2900 */ 2901 switch (cmd->logout_reason) { 2902 case ISCSI_LOGOUT_REASON_CLOSE_SESSION: 2903 pr_debug("iSCSI session logout successful, setting" 2904 " logout response to ISCSI_LOGOUT_SUCCESS.\n"); 2905 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2906 break; 2907 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: 2908 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND) 2909 break; 2910 /* 2911 * For CLOSECONNECTION logout requests carrying 2912 * a matching logout CID -> local CID, the reference 2913 * for the local CID will have been incremented in 2914 * iscsi_logout_closeconnection(). 2915 * 2916 * For CLOSECONNECTION logout requests carrying 2917 * a different CID than the connection it arrived 2918 * on, the connection responding to cmd->logout_cid 2919 * is stopped in iscsit_logout_post_handler_diffcid(). 2920 */ 2921 2922 pr_debug("iSCSI CID: %hu logout on CID: %hu" 2923 " successful.\n", cmd->logout_cid, conn->cid); 2924 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2925 break; 2926 case ISCSI_LOGOUT_REASON_RECOVERY: 2927 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) || 2928 (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED)) 2929 break; 2930 /* 2931 * If the connection is still active from our point of view 2932 * force connection recovery to occur. 2933 */ 2934 logout_conn = iscsit_get_conn_from_cid_rcfr(sess, 2935 cmd->logout_cid); 2936 if (logout_conn) { 2937 iscsit_connection_reinstatement_rcfr(logout_conn); 2938 iscsit_dec_conn_usage_count(logout_conn); 2939 } 2940 2941 cr = iscsit_get_inactive_connection_recovery_entry( 2942 conn->sess, cmd->logout_cid); 2943 if (!cr) { 2944 pr_err("Unable to locate CID: %hu for" 2945 " REMOVECONNFORRECOVERY Logout Request.\n", 2946 cmd->logout_cid); 2947 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND; 2948 break; 2949 } 2950 2951 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn); 2952 2953 pr_debug("iSCSI REMOVECONNFORRECOVERY logout" 2954 " for recovery for CID: %hu on CID: %hu successful.\n", 2955 cmd->logout_cid, conn->cid); 2956 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2957 break; 2958 default: 2959 pr_err("Unknown cmd->logout_reason: 0x%02x\n", 2960 cmd->logout_reason); 2961 return -1; 2962 } 2963 2964 hdr->opcode = ISCSI_OP_LOGOUT_RSP; 2965 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2966 hdr->response = cmd->logout_response; 2967 hdr->itt = cmd->init_task_tag; 2968 cmd->stat_sn = conn->stat_sn++; 2969 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2970 2971 iscsit_increment_maxcmdsn(cmd, conn->sess); 2972 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2973 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 2974 2975 pr_debug("Built Logout Response ITT: 0x%08x StatSN:" 2976 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n", 2977 cmd->init_task_tag, cmd->stat_sn, hdr->response, 2978 cmd->logout_cid, conn->cid); 2979 2980 return 0; 2981 } 2982 EXPORT_SYMBOL(iscsit_build_logout_rsp); 2983 2984 static int 2985 iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2986 { 2987 int rc; 2988 2989 rc = iscsit_build_logout_rsp(cmd, conn, 2990 (struct iscsi_logout_rsp *)&cmd->pdu[0]); 2991 if (rc < 0) 2992 return rc; 2993 2994 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 2995 } 2996 2997 void 2998 iscsit_build_nopin_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2999 struct iscsi_nopin *hdr, bool nopout_response) 3000 { 3001 hdr->opcode = ISCSI_OP_NOOP_IN; 3002 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3003 hton24(hdr->dlength, cmd->buf_ptr_size); 3004 if (nopout_response) 3005 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); 3006 hdr->itt = cmd->init_task_tag; 3007 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 3008 cmd->stat_sn = (nopout_response) ? conn->stat_sn++ : 3009 conn->stat_sn; 3010 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3011 3012 if (nopout_response) 3013 iscsit_increment_maxcmdsn(cmd, conn->sess); 3014 3015 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3016 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3017 3018 pr_debug("Built NOPIN %s Response ITT: 0x%08x, TTT: 0x%08x," 3019 " StatSN: 0x%08x, Length %u\n", (nopout_response) ? 3020 "Solicited" : "Unsolicited", cmd->init_task_tag, 3021 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size); 3022 } 3023 EXPORT_SYMBOL(iscsit_build_nopin_rsp); 3024 3025 /* 3026 * Unsolicited NOPIN, either requesting a response or not. 3027 */ 3028 static int iscsit_send_unsolicited_nopin( 3029 struct iscsi_cmd *cmd, 3030 struct iscsi_conn *conn, 3031 int want_response) 3032 { 3033 struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; 3034 int ret; 3035 3036 iscsit_build_nopin_rsp(cmd, conn, hdr, false); 3037 3038 pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:" 3039 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid); 3040 3041 ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 3042 if (ret < 0) 3043 return ret; 3044 3045 spin_lock_bh(&cmd->istate_lock); 3046 cmd->i_state = want_response ? 3047 ISTATE_SENT_NOPIN_WANT_RESPONSE : ISTATE_SENT_STATUS; 3048 spin_unlock_bh(&cmd->istate_lock); 3049 3050 return 0; 3051 } 3052 3053 static int 3054 iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3055 { 3056 struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; 3057 3058 iscsit_build_nopin_rsp(cmd, conn, hdr, true); 3059 3060 /* 3061 * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr. 3062 * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size. 3063 */ 3064 pr_debug("Echoing back %u bytes of ping data.\n", cmd->buf_ptr_size); 3065 3066 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 3067 cmd->buf_ptr, 3068 cmd->buf_ptr_size); 3069 } 3070 3071 static int iscsit_send_r2t( 3072 struct iscsi_cmd *cmd, 3073 struct iscsi_conn *conn) 3074 { 3075 struct iscsi_r2t *r2t; 3076 struct iscsi_r2t_rsp *hdr; 3077 int ret; 3078 3079 r2t = iscsit_get_r2t_from_list(cmd); 3080 if (!r2t) 3081 return -1; 3082 3083 hdr = (struct iscsi_r2t_rsp *) cmd->pdu; 3084 memset(hdr, 0, ISCSI_HDR_LEN); 3085 hdr->opcode = ISCSI_OP_R2T; 3086 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3087 int_to_scsilun(cmd->se_cmd.orig_fe_lun, 3088 (struct scsi_lun *)&hdr->lun); 3089 hdr->itt = cmd->init_task_tag; 3090 if (conn->conn_transport->iscsit_get_r2t_ttt) 3091 conn->conn_transport->iscsit_get_r2t_ttt(conn, cmd, r2t); 3092 else 3093 r2t->targ_xfer_tag = session_get_next_ttt(conn->sess); 3094 hdr->ttt = cpu_to_be32(r2t->targ_xfer_tag); 3095 hdr->statsn = cpu_to_be32(conn->stat_sn); 3096 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3097 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3098 hdr->r2tsn = cpu_to_be32(r2t->r2t_sn); 3099 hdr->data_offset = cpu_to_be32(r2t->offset); 3100 hdr->data_length = cpu_to_be32(r2t->xfer_len); 3101 3102 pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:" 3103 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n", 3104 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag, 3105 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn, 3106 r2t->offset, r2t->xfer_len, conn->cid); 3107 3108 spin_lock_bh(&cmd->r2t_lock); 3109 r2t->sent_r2t = 1; 3110 spin_unlock_bh(&cmd->r2t_lock); 3111 3112 ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 3113 if (ret < 0) { 3114 return ret; 3115 } 3116 3117 spin_lock_bh(&cmd->dataout_timeout_lock); 3118 iscsit_start_dataout_timer(cmd, conn); 3119 spin_unlock_bh(&cmd->dataout_timeout_lock); 3120 3121 return 0; 3122 } 3123 3124 /* 3125 * @recovery: If called from iscsi_task_reassign_complete_write() for 3126 * connection recovery. 3127 */ 3128 int iscsit_build_r2ts_for_cmd( 3129 struct iscsi_conn *conn, 3130 struct iscsi_cmd *cmd, 3131 bool recovery) 3132 { 3133 int first_r2t = 1; 3134 u32 offset = 0, xfer_len = 0; 3135 3136 spin_lock_bh(&cmd->r2t_lock); 3137 if (cmd->cmd_flags & ICF_SENT_LAST_R2T) { 3138 spin_unlock_bh(&cmd->r2t_lock); 3139 return 0; 3140 } 3141 3142 if (conn->sess->sess_ops->DataSequenceInOrder && 3143 !recovery) 3144 cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done); 3145 3146 while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) { 3147 if (conn->sess->sess_ops->DataSequenceInOrder) { 3148 offset = cmd->r2t_offset; 3149 3150 if (first_r2t && recovery) { 3151 int new_data_end = offset + 3152 conn->sess->sess_ops->MaxBurstLength - 3153 cmd->next_burst_len; 3154 3155 if (new_data_end > cmd->se_cmd.data_length) 3156 xfer_len = cmd->se_cmd.data_length - offset; 3157 else 3158 xfer_len = 3159 conn->sess->sess_ops->MaxBurstLength - 3160 cmd->next_burst_len; 3161 } else { 3162 int new_data_end = offset + 3163 conn->sess->sess_ops->MaxBurstLength; 3164 3165 if (new_data_end > cmd->se_cmd.data_length) 3166 xfer_len = cmd->se_cmd.data_length - offset; 3167 else 3168 xfer_len = conn->sess->sess_ops->MaxBurstLength; 3169 } 3170 3171 if ((s32)xfer_len < 0) { 3172 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 3173 break; 3174 } 3175 3176 cmd->r2t_offset += xfer_len; 3177 3178 if (cmd->r2t_offset == cmd->se_cmd.data_length) 3179 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 3180 } else { 3181 struct iscsi_seq *seq; 3182 3183 seq = iscsit_get_seq_holder_for_r2t(cmd); 3184 if (!seq) { 3185 spin_unlock_bh(&cmd->r2t_lock); 3186 return -1; 3187 } 3188 3189 offset = seq->offset; 3190 xfer_len = seq->xfer_len; 3191 3192 if (cmd->seq_send_order == cmd->seq_count) 3193 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 3194 } 3195 cmd->outstanding_r2ts++; 3196 first_r2t = 0; 3197 3198 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) { 3199 spin_unlock_bh(&cmd->r2t_lock); 3200 return -1; 3201 } 3202 3203 if (cmd->cmd_flags & ICF_SENT_LAST_R2T) 3204 break; 3205 } 3206 spin_unlock_bh(&cmd->r2t_lock); 3207 3208 return 0; 3209 } 3210 EXPORT_SYMBOL(iscsit_build_r2ts_for_cmd); 3211 3212 void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3213 bool inc_stat_sn, struct iscsi_scsi_rsp *hdr) 3214 { 3215 if (inc_stat_sn) 3216 cmd->stat_sn = conn->stat_sn++; 3217 3218 atomic_long_inc(&conn->sess->rsp_pdus); 3219 3220 memset(hdr, 0, ISCSI_HDR_LEN); 3221 hdr->opcode = ISCSI_OP_SCSI_CMD_RSP; 3222 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3223 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { 3224 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW; 3225 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 3226 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { 3227 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW; 3228 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 3229 } 3230 hdr->response = cmd->iscsi_response; 3231 hdr->cmd_status = cmd->se_cmd.scsi_status; 3232 hdr->itt = cmd->init_task_tag; 3233 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3234 3235 iscsit_increment_maxcmdsn(cmd, conn->sess); 3236 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3237 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3238 3239 pr_debug("Built SCSI Response, ITT: 0x%08x, StatSN: 0x%08x," 3240 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n", 3241 cmd->init_task_tag, cmd->stat_sn, cmd->se_cmd.scsi_status, 3242 cmd->se_cmd.scsi_status, conn->cid); 3243 } 3244 EXPORT_SYMBOL(iscsit_build_rsp_pdu); 3245 3246 static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3247 { 3248 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0]; 3249 bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS); 3250 void *data_buf = NULL; 3251 u32 padding = 0, data_buf_len = 0; 3252 3253 iscsit_build_rsp_pdu(cmd, conn, inc_stat_sn, hdr); 3254 3255 /* 3256 * Attach SENSE DATA payload to iSCSI Response PDU 3257 */ 3258 if (cmd->se_cmd.sense_buffer && 3259 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 3260 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 3261 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, cmd->sense_buffer); 3262 cmd->se_cmd.scsi_sense_length += sizeof (__be16); 3263 3264 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 3265 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 3266 data_buf = cmd->sense_buffer; 3267 data_buf_len = cmd->se_cmd.scsi_sense_length + padding; 3268 3269 if (padding) { 3270 memset(cmd->sense_buffer + 3271 cmd->se_cmd.scsi_sense_length, 0, padding); 3272 pr_debug("Adding %u bytes of padding to" 3273 " SENSE.\n", padding); 3274 } 3275 3276 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI" 3277 " Response PDU\n", 3278 cmd->se_cmd.scsi_sense_length); 3279 } 3280 3281 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, data_buf, 3282 data_buf_len); 3283 } 3284 3285 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr) 3286 { 3287 switch (se_tmr->response) { 3288 case TMR_FUNCTION_COMPLETE: 3289 return ISCSI_TMF_RSP_COMPLETE; 3290 case TMR_TASK_DOES_NOT_EXIST: 3291 return ISCSI_TMF_RSP_NO_TASK; 3292 case TMR_LUN_DOES_NOT_EXIST: 3293 return ISCSI_TMF_RSP_NO_LUN; 3294 case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED: 3295 return ISCSI_TMF_RSP_NOT_SUPPORTED; 3296 case TMR_FUNCTION_REJECTED: 3297 default: 3298 return ISCSI_TMF_RSP_REJECTED; 3299 } 3300 } 3301 3302 void 3303 iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3304 struct iscsi_tm_rsp *hdr) 3305 { 3306 struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req; 3307 3308 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP; 3309 hdr->flags = ISCSI_FLAG_CMD_FINAL; 3310 hdr->response = iscsit_convert_tcm_tmr_rsp(se_tmr); 3311 hdr->itt = cmd->init_task_tag; 3312 cmd->stat_sn = conn->stat_sn++; 3313 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3314 3315 iscsit_increment_maxcmdsn(cmd, conn->sess); 3316 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3317 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3318 3319 pr_debug("Built Task Management Response ITT: 0x%08x," 3320 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n", 3321 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid); 3322 } 3323 EXPORT_SYMBOL(iscsit_build_task_mgt_rsp); 3324 3325 static int 3326 iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 3327 { 3328 struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0]; 3329 3330 iscsit_build_task_mgt_rsp(cmd, conn, hdr); 3331 3332 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); 3333 } 3334 3335 #define SENDTARGETS_BUF_LIMIT 32768U 3336 3337 static int 3338 iscsit_build_sendtargets_response(struct iscsi_cmd *cmd, 3339 enum iscsit_transport_type network_transport, 3340 int skip_bytes, bool *completed) 3341 { 3342 char *payload = NULL; 3343 struct iscsi_conn *conn = cmd->conn; 3344 struct iscsi_portal_group *tpg; 3345 struct iscsi_tiqn *tiqn; 3346 struct iscsi_tpg_np *tpg_np; 3347 int buffer_len, end_of_buf = 0, len = 0, payload_len = 0; 3348 int target_name_printed; 3349 unsigned char buf[ISCSI_IQN_LEN+12]; /* iqn + "TargetName=" + \0 */ 3350 unsigned char *text_in = cmd->text_in_ptr, *text_ptr = NULL; 3351 bool active; 3352 3353 buffer_len = min(conn->conn_ops->MaxRecvDataSegmentLength, 3354 SENDTARGETS_BUF_LIMIT); 3355 3356 payload = kzalloc(buffer_len, GFP_KERNEL); 3357 if (!payload) 3358 return -ENOMEM; 3359 3360 /* 3361 * Locate pointer to iqn./eui. string for ICF_SENDTARGETS_SINGLE 3362 * explicit case.. 3363 */ 3364 if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) { 3365 text_ptr = strchr(text_in, '='); 3366 if (!text_ptr) { 3367 pr_err("Unable to locate '=' string in text_in:" 3368 " %s\n", text_in); 3369 kfree(payload); 3370 return -EINVAL; 3371 } 3372 /* 3373 * Skip over '=' character.. 3374 */ 3375 text_ptr += 1; 3376 } 3377 3378 spin_lock(&tiqn_lock); 3379 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) { 3380 if ((cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) && 3381 strcmp(tiqn->tiqn, text_ptr)) { 3382 continue; 3383 } 3384 3385 target_name_printed = 0; 3386 3387 spin_lock(&tiqn->tiqn_tpg_lock); 3388 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) { 3389 3390 /* If demo_mode_discovery=0 and generate_node_acls=0 3391 * (demo mode dislabed) do not return 3392 * TargetName+TargetAddress unless a NodeACL exists. 3393 */ 3394 3395 if ((tpg->tpg_attrib.generate_node_acls == 0) && 3396 (tpg->tpg_attrib.demo_mode_discovery == 0) && 3397 (!target_tpg_has_node_acl(&tpg->tpg_se_tpg, 3398 cmd->conn->sess->sess_ops->InitiatorName))) { 3399 continue; 3400 } 3401 3402 spin_lock(&tpg->tpg_state_lock); 3403 active = (tpg->tpg_state == TPG_STATE_ACTIVE); 3404 spin_unlock(&tpg->tpg_state_lock); 3405 3406 if (!active && tpg->tpg_attrib.tpg_enabled_sendtargets) 3407 continue; 3408 3409 spin_lock(&tpg->tpg_np_lock); 3410 list_for_each_entry(tpg_np, &tpg->tpg_gnp_list, 3411 tpg_np_list) { 3412 struct iscsi_np *np = tpg_np->tpg_np; 3413 struct sockaddr_storage *sockaddr; 3414 3415 if (np->np_network_transport != network_transport) 3416 continue; 3417 3418 if (!target_name_printed) { 3419 len = sprintf(buf, "TargetName=%s", 3420 tiqn->tiqn); 3421 len += 1; 3422 3423 if ((len + payload_len) > buffer_len) { 3424 spin_unlock(&tpg->tpg_np_lock); 3425 spin_unlock(&tiqn->tiqn_tpg_lock); 3426 end_of_buf = 1; 3427 goto eob; 3428 } 3429 3430 if (skip_bytes && len <= skip_bytes) { 3431 skip_bytes -= len; 3432 } else { 3433 memcpy(payload + payload_len, buf, len); 3434 payload_len += len; 3435 target_name_printed = 1; 3436 if (len > skip_bytes) 3437 skip_bytes = 0; 3438 } 3439 } 3440 3441 if (inet_addr_is_any((struct sockaddr *)&np->np_sockaddr)) 3442 sockaddr = &conn->local_sockaddr; 3443 else 3444 sockaddr = &np->np_sockaddr; 3445 3446 len = sprintf(buf, "TargetAddress=" 3447 "%pISpc,%hu", 3448 sockaddr, 3449 tpg->tpgt); 3450 len += 1; 3451 3452 if ((len + payload_len) > buffer_len) { 3453 spin_unlock(&tpg->tpg_np_lock); 3454 spin_unlock(&tiqn->tiqn_tpg_lock); 3455 end_of_buf = 1; 3456 goto eob; 3457 } 3458 3459 if (skip_bytes && len <= skip_bytes) { 3460 skip_bytes -= len; 3461 } else { 3462 memcpy(payload + payload_len, buf, len); 3463 payload_len += len; 3464 if (len > skip_bytes) 3465 skip_bytes = 0; 3466 } 3467 } 3468 spin_unlock(&tpg->tpg_np_lock); 3469 } 3470 spin_unlock(&tiqn->tiqn_tpg_lock); 3471 eob: 3472 if (end_of_buf) { 3473 *completed = false; 3474 break; 3475 } 3476 3477 if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) 3478 break; 3479 } 3480 spin_unlock(&tiqn_lock); 3481 3482 cmd->buf_ptr = payload; 3483 3484 return payload_len; 3485 } 3486 3487 int 3488 iscsit_build_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3489 struct iscsi_text_rsp *hdr, 3490 enum iscsit_transport_type network_transport) 3491 { 3492 int text_length, padding; 3493 bool completed = true; 3494 3495 text_length = iscsit_build_sendtargets_response(cmd, network_transport, 3496 cmd->read_data_done, 3497 &completed); 3498 if (text_length < 0) 3499 return text_length; 3500 3501 if (completed) { 3502 hdr->flags = ISCSI_FLAG_CMD_FINAL; 3503 } else { 3504 hdr->flags = ISCSI_FLAG_TEXT_CONTINUE; 3505 cmd->read_data_done += text_length; 3506 if (cmd->targ_xfer_tag == 0xFFFFFFFF) 3507 cmd->targ_xfer_tag = session_get_next_ttt(conn->sess); 3508 } 3509 hdr->opcode = ISCSI_OP_TEXT_RSP; 3510 padding = ((-text_length) & 3); 3511 hton24(hdr->dlength, text_length); 3512 hdr->itt = cmd->init_task_tag; 3513 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 3514 cmd->stat_sn = conn->stat_sn++; 3515 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3516 3517 iscsit_increment_maxcmdsn(cmd, conn->sess); 3518 /* 3519 * Reset maxcmdsn_inc in multi-part text payload exchanges to 3520 * correctly increment MaxCmdSN for each response answering a 3521 * non immediate text request with a valid CmdSN. 3522 */ 3523 cmd->maxcmdsn_inc = 0; 3524 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3525 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3526 3527 pr_debug("Built Text Response: ITT: 0x%08x, TTT: 0x%08x, StatSN: 0x%08x," 3528 " Length: %u, CID: %hu F: %d C: %d\n", cmd->init_task_tag, 3529 cmd->targ_xfer_tag, cmd->stat_sn, text_length, conn->cid, 3530 !!(hdr->flags & ISCSI_FLAG_CMD_FINAL), 3531 !!(hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)); 3532 3533 return text_length + padding; 3534 } 3535 EXPORT_SYMBOL(iscsit_build_text_rsp); 3536 3537 static int iscsit_send_text_rsp( 3538 struct iscsi_cmd *cmd, 3539 struct iscsi_conn *conn) 3540 { 3541 struct iscsi_text_rsp *hdr = (struct iscsi_text_rsp *)cmd->pdu; 3542 int text_length; 3543 3544 text_length = iscsit_build_text_rsp(cmd, conn, hdr, 3545 conn->conn_transport->transport_type); 3546 if (text_length < 0) 3547 return text_length; 3548 3549 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 3550 cmd->buf_ptr, 3551 text_length); 3552 } 3553 3554 void 3555 iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 3556 struct iscsi_reject *hdr) 3557 { 3558 hdr->opcode = ISCSI_OP_REJECT; 3559 hdr->reason = cmd->reject_reason; 3560 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3561 hton24(hdr->dlength, ISCSI_HDR_LEN); 3562 hdr->ffffffff = cpu_to_be32(0xffffffff); 3563 cmd->stat_sn = conn->stat_sn++; 3564 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3565 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3566 hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn)); 3567 3568 } 3569 EXPORT_SYMBOL(iscsit_build_reject); 3570 3571 static int iscsit_send_reject( 3572 struct iscsi_cmd *cmd, 3573 struct iscsi_conn *conn) 3574 { 3575 struct iscsi_reject *hdr = (struct iscsi_reject *)&cmd->pdu[0]; 3576 3577 iscsit_build_reject(cmd, conn, hdr); 3578 3579 pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x," 3580 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid); 3581 3582 return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, 3583 cmd->buf_ptr, 3584 ISCSI_HDR_LEN); 3585 } 3586 3587 void iscsit_thread_get_cpumask(struct iscsi_conn *conn) 3588 { 3589 int ord, cpu; 3590 /* 3591 * bitmap_id is assigned from iscsit_global->ts_bitmap from 3592 * within iscsit_start_kthreads() 3593 * 3594 * Here we use bitmap_id to determine which CPU that this 3595 * iSCSI connection's RX/TX threads will be scheduled to 3596 * execute upon. 3597 */ 3598 ord = conn->bitmap_id % cpumask_weight(cpu_online_mask); 3599 for_each_online_cpu(cpu) { 3600 if (ord-- == 0) { 3601 cpumask_set_cpu(cpu, conn->conn_cpumask); 3602 return; 3603 } 3604 } 3605 /* 3606 * This should never be reached.. 3607 */ 3608 dump_stack(); 3609 cpumask_setall(conn->conn_cpumask); 3610 } 3611 3612 int 3613 iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3614 { 3615 int ret; 3616 3617 switch (state) { 3618 case ISTATE_SEND_R2T: 3619 ret = iscsit_send_r2t(cmd, conn); 3620 if (ret < 0) 3621 goto err; 3622 break; 3623 case ISTATE_REMOVE: 3624 spin_lock_bh(&conn->cmd_lock); 3625 list_del_init(&cmd->i_conn_node); 3626 spin_unlock_bh(&conn->cmd_lock); 3627 3628 iscsit_free_cmd(cmd, false); 3629 break; 3630 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 3631 iscsit_mod_nopin_response_timer(conn); 3632 ret = iscsit_send_unsolicited_nopin(cmd, conn, 1); 3633 if (ret < 0) 3634 goto err; 3635 break; 3636 case ISTATE_SEND_NOPIN_NO_RESPONSE: 3637 ret = iscsit_send_unsolicited_nopin(cmd, conn, 0); 3638 if (ret < 0) 3639 goto err; 3640 break; 3641 default: 3642 pr_err("Unknown Opcode: 0x%02x ITT:" 3643 " 0x%08x, i_state: %d on CID: %hu\n", 3644 cmd->iscsi_opcode, cmd->init_task_tag, state, 3645 conn->cid); 3646 goto err; 3647 } 3648 3649 return 0; 3650 3651 err: 3652 return -1; 3653 } 3654 EXPORT_SYMBOL(iscsit_immediate_queue); 3655 3656 static int 3657 iscsit_handle_immediate_queue(struct iscsi_conn *conn) 3658 { 3659 struct iscsit_transport *t = conn->conn_transport; 3660 struct iscsi_queue_req *qr; 3661 struct iscsi_cmd *cmd; 3662 u8 state; 3663 int ret; 3664 3665 while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) { 3666 atomic_set(&conn->check_immediate_queue, 0); 3667 cmd = qr->cmd; 3668 state = qr->state; 3669 kmem_cache_free(lio_qr_cache, qr); 3670 3671 ret = t->iscsit_immediate_queue(conn, cmd, state); 3672 if (ret < 0) 3673 return ret; 3674 } 3675 3676 return 0; 3677 } 3678 3679 int 3680 iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3681 { 3682 int ret; 3683 3684 check_rsp_state: 3685 switch (state) { 3686 case ISTATE_SEND_DATAIN: 3687 ret = iscsit_send_datain(cmd, conn); 3688 if (ret < 0) 3689 goto err; 3690 else if (!ret) 3691 /* more drs */ 3692 goto check_rsp_state; 3693 else if (ret == 1) { 3694 /* all done */ 3695 spin_lock_bh(&cmd->istate_lock); 3696 cmd->i_state = ISTATE_SENT_STATUS; 3697 spin_unlock_bh(&cmd->istate_lock); 3698 3699 if (atomic_read(&conn->check_immediate_queue)) 3700 return 1; 3701 3702 return 0; 3703 } else if (ret == 2) { 3704 /* Still must send status, 3705 SCF_TRANSPORT_TASK_SENSE was set */ 3706 spin_lock_bh(&cmd->istate_lock); 3707 cmd->i_state = ISTATE_SEND_STATUS; 3708 spin_unlock_bh(&cmd->istate_lock); 3709 state = ISTATE_SEND_STATUS; 3710 goto check_rsp_state; 3711 } 3712 3713 break; 3714 case ISTATE_SEND_STATUS: 3715 case ISTATE_SEND_STATUS_RECOVERY: 3716 ret = iscsit_send_response(cmd, conn); 3717 break; 3718 case ISTATE_SEND_LOGOUTRSP: 3719 ret = iscsit_send_logout(cmd, conn); 3720 break; 3721 case ISTATE_SEND_ASYNCMSG: 3722 ret = iscsit_send_conn_drop_async_message( 3723 cmd, conn); 3724 break; 3725 case ISTATE_SEND_NOPIN: 3726 ret = iscsit_send_nopin(cmd, conn); 3727 break; 3728 case ISTATE_SEND_REJECT: 3729 ret = iscsit_send_reject(cmd, conn); 3730 break; 3731 case ISTATE_SEND_TASKMGTRSP: 3732 ret = iscsit_send_task_mgt_rsp(cmd, conn); 3733 if (ret != 0) 3734 break; 3735 ret = iscsit_tmr_post_handler(cmd, conn); 3736 if (ret != 0) 3737 iscsit_fall_back_to_erl0(conn->sess); 3738 break; 3739 case ISTATE_SEND_TEXTRSP: 3740 ret = iscsit_send_text_rsp(cmd, conn); 3741 break; 3742 default: 3743 pr_err("Unknown Opcode: 0x%02x ITT:" 3744 " 0x%08x, i_state: %d on CID: %hu\n", 3745 cmd->iscsi_opcode, cmd->init_task_tag, 3746 state, conn->cid); 3747 goto err; 3748 } 3749 if (ret < 0) 3750 goto err; 3751 3752 switch (state) { 3753 case ISTATE_SEND_LOGOUTRSP: 3754 if (!iscsit_logout_post_handler(cmd, conn)) 3755 return -ECONNRESET; 3756 fallthrough; 3757 case ISTATE_SEND_STATUS: 3758 case ISTATE_SEND_ASYNCMSG: 3759 case ISTATE_SEND_NOPIN: 3760 case ISTATE_SEND_STATUS_RECOVERY: 3761 case ISTATE_SEND_TEXTRSP: 3762 case ISTATE_SEND_TASKMGTRSP: 3763 case ISTATE_SEND_REJECT: 3764 spin_lock_bh(&cmd->istate_lock); 3765 cmd->i_state = ISTATE_SENT_STATUS; 3766 spin_unlock_bh(&cmd->istate_lock); 3767 break; 3768 default: 3769 pr_err("Unknown Opcode: 0x%02x ITT:" 3770 " 0x%08x, i_state: %d on CID: %hu\n", 3771 cmd->iscsi_opcode, cmd->init_task_tag, 3772 cmd->i_state, conn->cid); 3773 goto err; 3774 } 3775 3776 if (atomic_read(&conn->check_immediate_queue)) 3777 return 1; 3778 3779 return 0; 3780 3781 err: 3782 return -1; 3783 } 3784 EXPORT_SYMBOL(iscsit_response_queue); 3785 3786 static int iscsit_handle_response_queue(struct iscsi_conn *conn) 3787 { 3788 struct iscsit_transport *t = conn->conn_transport; 3789 struct iscsi_queue_req *qr; 3790 struct iscsi_cmd *cmd; 3791 u8 state; 3792 int ret; 3793 3794 while ((qr = iscsit_get_cmd_from_response_queue(conn))) { 3795 cmd = qr->cmd; 3796 state = qr->state; 3797 kmem_cache_free(lio_qr_cache, qr); 3798 3799 ret = t->iscsit_response_queue(conn, cmd, state); 3800 if (ret == 1 || ret < 0) 3801 return ret; 3802 } 3803 3804 return 0; 3805 } 3806 3807 int iscsi_target_tx_thread(void *arg) 3808 { 3809 int ret = 0; 3810 struct iscsi_conn *conn = arg; 3811 bool conn_freed = false; 3812 3813 /* 3814 * Allow ourselves to be interrupted by SIGINT so that a 3815 * connection recovery / failure event can be triggered externally. 3816 */ 3817 allow_signal(SIGINT); 3818 3819 while (!kthread_should_stop()) { 3820 /* 3821 * Ensure that both TX and RX per connection kthreads 3822 * are scheduled to run on the same CPU. 3823 */ 3824 iscsit_thread_check_cpumask(conn, current, 1); 3825 3826 wait_event_interruptible(conn->queues_wq, 3827 !iscsit_conn_all_queues_empty(conn)); 3828 3829 if (signal_pending(current)) 3830 goto transport_err; 3831 3832 get_immediate: 3833 ret = iscsit_handle_immediate_queue(conn); 3834 if (ret < 0) 3835 goto transport_err; 3836 3837 ret = iscsit_handle_response_queue(conn); 3838 if (ret == 1) { 3839 goto get_immediate; 3840 } else if (ret == -ECONNRESET) { 3841 conn_freed = true; 3842 goto out; 3843 } else if (ret < 0) { 3844 goto transport_err; 3845 } 3846 } 3847 3848 transport_err: 3849 /* 3850 * Avoid the normal connection failure code-path if this connection 3851 * is still within LOGIN mode, and iscsi_np process context is 3852 * responsible for cleaning up the early connection failure. 3853 */ 3854 if (conn->conn_state != TARG_CONN_STATE_IN_LOGIN) 3855 iscsit_take_action_for_connection_exit(conn, &conn_freed); 3856 out: 3857 if (!conn_freed) { 3858 while (!kthread_should_stop()) { 3859 msleep(100); 3860 } 3861 } 3862 return 0; 3863 } 3864 3865 static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf) 3866 { 3867 struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf; 3868 struct iscsi_cmd *cmd; 3869 int ret = 0; 3870 3871 switch (hdr->opcode & ISCSI_OPCODE_MASK) { 3872 case ISCSI_OP_SCSI_CMD: 3873 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3874 if (!cmd) 3875 goto reject; 3876 3877 ret = iscsit_handle_scsi_cmd(conn, cmd, buf); 3878 break; 3879 case ISCSI_OP_SCSI_DATA_OUT: 3880 ret = iscsit_handle_data_out(conn, buf); 3881 break; 3882 case ISCSI_OP_NOOP_OUT: 3883 cmd = NULL; 3884 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 3885 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3886 if (!cmd) 3887 goto reject; 3888 } 3889 ret = iscsit_handle_nop_out(conn, cmd, buf); 3890 break; 3891 case ISCSI_OP_SCSI_TMFUNC: 3892 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3893 if (!cmd) 3894 goto reject; 3895 3896 ret = iscsit_handle_task_mgt_cmd(conn, cmd, buf); 3897 break; 3898 case ISCSI_OP_TEXT: 3899 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 3900 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 3901 if (!cmd) 3902 goto reject; 3903 } else { 3904 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3905 if (!cmd) 3906 goto reject; 3907 } 3908 3909 ret = iscsit_handle_text_cmd(conn, cmd, buf); 3910 break; 3911 case ISCSI_OP_LOGOUT: 3912 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 3913 if (!cmd) 3914 goto reject; 3915 3916 ret = iscsit_handle_logout_cmd(conn, cmd, buf); 3917 if (ret > 0) 3918 wait_for_completion_timeout(&conn->conn_logout_comp, 3919 SECONDS_FOR_LOGOUT_COMP * HZ); 3920 break; 3921 case ISCSI_OP_SNACK: 3922 ret = iscsit_handle_snack(conn, buf); 3923 break; 3924 default: 3925 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", hdr->opcode); 3926 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 3927 pr_err("Cannot recover from unknown" 3928 " opcode while ERL=0, closing iSCSI connection.\n"); 3929 return -1; 3930 } 3931 pr_err("Unable to recover from unknown opcode while OFMarker=No," 3932 " closing iSCSI connection.\n"); 3933 ret = -1; 3934 break; 3935 } 3936 3937 return ret; 3938 reject: 3939 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); 3940 } 3941 3942 static bool iscsi_target_check_conn_state(struct iscsi_conn *conn) 3943 { 3944 bool ret; 3945 3946 spin_lock_bh(&conn->state_lock); 3947 ret = (conn->conn_state != TARG_CONN_STATE_LOGGED_IN); 3948 spin_unlock_bh(&conn->state_lock); 3949 3950 return ret; 3951 } 3952 3953 static void iscsit_get_rx_pdu(struct iscsi_conn *conn) 3954 { 3955 int ret; 3956 u8 *buffer, opcode; 3957 u32 checksum = 0, digest = 0; 3958 struct kvec iov; 3959 3960 buffer = kcalloc(ISCSI_HDR_LEN, sizeof(*buffer), GFP_KERNEL); 3961 if (!buffer) 3962 return; 3963 3964 while (!kthread_should_stop()) { 3965 /* 3966 * Ensure that both TX and RX per connection kthreads 3967 * are scheduled to run on the same CPU. 3968 */ 3969 iscsit_thread_check_cpumask(conn, current, 0); 3970 3971 memset(&iov, 0, sizeof(struct kvec)); 3972 3973 iov.iov_base = buffer; 3974 iov.iov_len = ISCSI_HDR_LEN; 3975 3976 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN); 3977 if (ret != ISCSI_HDR_LEN) { 3978 iscsit_rx_thread_wait_for_tcp(conn); 3979 break; 3980 } 3981 3982 if (conn->conn_ops->HeaderDigest) { 3983 iov.iov_base = &digest; 3984 iov.iov_len = ISCSI_CRC_LEN; 3985 3986 ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN); 3987 if (ret != ISCSI_CRC_LEN) { 3988 iscsit_rx_thread_wait_for_tcp(conn); 3989 break; 3990 } 3991 3992 iscsit_do_crypto_hash_buf(conn->conn_rx_hash, buffer, 3993 ISCSI_HDR_LEN, 0, NULL, 3994 &checksum); 3995 3996 if (digest != checksum) { 3997 pr_err("HeaderDigest CRC32C failed," 3998 " received 0x%08x, computed 0x%08x\n", 3999 digest, checksum); 4000 /* 4001 * Set the PDU to 0xff so it will intentionally 4002 * hit default in the switch below. 4003 */ 4004 memset(buffer, 0xff, ISCSI_HDR_LEN); 4005 atomic_long_inc(&conn->sess->conn_digest_errors); 4006 } else { 4007 pr_debug("Got HeaderDigest CRC32C" 4008 " 0x%08x\n", checksum); 4009 } 4010 } 4011 4012 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 4013 break; 4014 4015 opcode = buffer[0] & ISCSI_OPCODE_MASK; 4016 4017 if (conn->sess->sess_ops->SessionType && 4018 ((!(opcode & ISCSI_OP_TEXT)) || 4019 (!(opcode & ISCSI_OP_LOGOUT)))) { 4020 pr_err("Received illegal iSCSI Opcode: 0x%02x" 4021 " while in Discovery Session, rejecting.\n", opcode); 4022 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 4023 buffer); 4024 break; 4025 } 4026 4027 ret = iscsi_target_rx_opcode(conn, buffer); 4028 if (ret < 0) 4029 break; 4030 } 4031 4032 kfree(buffer); 4033 } 4034 4035 int iscsi_target_rx_thread(void *arg) 4036 { 4037 int rc; 4038 struct iscsi_conn *conn = arg; 4039 bool conn_freed = false; 4040 4041 /* 4042 * Allow ourselves to be interrupted by SIGINT so that a 4043 * connection recovery / failure event can be triggered externally. 4044 */ 4045 allow_signal(SIGINT); 4046 /* 4047 * Wait for iscsi_post_login_handler() to complete before allowing 4048 * incoming iscsi/tcp socket I/O, and/or failing the connection. 4049 */ 4050 rc = wait_for_completion_interruptible(&conn->rx_login_comp); 4051 if (rc < 0 || iscsi_target_check_conn_state(conn)) 4052 goto out; 4053 4054 if (!conn->conn_transport->iscsit_get_rx_pdu) 4055 return 0; 4056 4057 conn->conn_transport->iscsit_get_rx_pdu(conn); 4058 4059 if (!signal_pending(current)) 4060 atomic_set(&conn->transport_failed, 1); 4061 iscsit_take_action_for_connection_exit(conn, &conn_freed); 4062 4063 out: 4064 if (!conn_freed) { 4065 while (!kthread_should_stop()) { 4066 msleep(100); 4067 } 4068 } 4069 4070 return 0; 4071 } 4072 4073 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn) 4074 { 4075 LIST_HEAD(tmp_list); 4076 struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL; 4077 struct iscsi_session *sess = conn->sess; 4078 /* 4079 * We expect this function to only ever be called from either RX or TX 4080 * thread context via iscsit_close_connection() once the other context 4081 * has been reset -> returned sleeping pre-handler state. 4082 */ 4083 spin_lock_bh(&conn->cmd_lock); 4084 list_splice_init(&conn->conn_cmd_list, &tmp_list); 4085 4086 list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) { 4087 struct se_cmd *se_cmd = &cmd->se_cmd; 4088 4089 if (se_cmd->se_tfo != NULL) { 4090 spin_lock_irq(&se_cmd->t_state_lock); 4091 if (se_cmd->transport_state & CMD_T_ABORTED) { 4092 /* 4093 * LIO's abort path owns the cleanup for this, 4094 * so put it back on the list and let 4095 * aborted_task handle it. 4096 */ 4097 list_move_tail(&cmd->i_conn_node, 4098 &conn->conn_cmd_list); 4099 } else { 4100 se_cmd->transport_state |= CMD_T_FABRIC_STOP; 4101 } 4102 spin_unlock_irq(&se_cmd->t_state_lock); 4103 } 4104 } 4105 spin_unlock_bh(&conn->cmd_lock); 4106 4107 list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) { 4108 list_del_init(&cmd->i_conn_node); 4109 4110 iscsit_increment_maxcmdsn(cmd, sess); 4111 iscsit_free_cmd(cmd, true); 4112 4113 } 4114 } 4115 4116 static void iscsit_stop_timers_for_cmds( 4117 struct iscsi_conn *conn) 4118 { 4119 struct iscsi_cmd *cmd; 4120 4121 spin_lock_bh(&conn->cmd_lock); 4122 list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) { 4123 if (cmd->data_direction == DMA_TO_DEVICE) 4124 iscsit_stop_dataout_timer(cmd); 4125 } 4126 spin_unlock_bh(&conn->cmd_lock); 4127 } 4128 4129 int iscsit_close_connection( 4130 struct iscsi_conn *conn) 4131 { 4132 int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT); 4133 struct iscsi_session *sess = conn->sess; 4134 4135 pr_debug("Closing iSCSI connection CID %hu on SID:" 4136 " %u\n", conn->cid, sess->sid); 4137 /* 4138 * Always up conn_logout_comp for the traditional TCP and HW_OFFLOAD 4139 * case just in case the RX Thread in iscsi_target_rx_opcode() is 4140 * sleeping and the logout response never got sent because the 4141 * connection failed. 4142 * 4143 * However for iser-target, isert_wait4logout() is using conn_logout_comp 4144 * to signal logout response TX interrupt completion. Go ahead and skip 4145 * this for iser since isert_rx_opcode() does not wait on logout failure, 4146 * and to avoid iscsi_conn pointer dereference in iser-target code. 4147 */ 4148 if (!conn->conn_transport->rdma_shutdown) 4149 complete(&conn->conn_logout_comp); 4150 4151 if (!strcmp(current->comm, ISCSI_RX_THREAD_NAME)) { 4152 if (conn->tx_thread && 4153 cmpxchg(&conn->tx_thread_active, true, false)) { 4154 send_sig(SIGINT, conn->tx_thread, 1); 4155 kthread_stop(conn->tx_thread); 4156 } 4157 } else if (!strcmp(current->comm, ISCSI_TX_THREAD_NAME)) { 4158 if (conn->rx_thread && 4159 cmpxchg(&conn->rx_thread_active, true, false)) { 4160 send_sig(SIGINT, conn->rx_thread, 1); 4161 kthread_stop(conn->rx_thread); 4162 } 4163 } 4164 4165 spin_lock(&iscsit_global->ts_bitmap_lock); 4166 bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id, 4167 get_order(1)); 4168 spin_unlock(&iscsit_global->ts_bitmap_lock); 4169 4170 iscsit_stop_timers_for_cmds(conn); 4171 iscsit_stop_nopin_response_timer(conn); 4172 iscsit_stop_nopin_timer(conn); 4173 4174 if (conn->conn_transport->iscsit_wait_conn) 4175 conn->conn_transport->iscsit_wait_conn(conn); 4176 4177 /* 4178 * During Connection recovery drop unacknowledged out of order 4179 * commands for this connection, and prepare the other commands 4180 * for reallegiance. 4181 * 4182 * During normal operation clear the out of order commands (but 4183 * do not free the struct iscsi_ooo_cmdsn's) and release all 4184 * struct iscsi_cmds. 4185 */ 4186 if (atomic_read(&conn->connection_recovery)) { 4187 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn); 4188 iscsit_prepare_cmds_for_reallegiance(conn); 4189 } else { 4190 iscsit_clear_ooo_cmdsns_for_conn(conn); 4191 iscsit_release_commands_from_conn(conn); 4192 } 4193 iscsit_free_queue_reqs_for_conn(conn); 4194 4195 /* 4196 * Handle decrementing session or connection usage count if 4197 * a logout response was not able to be sent because the 4198 * connection failed. Fall back to Session Recovery here. 4199 */ 4200 if (atomic_read(&conn->conn_logout_remove)) { 4201 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) { 4202 iscsit_dec_conn_usage_count(conn); 4203 iscsit_dec_session_usage_count(sess); 4204 } 4205 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) 4206 iscsit_dec_conn_usage_count(conn); 4207 4208 atomic_set(&conn->conn_logout_remove, 0); 4209 atomic_set(&sess->session_reinstatement, 0); 4210 atomic_set(&sess->session_fall_back_to_erl0, 1); 4211 } 4212 4213 spin_lock_bh(&sess->conn_lock); 4214 list_del(&conn->conn_list); 4215 4216 /* 4217 * Attempt to let the Initiator know this connection failed by 4218 * sending an Connection Dropped Async Message on another 4219 * active connection. 4220 */ 4221 if (atomic_read(&conn->connection_recovery)) 4222 iscsit_build_conn_drop_async_message(conn); 4223 4224 spin_unlock_bh(&sess->conn_lock); 4225 4226 /* 4227 * If connection reinstatement is being performed on this connection, 4228 * up the connection reinstatement semaphore that is being blocked on 4229 * in iscsit_cause_connection_reinstatement(). 4230 */ 4231 spin_lock_bh(&conn->state_lock); 4232 if (atomic_read(&conn->sleep_on_conn_wait_comp)) { 4233 spin_unlock_bh(&conn->state_lock); 4234 complete(&conn->conn_wait_comp); 4235 wait_for_completion(&conn->conn_post_wait_comp); 4236 spin_lock_bh(&conn->state_lock); 4237 } 4238 4239 /* 4240 * If connection reinstatement is being performed on this connection 4241 * by receiving a REMOVECONNFORRECOVERY logout request, up the 4242 * connection wait rcfr semaphore that is being blocked on 4243 * an iscsit_connection_reinstatement_rcfr(). 4244 */ 4245 if (atomic_read(&conn->connection_wait_rcfr)) { 4246 spin_unlock_bh(&conn->state_lock); 4247 complete(&conn->conn_wait_rcfr_comp); 4248 wait_for_completion(&conn->conn_post_wait_comp); 4249 spin_lock_bh(&conn->state_lock); 4250 } 4251 atomic_set(&conn->connection_reinstatement, 1); 4252 spin_unlock_bh(&conn->state_lock); 4253 4254 /* 4255 * If any other processes are accessing this connection pointer we 4256 * must wait until they have completed. 4257 */ 4258 iscsit_check_conn_usage_count(conn); 4259 4260 ahash_request_free(conn->conn_tx_hash); 4261 if (conn->conn_rx_hash) { 4262 struct crypto_ahash *tfm; 4263 4264 tfm = crypto_ahash_reqtfm(conn->conn_rx_hash); 4265 ahash_request_free(conn->conn_rx_hash); 4266 crypto_free_ahash(tfm); 4267 } 4268 4269 if (conn->sock) 4270 sock_release(conn->sock); 4271 4272 if (conn->conn_transport->iscsit_free_conn) 4273 conn->conn_transport->iscsit_free_conn(conn); 4274 4275 pr_debug("Moving to TARG_CONN_STATE_FREE.\n"); 4276 conn->conn_state = TARG_CONN_STATE_FREE; 4277 iscsit_free_conn(conn); 4278 4279 spin_lock_bh(&sess->conn_lock); 4280 atomic_dec(&sess->nconn); 4281 pr_debug("Decremented iSCSI connection count to %hu from node:" 4282 " %s\n", atomic_read(&sess->nconn), 4283 sess->sess_ops->InitiatorName); 4284 /* 4285 * Make sure that if one connection fails in an non ERL=2 iSCSI 4286 * Session that they all fail. 4287 */ 4288 if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout && 4289 !atomic_read(&sess->session_logout)) 4290 atomic_set(&sess->session_fall_back_to_erl0, 1); 4291 4292 /* 4293 * If this was not the last connection in the session, and we are 4294 * performing session reinstatement or falling back to ERL=0, call 4295 * iscsit_stop_session() without sleeping to shutdown the other 4296 * active connections. 4297 */ 4298 if (atomic_read(&sess->nconn)) { 4299 if (!atomic_read(&sess->session_reinstatement) && 4300 !atomic_read(&sess->session_fall_back_to_erl0)) { 4301 spin_unlock_bh(&sess->conn_lock); 4302 return 0; 4303 } 4304 if (!atomic_read(&sess->session_stop_active)) { 4305 atomic_set(&sess->session_stop_active, 1); 4306 spin_unlock_bh(&sess->conn_lock); 4307 iscsit_stop_session(sess, 0, 0); 4308 return 0; 4309 } 4310 spin_unlock_bh(&sess->conn_lock); 4311 return 0; 4312 } 4313 4314 /* 4315 * If this was the last connection in the session and one of the 4316 * following is occurring: 4317 * 4318 * Session Reinstatement is not being performed, and are falling back 4319 * to ERL=0 call iscsit_close_session(). 4320 * 4321 * Session Logout was requested. iscsit_close_session() will be called 4322 * elsewhere. 4323 * 4324 * Session Continuation is not being performed, start the Time2Retain 4325 * handler and check if sleep_on_sess_wait_sem is active. 4326 */ 4327 if (!atomic_read(&sess->session_reinstatement) && 4328 atomic_read(&sess->session_fall_back_to_erl0)) { 4329 spin_unlock_bh(&sess->conn_lock); 4330 complete_all(&sess->session_wait_comp); 4331 iscsit_close_session(sess, true); 4332 4333 return 0; 4334 } else if (atomic_read(&sess->session_logout)) { 4335 pr_debug("Moving to TARG_SESS_STATE_FREE.\n"); 4336 sess->session_state = TARG_SESS_STATE_FREE; 4337 4338 if (atomic_read(&sess->session_close)) { 4339 spin_unlock_bh(&sess->conn_lock); 4340 complete_all(&sess->session_wait_comp); 4341 iscsit_close_session(sess, true); 4342 } else { 4343 spin_unlock_bh(&sess->conn_lock); 4344 } 4345 4346 return 0; 4347 } else { 4348 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); 4349 sess->session_state = TARG_SESS_STATE_FAILED; 4350 4351 if (!atomic_read(&sess->session_continuation)) 4352 iscsit_start_time2retain_handler(sess); 4353 4354 if (atomic_read(&sess->session_close)) { 4355 spin_unlock_bh(&sess->conn_lock); 4356 complete_all(&sess->session_wait_comp); 4357 iscsit_close_session(sess, true); 4358 } else { 4359 spin_unlock_bh(&sess->conn_lock); 4360 } 4361 4362 return 0; 4363 } 4364 } 4365 4366 /* 4367 * If the iSCSI Session for the iSCSI Initiator Node exists, 4368 * forcefully shutdown the iSCSI NEXUS. 4369 */ 4370 int iscsit_close_session(struct iscsi_session *sess, bool can_sleep) 4371 { 4372 struct iscsi_portal_group *tpg = sess->tpg; 4373 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; 4374 4375 if (atomic_read(&sess->nconn)) { 4376 pr_err("%d connection(s) still exist for iSCSI session" 4377 " to %s\n", atomic_read(&sess->nconn), 4378 sess->sess_ops->InitiatorName); 4379 BUG(); 4380 } 4381 4382 spin_lock_bh(&se_tpg->session_lock); 4383 atomic_set(&sess->session_logout, 1); 4384 atomic_set(&sess->session_reinstatement, 1); 4385 iscsit_stop_time2retain_timer(sess); 4386 spin_unlock_bh(&se_tpg->session_lock); 4387 4388 /* 4389 * transport_deregister_session_configfs() will clear the 4390 * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context 4391 * can be setting it again with __transport_register_session() in 4392 * iscsi_post_login_handler() again after the iscsit_stop_session() 4393 * completes in iscsi_np context. 4394 */ 4395 transport_deregister_session_configfs(sess->se_sess); 4396 4397 /* 4398 * If any other processes are accessing this session pointer we must 4399 * wait until they have completed. If we are in an interrupt (the 4400 * time2retain handler) and contain and active session usage count we 4401 * restart the timer and exit. 4402 */ 4403 if (iscsit_check_session_usage_count(sess, can_sleep)) { 4404 atomic_set(&sess->session_logout, 0); 4405 iscsit_start_time2retain_handler(sess); 4406 return 0; 4407 } 4408 4409 transport_deregister_session(sess->se_sess); 4410 4411 if (sess->sess_ops->ErrorRecoveryLevel == 2) 4412 iscsit_free_connection_recovery_entries(sess); 4413 4414 iscsit_free_all_ooo_cmdsns(sess); 4415 4416 spin_lock_bh(&se_tpg->session_lock); 4417 pr_debug("Moving to TARG_SESS_STATE_FREE.\n"); 4418 sess->session_state = TARG_SESS_STATE_FREE; 4419 pr_debug("Released iSCSI session from node: %s\n", 4420 sess->sess_ops->InitiatorName); 4421 tpg->nsessions--; 4422 if (tpg->tpg_tiqn) 4423 tpg->tpg_tiqn->tiqn_nsessions--; 4424 4425 pr_debug("Decremented number of active iSCSI Sessions on" 4426 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions); 4427 4428 ida_free(&sess_ida, sess->session_index); 4429 kfree(sess->sess_ops); 4430 sess->sess_ops = NULL; 4431 spin_unlock_bh(&se_tpg->session_lock); 4432 4433 kfree(sess); 4434 return 0; 4435 } 4436 4437 static void iscsit_logout_post_handler_closesession( 4438 struct iscsi_conn *conn) 4439 { 4440 struct iscsi_session *sess = conn->sess; 4441 int sleep = 1; 4442 /* 4443 * Traditional iscsi/tcp will invoke this logic from TX thread 4444 * context during session logout, so clear tx_thread_active and 4445 * sleep if iscsit_close_connection() has not already occured. 4446 * 4447 * Since iser-target invokes this logic from it's own workqueue, 4448 * always sleep waiting for RX/TX thread shutdown to complete 4449 * within iscsit_close_connection(). 4450 */ 4451 if (!conn->conn_transport->rdma_shutdown) { 4452 sleep = cmpxchg(&conn->tx_thread_active, true, false); 4453 if (!sleep) 4454 return; 4455 } 4456 4457 atomic_set(&conn->conn_logout_remove, 0); 4458 complete(&conn->conn_logout_comp); 4459 4460 iscsit_dec_conn_usage_count(conn); 4461 atomic_set(&sess->session_close, 1); 4462 iscsit_stop_session(sess, sleep, sleep); 4463 iscsit_dec_session_usage_count(sess); 4464 } 4465 4466 static void iscsit_logout_post_handler_samecid( 4467 struct iscsi_conn *conn) 4468 { 4469 int sleep = 1; 4470 4471 if (!conn->conn_transport->rdma_shutdown) { 4472 sleep = cmpxchg(&conn->tx_thread_active, true, false); 4473 if (!sleep) 4474 return; 4475 } 4476 4477 atomic_set(&conn->conn_logout_remove, 0); 4478 complete(&conn->conn_logout_comp); 4479 4480 iscsit_cause_connection_reinstatement(conn, sleep); 4481 iscsit_dec_conn_usage_count(conn); 4482 } 4483 4484 static void iscsit_logout_post_handler_diffcid( 4485 struct iscsi_conn *conn, 4486 u16 cid) 4487 { 4488 struct iscsi_conn *l_conn; 4489 struct iscsi_session *sess = conn->sess; 4490 bool conn_found = false; 4491 4492 if (!sess) 4493 return; 4494 4495 spin_lock_bh(&sess->conn_lock); 4496 list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) { 4497 if (l_conn->cid == cid) { 4498 iscsit_inc_conn_usage_count(l_conn); 4499 conn_found = true; 4500 break; 4501 } 4502 } 4503 spin_unlock_bh(&sess->conn_lock); 4504 4505 if (!conn_found) 4506 return; 4507 4508 if (l_conn->sock) 4509 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN); 4510 4511 spin_lock_bh(&l_conn->state_lock); 4512 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 4513 l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT; 4514 spin_unlock_bh(&l_conn->state_lock); 4515 4516 iscsit_cause_connection_reinstatement(l_conn, 1); 4517 iscsit_dec_conn_usage_count(l_conn); 4518 } 4519 4520 /* 4521 * Return of 0 causes the TX thread to restart. 4522 */ 4523 int iscsit_logout_post_handler( 4524 struct iscsi_cmd *cmd, 4525 struct iscsi_conn *conn) 4526 { 4527 int ret = 0; 4528 4529 switch (cmd->logout_reason) { 4530 case ISCSI_LOGOUT_REASON_CLOSE_SESSION: 4531 switch (cmd->logout_response) { 4532 case ISCSI_LOGOUT_SUCCESS: 4533 case ISCSI_LOGOUT_CLEANUP_FAILED: 4534 default: 4535 iscsit_logout_post_handler_closesession(conn); 4536 break; 4537 } 4538 break; 4539 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: 4540 if (conn->cid == cmd->logout_cid) { 4541 switch (cmd->logout_response) { 4542 case ISCSI_LOGOUT_SUCCESS: 4543 case ISCSI_LOGOUT_CLEANUP_FAILED: 4544 default: 4545 iscsit_logout_post_handler_samecid(conn); 4546 break; 4547 } 4548 } else { 4549 switch (cmd->logout_response) { 4550 case ISCSI_LOGOUT_SUCCESS: 4551 iscsit_logout_post_handler_diffcid(conn, 4552 cmd->logout_cid); 4553 break; 4554 case ISCSI_LOGOUT_CID_NOT_FOUND: 4555 case ISCSI_LOGOUT_CLEANUP_FAILED: 4556 default: 4557 break; 4558 } 4559 ret = 1; 4560 } 4561 break; 4562 case ISCSI_LOGOUT_REASON_RECOVERY: 4563 switch (cmd->logout_response) { 4564 case ISCSI_LOGOUT_SUCCESS: 4565 case ISCSI_LOGOUT_CID_NOT_FOUND: 4566 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED: 4567 case ISCSI_LOGOUT_CLEANUP_FAILED: 4568 default: 4569 break; 4570 } 4571 ret = 1; 4572 break; 4573 default: 4574 break; 4575 4576 } 4577 return ret; 4578 } 4579 EXPORT_SYMBOL(iscsit_logout_post_handler); 4580 4581 void iscsit_fail_session(struct iscsi_session *sess) 4582 { 4583 struct iscsi_conn *conn; 4584 4585 spin_lock_bh(&sess->conn_lock); 4586 list_for_each_entry(conn, &sess->sess_conn_list, conn_list) { 4587 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n"); 4588 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT; 4589 } 4590 spin_unlock_bh(&sess->conn_lock); 4591 4592 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); 4593 sess->session_state = TARG_SESS_STATE_FAILED; 4594 } 4595 4596 void iscsit_stop_session( 4597 struct iscsi_session *sess, 4598 int session_sleep, 4599 int connection_sleep) 4600 { 4601 u16 conn_count = atomic_read(&sess->nconn); 4602 struct iscsi_conn *conn, *conn_tmp = NULL; 4603 int is_last; 4604 4605 spin_lock_bh(&sess->conn_lock); 4606 4607 if (connection_sleep) { 4608 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list, 4609 conn_list) { 4610 if (conn_count == 0) 4611 break; 4612 4613 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) { 4614 is_last = 1; 4615 } else { 4616 iscsit_inc_conn_usage_count(conn_tmp); 4617 is_last = 0; 4618 } 4619 iscsit_inc_conn_usage_count(conn); 4620 4621 spin_unlock_bh(&sess->conn_lock); 4622 iscsit_cause_connection_reinstatement(conn, 1); 4623 spin_lock_bh(&sess->conn_lock); 4624 4625 iscsit_dec_conn_usage_count(conn); 4626 if (is_last == 0) 4627 iscsit_dec_conn_usage_count(conn_tmp); 4628 conn_count--; 4629 } 4630 } else { 4631 list_for_each_entry(conn, &sess->sess_conn_list, conn_list) 4632 iscsit_cause_connection_reinstatement(conn, 0); 4633 } 4634 4635 if (session_sleep && atomic_read(&sess->nconn)) { 4636 spin_unlock_bh(&sess->conn_lock); 4637 wait_for_completion(&sess->session_wait_comp); 4638 } else 4639 spin_unlock_bh(&sess->conn_lock); 4640 } 4641 4642 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force) 4643 { 4644 struct iscsi_session *sess; 4645 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; 4646 struct se_session *se_sess, *se_sess_tmp; 4647 LIST_HEAD(free_list); 4648 int session_count = 0; 4649 4650 spin_lock_bh(&se_tpg->session_lock); 4651 if (tpg->nsessions && !force) { 4652 spin_unlock_bh(&se_tpg->session_lock); 4653 return -1; 4654 } 4655 4656 list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list, 4657 sess_list) { 4658 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr; 4659 4660 spin_lock(&sess->conn_lock); 4661 if (atomic_read(&sess->session_fall_back_to_erl0) || 4662 atomic_read(&sess->session_logout) || 4663 atomic_read(&sess->session_close) || 4664 (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) { 4665 spin_unlock(&sess->conn_lock); 4666 continue; 4667 } 4668 iscsit_inc_session_usage_count(sess); 4669 atomic_set(&sess->session_reinstatement, 1); 4670 atomic_set(&sess->session_fall_back_to_erl0, 1); 4671 atomic_set(&sess->session_close, 1); 4672 spin_unlock(&sess->conn_lock); 4673 4674 list_move_tail(&se_sess->sess_list, &free_list); 4675 } 4676 spin_unlock_bh(&se_tpg->session_lock); 4677 4678 list_for_each_entry_safe(se_sess, se_sess_tmp, &free_list, sess_list) { 4679 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr; 4680 4681 list_del_init(&se_sess->sess_list); 4682 iscsit_stop_session(sess, 1, 1); 4683 iscsit_dec_session_usage_count(sess); 4684 session_count++; 4685 } 4686 4687 pr_debug("Released %d iSCSI Session(s) from Target Portal" 4688 " Group: %hu\n", session_count, tpg->tpgt); 4689 return 0; 4690 } 4691 4692 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure"); 4693 MODULE_VERSION("4.1.x"); 4694 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 4695 MODULE_LICENSE("GPL"); 4696 4697 module_init(iscsi_target_init_module); 4698 module_exit(iscsi_target_cleanup_module); 4699