1 /******************************************************************************* 2 * This file contains main functions related to the iSCSI Target Core Driver. 3 * 4 * \u00a9 Copyright 2007-2011 RisingTide Systems LLC. 5 * 6 * Licensed to the Linux Foundation under the General Public License (GPL) version 2. 7 * 8 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License as published by 12 * the Free Software Foundation; either version 2 of the License, or 13 * (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 ******************************************************************************/ 20 21 #include <linux/string.h> 22 #include <linux/kthread.h> 23 #include <linux/crypto.h> 24 #include <linux/completion.h> 25 #include <linux/module.h> 26 #include <linux/idr.h> 27 #include <asm/unaligned.h> 28 #include <scsi/scsi_device.h> 29 #include <scsi/iscsi_proto.h> 30 #include <scsi/scsi_tcq.h> 31 #include <target/target_core_base.h> 32 #include <target/target_core_fabric.h> 33 #include <target/target_core_configfs.h> 34 35 #include "iscsi_target_core.h" 36 #include "iscsi_target_parameters.h" 37 #include "iscsi_target_seq_pdu_list.h" 38 #include "iscsi_target_tq.h" 39 #include "iscsi_target_configfs.h" 40 #include "iscsi_target_datain_values.h" 41 #include "iscsi_target_erl0.h" 42 #include "iscsi_target_erl1.h" 43 #include "iscsi_target_erl2.h" 44 #include "iscsi_target_login.h" 45 #include "iscsi_target_tmr.h" 46 #include "iscsi_target_tpg.h" 47 #include "iscsi_target_util.h" 48 #include "iscsi_target.h" 49 #include "iscsi_target_device.h" 50 #include "iscsi_target_stat.h" 51 52 static LIST_HEAD(g_tiqn_list); 53 static LIST_HEAD(g_np_list); 54 static DEFINE_SPINLOCK(tiqn_lock); 55 static DEFINE_SPINLOCK(np_lock); 56 57 static struct idr tiqn_idr; 58 struct idr sess_idr; 59 struct mutex auth_id_lock; 60 spinlock_t sess_idr_lock; 61 62 struct iscsit_global *iscsit_global; 63 64 struct kmem_cache *lio_cmd_cache; 65 struct kmem_cache *lio_qr_cache; 66 struct kmem_cache *lio_dr_cache; 67 struct kmem_cache *lio_ooo_cache; 68 struct kmem_cache *lio_r2t_cache; 69 70 static int iscsit_handle_immediate_data(struct iscsi_cmd *, 71 unsigned char *buf, u32); 72 static int iscsit_logout_post_handler(struct iscsi_cmd *, struct iscsi_conn *); 73 74 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf) 75 { 76 struct iscsi_tiqn *tiqn = NULL; 77 78 spin_lock(&tiqn_lock); 79 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) { 80 if (!strcmp(tiqn->tiqn, buf)) { 81 82 spin_lock(&tiqn->tiqn_state_lock); 83 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) { 84 tiqn->tiqn_access_count++; 85 spin_unlock(&tiqn->tiqn_state_lock); 86 spin_unlock(&tiqn_lock); 87 return tiqn; 88 } 89 spin_unlock(&tiqn->tiqn_state_lock); 90 } 91 } 92 spin_unlock(&tiqn_lock); 93 94 return NULL; 95 } 96 97 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn) 98 { 99 spin_lock(&tiqn->tiqn_state_lock); 100 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) { 101 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN; 102 spin_unlock(&tiqn->tiqn_state_lock); 103 return 0; 104 } 105 spin_unlock(&tiqn->tiqn_state_lock); 106 107 return -1; 108 } 109 110 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn) 111 { 112 spin_lock(&tiqn->tiqn_state_lock); 113 tiqn->tiqn_access_count--; 114 spin_unlock(&tiqn->tiqn_state_lock); 115 } 116 117 /* 118 * Note that IQN formatting is expected to be done in userspace, and 119 * no explict IQN format checks are done here. 120 */ 121 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf) 122 { 123 struct iscsi_tiqn *tiqn = NULL; 124 int ret; 125 126 if (strlen(buf) >= ISCSI_IQN_LEN) { 127 pr_err("Target IQN exceeds %d bytes\n", 128 ISCSI_IQN_LEN); 129 return ERR_PTR(-EINVAL); 130 } 131 132 tiqn = kzalloc(sizeof(struct iscsi_tiqn), GFP_KERNEL); 133 if (!tiqn) { 134 pr_err("Unable to allocate struct iscsi_tiqn\n"); 135 return ERR_PTR(-ENOMEM); 136 } 137 138 sprintf(tiqn->tiqn, "%s", buf); 139 INIT_LIST_HEAD(&tiqn->tiqn_list); 140 INIT_LIST_HEAD(&tiqn->tiqn_tpg_list); 141 spin_lock_init(&tiqn->tiqn_state_lock); 142 spin_lock_init(&tiqn->tiqn_tpg_lock); 143 spin_lock_init(&tiqn->sess_err_stats.lock); 144 spin_lock_init(&tiqn->login_stats.lock); 145 spin_lock_init(&tiqn->logout_stats.lock); 146 147 if (!idr_pre_get(&tiqn_idr, GFP_KERNEL)) { 148 pr_err("idr_pre_get() for tiqn_idr failed\n"); 149 kfree(tiqn); 150 return ERR_PTR(-ENOMEM); 151 } 152 tiqn->tiqn_state = TIQN_STATE_ACTIVE; 153 154 spin_lock(&tiqn_lock); 155 ret = idr_get_new(&tiqn_idr, NULL, &tiqn->tiqn_index); 156 if (ret < 0) { 157 pr_err("idr_get_new() failed for tiqn->tiqn_index\n"); 158 spin_unlock(&tiqn_lock); 159 kfree(tiqn); 160 return ERR_PTR(ret); 161 } 162 list_add_tail(&tiqn->tiqn_list, &g_tiqn_list); 163 spin_unlock(&tiqn_lock); 164 165 pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn); 166 167 return tiqn; 168 169 } 170 171 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn) 172 { 173 /* 174 * Wait for accesses to said struct iscsi_tiqn to end. 175 */ 176 spin_lock(&tiqn->tiqn_state_lock); 177 while (tiqn->tiqn_access_count != 0) { 178 spin_unlock(&tiqn->tiqn_state_lock); 179 msleep(10); 180 spin_lock(&tiqn->tiqn_state_lock); 181 } 182 spin_unlock(&tiqn->tiqn_state_lock); 183 } 184 185 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn) 186 { 187 /* 188 * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN 189 * while holding tiqn->tiqn_state_lock. This means that all subsequent 190 * attempts to access this struct iscsi_tiqn will fail from both transport 191 * fabric and control code paths. 192 */ 193 if (iscsit_set_tiqn_shutdown(tiqn) < 0) { 194 pr_err("iscsit_set_tiqn_shutdown() failed\n"); 195 return; 196 } 197 198 iscsit_wait_for_tiqn(tiqn); 199 200 spin_lock(&tiqn_lock); 201 list_del(&tiqn->tiqn_list); 202 idr_remove(&tiqn_idr, tiqn->tiqn_index); 203 spin_unlock(&tiqn_lock); 204 205 pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n", 206 tiqn->tiqn); 207 kfree(tiqn); 208 } 209 210 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg) 211 { 212 int ret; 213 /* 214 * Determine if the network portal is accepting storage traffic. 215 */ 216 spin_lock_bh(&np->np_thread_lock); 217 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { 218 spin_unlock_bh(&np->np_thread_lock); 219 return -1; 220 } 221 if (np->np_login_tpg) { 222 pr_err("np->np_login_tpg() is not NULL!\n"); 223 spin_unlock_bh(&np->np_thread_lock); 224 return -1; 225 } 226 spin_unlock_bh(&np->np_thread_lock); 227 /* 228 * Determine if the portal group is accepting storage traffic. 229 */ 230 spin_lock_bh(&tpg->tpg_state_lock); 231 if (tpg->tpg_state != TPG_STATE_ACTIVE) { 232 spin_unlock_bh(&tpg->tpg_state_lock); 233 return -1; 234 } 235 spin_unlock_bh(&tpg->tpg_state_lock); 236 237 /* 238 * Here we serialize access across the TIQN+TPG Tuple. 239 */ 240 ret = mutex_lock_interruptible(&tpg->np_login_lock); 241 if ((ret != 0) || signal_pending(current)) 242 return -1; 243 244 spin_lock_bh(&np->np_thread_lock); 245 np->np_login_tpg = tpg; 246 spin_unlock_bh(&np->np_thread_lock); 247 248 return 0; 249 } 250 251 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg) 252 { 253 struct iscsi_tiqn *tiqn = tpg->tpg_tiqn; 254 255 spin_lock_bh(&np->np_thread_lock); 256 np->np_login_tpg = NULL; 257 spin_unlock_bh(&np->np_thread_lock); 258 259 mutex_unlock(&tpg->np_login_lock); 260 261 if (tiqn) 262 iscsit_put_tiqn_for_login(tiqn); 263 264 return 0; 265 } 266 267 static struct iscsi_np *iscsit_get_np( 268 struct __kernel_sockaddr_storage *sockaddr, 269 int network_transport) 270 { 271 struct sockaddr_in *sock_in, *sock_in_e; 272 struct sockaddr_in6 *sock_in6, *sock_in6_e; 273 struct iscsi_np *np; 274 int ip_match = 0; 275 u16 port; 276 277 spin_lock_bh(&np_lock); 278 list_for_each_entry(np, &g_np_list, np_list) { 279 spin_lock(&np->np_thread_lock); 280 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { 281 spin_unlock(&np->np_thread_lock); 282 continue; 283 } 284 285 if (sockaddr->ss_family == AF_INET6) { 286 sock_in6 = (struct sockaddr_in6 *)sockaddr; 287 sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr; 288 289 if (!memcmp(&sock_in6->sin6_addr.in6_u, 290 &sock_in6_e->sin6_addr.in6_u, 291 sizeof(struct in6_addr))) 292 ip_match = 1; 293 294 port = ntohs(sock_in6->sin6_port); 295 } else { 296 sock_in = (struct sockaddr_in *)sockaddr; 297 sock_in_e = (struct sockaddr_in *)&np->np_sockaddr; 298 299 if (sock_in->sin_addr.s_addr == 300 sock_in_e->sin_addr.s_addr) 301 ip_match = 1; 302 303 port = ntohs(sock_in->sin_port); 304 } 305 306 if ((ip_match == 1) && (np->np_port == port) && 307 (np->np_network_transport == network_transport)) { 308 /* 309 * Increment the np_exports reference count now to 310 * prevent iscsit_del_np() below from being called 311 * while iscsi_tpg_add_network_portal() is called. 312 */ 313 np->np_exports++; 314 spin_unlock(&np->np_thread_lock); 315 spin_unlock_bh(&np_lock); 316 return np; 317 } 318 spin_unlock(&np->np_thread_lock); 319 } 320 spin_unlock_bh(&np_lock); 321 322 return NULL; 323 } 324 325 struct iscsi_np *iscsit_add_np( 326 struct __kernel_sockaddr_storage *sockaddr, 327 char *ip_str, 328 int network_transport) 329 { 330 struct sockaddr_in *sock_in; 331 struct sockaddr_in6 *sock_in6; 332 struct iscsi_np *np; 333 int ret; 334 /* 335 * Locate the existing struct iscsi_np if already active.. 336 */ 337 np = iscsit_get_np(sockaddr, network_transport); 338 if (np) 339 return np; 340 341 np = kzalloc(sizeof(struct iscsi_np), GFP_KERNEL); 342 if (!np) { 343 pr_err("Unable to allocate memory for struct iscsi_np\n"); 344 return ERR_PTR(-ENOMEM); 345 } 346 347 np->np_flags |= NPF_IP_NETWORK; 348 if (sockaddr->ss_family == AF_INET6) { 349 sock_in6 = (struct sockaddr_in6 *)sockaddr; 350 snprintf(np->np_ip, IPV6_ADDRESS_SPACE, "%s", ip_str); 351 np->np_port = ntohs(sock_in6->sin6_port); 352 } else { 353 sock_in = (struct sockaddr_in *)sockaddr; 354 sprintf(np->np_ip, "%s", ip_str); 355 np->np_port = ntohs(sock_in->sin_port); 356 } 357 358 np->np_network_transport = network_transport; 359 spin_lock_init(&np->np_thread_lock); 360 init_completion(&np->np_restart_comp); 361 INIT_LIST_HEAD(&np->np_list); 362 363 ret = iscsi_target_setup_login_socket(np, sockaddr); 364 if (ret != 0) { 365 kfree(np); 366 return ERR_PTR(ret); 367 } 368 369 np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np"); 370 if (IS_ERR(np->np_thread)) { 371 pr_err("Unable to create kthread: iscsi_np\n"); 372 ret = PTR_ERR(np->np_thread); 373 kfree(np); 374 return ERR_PTR(ret); 375 } 376 /* 377 * Increment the np_exports reference count now to prevent 378 * iscsit_del_np() below from being run while a new call to 379 * iscsi_tpg_add_network_portal() for a matching iscsi_np is 380 * active. We don't need to hold np->np_thread_lock at this 381 * point because iscsi_np has not been added to g_np_list yet. 382 */ 383 np->np_exports = 1; 384 385 spin_lock_bh(&np_lock); 386 list_add_tail(&np->np_list, &g_np_list); 387 spin_unlock_bh(&np_lock); 388 389 pr_debug("CORE[0] - Added Network Portal: %s:%hu on %s\n", 390 np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ? 391 "TCP" : "SCTP"); 392 393 return np; 394 } 395 396 int iscsit_reset_np_thread( 397 struct iscsi_np *np, 398 struct iscsi_tpg_np *tpg_np, 399 struct iscsi_portal_group *tpg) 400 { 401 spin_lock_bh(&np->np_thread_lock); 402 if (tpg && tpg_np) { 403 /* 404 * The reset operation need only be performed when the 405 * passed struct iscsi_portal_group has a login in progress 406 * to one of the network portals. 407 */ 408 if (tpg_np->tpg_np->np_login_tpg != tpg) { 409 spin_unlock_bh(&np->np_thread_lock); 410 return 0; 411 } 412 } 413 if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) { 414 spin_unlock_bh(&np->np_thread_lock); 415 return 0; 416 } 417 np->np_thread_state = ISCSI_NP_THREAD_RESET; 418 419 if (np->np_thread) { 420 spin_unlock_bh(&np->np_thread_lock); 421 send_sig(SIGINT, np->np_thread, 1); 422 wait_for_completion(&np->np_restart_comp); 423 spin_lock_bh(&np->np_thread_lock); 424 } 425 spin_unlock_bh(&np->np_thread_lock); 426 427 return 0; 428 } 429 430 int iscsit_del_np_comm(struct iscsi_np *np) 431 { 432 if (!np->np_socket) 433 return 0; 434 435 /* 436 * Some network transports allocate their own struct sock->file, 437 * see if we need to free any additional allocated resources. 438 */ 439 if (np->np_flags & NPF_SCTP_STRUCT_FILE) { 440 kfree(np->np_socket->file); 441 np->np_socket->file = NULL; 442 } 443 444 sock_release(np->np_socket); 445 return 0; 446 } 447 448 int iscsit_del_np(struct iscsi_np *np) 449 { 450 spin_lock_bh(&np->np_thread_lock); 451 np->np_exports--; 452 if (np->np_exports) { 453 spin_unlock_bh(&np->np_thread_lock); 454 return 0; 455 } 456 np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN; 457 spin_unlock_bh(&np->np_thread_lock); 458 459 if (np->np_thread) { 460 /* 461 * We need to send the signal to wakeup Linux/Net 462 * which may be sleeping in sock_accept().. 463 */ 464 send_sig(SIGINT, np->np_thread, 1); 465 kthread_stop(np->np_thread); 466 } 467 iscsit_del_np_comm(np); 468 469 spin_lock_bh(&np_lock); 470 list_del(&np->np_list); 471 spin_unlock_bh(&np_lock); 472 473 pr_debug("CORE[0] - Removed Network Portal: %s:%hu on %s\n", 474 np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ? 475 "TCP" : "SCTP"); 476 477 kfree(np); 478 return 0; 479 } 480 481 static int __init iscsi_target_init_module(void) 482 { 483 int ret = 0; 484 485 pr_debug("iSCSI-Target "ISCSIT_VERSION"\n"); 486 487 iscsit_global = kzalloc(sizeof(struct iscsit_global), GFP_KERNEL); 488 if (!iscsit_global) { 489 pr_err("Unable to allocate memory for iscsit_global\n"); 490 return -1; 491 } 492 mutex_init(&auth_id_lock); 493 spin_lock_init(&sess_idr_lock); 494 idr_init(&tiqn_idr); 495 idr_init(&sess_idr); 496 497 ret = iscsi_target_register_configfs(); 498 if (ret < 0) 499 goto out; 500 501 ret = iscsi_thread_set_init(); 502 if (ret < 0) 503 goto configfs_out; 504 505 if (iscsi_allocate_thread_sets(TARGET_THREAD_SET_COUNT) != 506 TARGET_THREAD_SET_COUNT) { 507 pr_err("iscsi_allocate_thread_sets() returned" 508 " unexpected value!\n"); 509 goto ts_out1; 510 } 511 512 lio_cmd_cache = kmem_cache_create("lio_cmd_cache", 513 sizeof(struct iscsi_cmd), __alignof__(struct iscsi_cmd), 514 0, NULL); 515 if (!lio_cmd_cache) { 516 pr_err("Unable to kmem_cache_create() for" 517 " lio_cmd_cache\n"); 518 goto ts_out2; 519 } 520 521 lio_qr_cache = kmem_cache_create("lio_qr_cache", 522 sizeof(struct iscsi_queue_req), 523 __alignof__(struct iscsi_queue_req), 0, NULL); 524 if (!lio_qr_cache) { 525 pr_err("nable to kmem_cache_create() for" 526 " lio_qr_cache\n"); 527 goto cmd_out; 528 } 529 530 lio_dr_cache = kmem_cache_create("lio_dr_cache", 531 sizeof(struct iscsi_datain_req), 532 __alignof__(struct iscsi_datain_req), 0, NULL); 533 if (!lio_dr_cache) { 534 pr_err("Unable to kmem_cache_create() for" 535 " lio_dr_cache\n"); 536 goto qr_out; 537 } 538 539 lio_ooo_cache = kmem_cache_create("lio_ooo_cache", 540 sizeof(struct iscsi_ooo_cmdsn), 541 __alignof__(struct iscsi_ooo_cmdsn), 0, NULL); 542 if (!lio_ooo_cache) { 543 pr_err("Unable to kmem_cache_create() for" 544 " lio_ooo_cache\n"); 545 goto dr_out; 546 } 547 548 lio_r2t_cache = kmem_cache_create("lio_r2t_cache", 549 sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t), 550 0, NULL); 551 if (!lio_r2t_cache) { 552 pr_err("Unable to kmem_cache_create() for" 553 " lio_r2t_cache\n"); 554 goto ooo_out; 555 } 556 557 if (iscsit_load_discovery_tpg() < 0) 558 goto r2t_out; 559 560 return ret; 561 r2t_out: 562 kmem_cache_destroy(lio_r2t_cache); 563 ooo_out: 564 kmem_cache_destroy(lio_ooo_cache); 565 dr_out: 566 kmem_cache_destroy(lio_dr_cache); 567 qr_out: 568 kmem_cache_destroy(lio_qr_cache); 569 cmd_out: 570 kmem_cache_destroy(lio_cmd_cache); 571 ts_out2: 572 iscsi_deallocate_thread_sets(); 573 ts_out1: 574 iscsi_thread_set_free(); 575 configfs_out: 576 iscsi_target_deregister_configfs(); 577 out: 578 kfree(iscsit_global); 579 return -ENOMEM; 580 } 581 582 static void __exit iscsi_target_cleanup_module(void) 583 { 584 iscsi_deallocate_thread_sets(); 585 iscsi_thread_set_free(); 586 iscsit_release_discovery_tpg(); 587 kmem_cache_destroy(lio_cmd_cache); 588 kmem_cache_destroy(lio_qr_cache); 589 kmem_cache_destroy(lio_dr_cache); 590 kmem_cache_destroy(lio_ooo_cache); 591 kmem_cache_destroy(lio_r2t_cache); 592 593 iscsi_target_deregister_configfs(); 594 595 kfree(iscsit_global); 596 } 597 598 static int iscsit_add_reject( 599 u8 reason, 600 int fail_conn, 601 unsigned char *buf, 602 struct iscsi_conn *conn) 603 { 604 struct iscsi_cmd *cmd; 605 struct iscsi_reject *hdr; 606 int ret; 607 608 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 609 if (!cmd) 610 return -1; 611 612 cmd->iscsi_opcode = ISCSI_OP_REJECT; 613 if (fail_conn) 614 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN; 615 616 hdr = (struct iscsi_reject *) cmd->pdu; 617 hdr->reason = reason; 618 619 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL); 620 if (!cmd->buf_ptr) { 621 pr_err("Unable to allocate memory for cmd->buf_ptr\n"); 622 iscsit_release_cmd(cmd); 623 return -1; 624 } 625 626 spin_lock_bh(&conn->cmd_lock); 627 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 628 spin_unlock_bh(&conn->cmd_lock); 629 630 cmd->i_state = ISTATE_SEND_REJECT; 631 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 632 633 ret = wait_for_completion_interruptible(&cmd->reject_comp); 634 if (ret != 0) 635 return -1; 636 637 return (!fail_conn) ? 0 : -1; 638 } 639 640 int iscsit_add_reject_from_cmd( 641 u8 reason, 642 int fail_conn, 643 int add_to_conn, 644 unsigned char *buf, 645 struct iscsi_cmd *cmd) 646 { 647 struct iscsi_conn *conn; 648 struct iscsi_reject *hdr; 649 int ret; 650 651 if (!cmd->conn) { 652 pr_err("cmd->conn is NULL for ITT: 0x%08x\n", 653 cmd->init_task_tag); 654 return -1; 655 } 656 conn = cmd->conn; 657 658 cmd->iscsi_opcode = ISCSI_OP_REJECT; 659 if (fail_conn) 660 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN; 661 662 hdr = (struct iscsi_reject *) cmd->pdu; 663 hdr->reason = reason; 664 665 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL); 666 if (!cmd->buf_ptr) { 667 pr_err("Unable to allocate memory for cmd->buf_ptr\n"); 668 iscsit_release_cmd(cmd); 669 return -1; 670 } 671 672 if (add_to_conn) { 673 spin_lock_bh(&conn->cmd_lock); 674 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 675 spin_unlock_bh(&conn->cmd_lock); 676 } 677 678 cmd->i_state = ISTATE_SEND_REJECT; 679 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 680 681 ret = wait_for_completion_interruptible(&cmd->reject_comp); 682 if (ret != 0) 683 return -1; 684 685 return (!fail_conn) ? 0 : -1; 686 } 687 688 /* 689 * Map some portion of the allocated scatterlist to an iovec, suitable for 690 * kernel sockets to copy data in/out. 691 */ 692 static int iscsit_map_iovec( 693 struct iscsi_cmd *cmd, 694 struct kvec *iov, 695 u32 data_offset, 696 u32 data_length) 697 { 698 u32 i = 0; 699 struct scatterlist *sg; 700 unsigned int page_off; 701 702 /* 703 * We know each entry in t_data_sg contains a page. 704 */ 705 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE]; 706 page_off = (data_offset % PAGE_SIZE); 707 708 cmd->first_data_sg = sg; 709 cmd->first_data_sg_off = page_off; 710 711 while (data_length) { 712 u32 cur_len = min_t(u32, data_length, sg->length - page_off); 713 714 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off; 715 iov[i].iov_len = cur_len; 716 717 data_length -= cur_len; 718 page_off = 0; 719 sg = sg_next(sg); 720 i++; 721 } 722 723 cmd->kmapped_nents = i; 724 725 return i; 726 } 727 728 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd) 729 { 730 u32 i; 731 struct scatterlist *sg; 732 733 sg = cmd->first_data_sg; 734 735 for (i = 0; i < cmd->kmapped_nents; i++) 736 kunmap(sg_page(&sg[i])); 737 } 738 739 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn) 740 { 741 struct iscsi_cmd *cmd; 742 743 conn->exp_statsn = exp_statsn; 744 745 spin_lock_bh(&conn->cmd_lock); 746 list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) { 747 spin_lock(&cmd->istate_lock); 748 if ((cmd->i_state == ISTATE_SENT_STATUS) && 749 (cmd->stat_sn < exp_statsn)) { 750 cmd->i_state = ISTATE_REMOVE; 751 spin_unlock(&cmd->istate_lock); 752 iscsit_add_cmd_to_immediate_queue(cmd, conn, 753 cmd->i_state); 754 continue; 755 } 756 spin_unlock(&cmd->istate_lock); 757 } 758 spin_unlock_bh(&conn->cmd_lock); 759 } 760 761 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd) 762 { 763 u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE)); 764 765 iov_count += ISCSI_IOV_DATA_BUFFER; 766 767 cmd->iov_data = kzalloc(iov_count * sizeof(struct kvec), GFP_KERNEL); 768 if (!cmd->iov_data) { 769 pr_err("Unable to allocate cmd->iov_data\n"); 770 return -ENOMEM; 771 } 772 773 cmd->orig_iov_data_count = iov_count; 774 return 0; 775 } 776 777 static int iscsit_handle_scsi_cmd( 778 struct iscsi_conn *conn, 779 unsigned char *buf) 780 { 781 int data_direction, cmdsn_ret = 0, immed_ret, ret, transport_ret; 782 int dump_immediate_data = 0, send_check_condition = 0, payload_length; 783 struct iscsi_cmd *cmd = NULL; 784 struct iscsi_scsi_req *hdr; 785 int iscsi_task_attr; 786 int sam_task_attr; 787 788 spin_lock_bh(&conn->sess->session_stats_lock); 789 conn->sess->cmd_pdus++; 790 if (conn->sess->se_sess->se_node_acl) { 791 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock); 792 conn->sess->se_sess->se_node_acl->num_cmds++; 793 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock); 794 } 795 spin_unlock_bh(&conn->sess->session_stats_lock); 796 797 hdr = (struct iscsi_scsi_req *) buf; 798 payload_length = ntoh24(hdr->dlength); 799 hdr->itt = be32_to_cpu(hdr->itt); 800 hdr->data_length = be32_to_cpu(hdr->data_length); 801 hdr->cmdsn = be32_to_cpu(hdr->cmdsn); 802 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 803 804 /* FIXME; Add checks for AdditionalHeaderSegment */ 805 806 if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) && 807 !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) { 808 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL" 809 " not set. Bad iSCSI Initiator.\n"); 810 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 811 buf, conn); 812 } 813 814 if (((hdr->flags & ISCSI_FLAG_CMD_READ) || 815 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) { 816 /* 817 * Vmware ESX v3.0 uses a modified Cisco Initiator (v3.4.2) 818 * that adds support for RESERVE/RELEASE. There is a bug 819 * add with this new functionality that sets R/W bits when 820 * neither CDB carries any READ or WRITE datapayloads. 821 */ 822 if ((hdr->cdb[0] == 0x16) || (hdr->cdb[0] == 0x17)) { 823 hdr->flags &= ~ISCSI_FLAG_CMD_READ; 824 hdr->flags &= ~ISCSI_FLAG_CMD_WRITE; 825 goto done; 826 } 827 828 pr_err("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE" 829 " set when Expected Data Transfer Length is 0 for" 830 " CDB: 0x%02x. Bad iSCSI Initiator.\n", hdr->cdb[0]); 831 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 832 buf, conn); 833 } 834 done: 835 836 if (!(hdr->flags & ISCSI_FLAG_CMD_READ) && 837 !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) { 838 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE" 839 " MUST be set if Expected Data Transfer Length is not 0." 840 " Bad iSCSI Initiator\n"); 841 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 842 buf, conn); 843 } 844 845 if ((hdr->flags & ISCSI_FLAG_CMD_READ) && 846 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) { 847 pr_err("Bidirectional operations not supported!\n"); 848 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 849 buf, conn); 850 } 851 852 if (hdr->opcode & ISCSI_OP_IMMEDIATE) { 853 pr_err("Illegally set Immediate Bit in iSCSI Initiator" 854 " Scsi Command PDU.\n"); 855 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 856 buf, conn); 857 } 858 859 if (payload_length && !conn->sess->sess_ops->ImmediateData) { 860 pr_err("ImmediateData=No but DataSegmentLength=%u," 861 " protocol error.\n", payload_length); 862 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 863 buf, conn); 864 } 865 866 if ((hdr->data_length == payload_length) && 867 (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) { 868 pr_err("Expected Data Transfer Length and Length of" 869 " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL" 870 " bit is not set protocol error\n"); 871 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 872 buf, conn); 873 } 874 875 if (payload_length > hdr->data_length) { 876 pr_err("DataSegmentLength: %u is greater than" 877 " EDTL: %u, protocol error.\n", payload_length, 878 hdr->data_length); 879 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 880 buf, conn); 881 } 882 883 if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) { 884 pr_err("DataSegmentLength: %u is greater than" 885 " MaxRecvDataSegmentLength: %u, protocol error.\n", 886 payload_length, conn->conn_ops->MaxRecvDataSegmentLength); 887 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 888 buf, conn); 889 } 890 891 if (payload_length > conn->sess->sess_ops->FirstBurstLength) { 892 pr_err("DataSegmentLength: %u is greater than" 893 " FirstBurstLength: %u, protocol error.\n", 894 payload_length, conn->sess->sess_ops->FirstBurstLength); 895 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, 896 buf, conn); 897 } 898 899 data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE : 900 (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE : 901 DMA_NONE; 902 903 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 904 if (!cmd) 905 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1, 906 buf, conn); 907 908 cmd->data_direction = data_direction; 909 iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK; 910 /* 911 * Figure out the SAM Task Attribute for the incoming SCSI CDB 912 */ 913 if ((iscsi_task_attr == ISCSI_ATTR_UNTAGGED) || 914 (iscsi_task_attr == ISCSI_ATTR_SIMPLE)) 915 sam_task_attr = MSG_SIMPLE_TAG; 916 else if (iscsi_task_attr == ISCSI_ATTR_ORDERED) 917 sam_task_attr = MSG_ORDERED_TAG; 918 else if (iscsi_task_attr == ISCSI_ATTR_HEAD_OF_QUEUE) 919 sam_task_attr = MSG_HEAD_TAG; 920 else if (iscsi_task_attr == ISCSI_ATTR_ACA) 921 sam_task_attr = MSG_ACA_TAG; 922 else { 923 pr_debug("Unknown iSCSI Task Attribute: 0x%02x, using" 924 " MSG_SIMPLE_TAG\n", iscsi_task_attr); 925 sam_task_attr = MSG_SIMPLE_TAG; 926 } 927 928 cmd->iscsi_opcode = ISCSI_OP_SCSI_CMD; 929 cmd->i_state = ISTATE_NEW_CMD; 930 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 931 cmd->immediate_data = (payload_length) ? 1 : 0; 932 cmd->unsolicited_data = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) && 933 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0); 934 if (cmd->unsolicited_data) 935 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA; 936 937 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 938 if (hdr->flags & ISCSI_FLAG_CMD_READ) { 939 spin_lock_bh(&conn->sess->ttt_lock); 940 cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++; 941 if (cmd->targ_xfer_tag == 0xFFFFFFFF) 942 cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++; 943 spin_unlock_bh(&conn->sess->ttt_lock); 944 } else if (hdr->flags & ISCSI_FLAG_CMD_WRITE) 945 cmd->targ_xfer_tag = 0xFFFFFFFF; 946 cmd->cmd_sn = hdr->cmdsn; 947 cmd->exp_stat_sn = hdr->exp_statsn; 948 cmd->first_burst_len = payload_length; 949 950 if (cmd->data_direction == DMA_FROM_DEVICE) { 951 struct iscsi_datain_req *dr; 952 953 dr = iscsit_allocate_datain_req(); 954 if (!dr) 955 return iscsit_add_reject_from_cmd( 956 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 957 1, 1, buf, cmd); 958 959 iscsit_attach_datain_req(cmd, dr); 960 } 961 962 /* 963 * Initialize struct se_cmd descriptor from target_core_mod infrastructure 964 */ 965 transport_init_se_cmd(&cmd->se_cmd, &lio_target_fabric_configfs->tf_ops, 966 conn->sess->se_sess, hdr->data_length, cmd->data_direction, 967 sam_task_attr, &cmd->sense_buffer[0]); 968 969 pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x," 970 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt, 971 hdr->cmdsn, hdr->data_length, payload_length, conn->cid); 972 973 /* 974 * The CDB is going to an se_device_t. 975 */ 976 ret = transport_lookup_cmd_lun(&cmd->se_cmd, 977 scsilun_to_int(&hdr->lun)); 978 if (ret < 0) { 979 if (cmd->se_cmd.scsi_sense_reason == TCM_NON_EXISTENT_LUN) { 980 pr_debug("Responding to non-acl'ed," 981 " non-existent or non-exported iSCSI LUN:" 982 " 0x%016Lx\n", get_unaligned_le64(&hdr->lun)); 983 } 984 send_check_condition = 1; 985 goto attach_cmd; 986 } 987 988 transport_ret = target_setup_cmd_from_cdb(&cmd->se_cmd, hdr->cdb); 989 if (transport_ret == -ENOMEM) { 990 return iscsit_add_reject_from_cmd( 991 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 992 1, 1, buf, cmd); 993 } else if (transport_ret < 0) { 994 /* 995 * Unsupported SAM Opcode. CHECK_CONDITION will be sent 996 * in iscsit_execute_cmd() during the CmdSN OOO Execution 997 * Mechinism. 998 */ 999 send_check_condition = 1; 1000 } else { 1001 if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) 1002 return iscsit_add_reject_from_cmd( 1003 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1004 1, 1, buf, cmd); 1005 } 1006 1007 attach_cmd: 1008 spin_lock_bh(&conn->cmd_lock); 1009 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 1010 spin_unlock_bh(&conn->cmd_lock); 1011 /* 1012 * Check if we need to delay processing because of ALUA 1013 * Active/NonOptimized primary access state.. 1014 */ 1015 core_alua_check_nonop_delay(&cmd->se_cmd); 1016 1017 ret = iscsit_allocate_iovecs(cmd); 1018 if (ret < 0) 1019 return iscsit_add_reject_from_cmd( 1020 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1021 1, 0, buf, cmd); 1022 /* 1023 * Check the CmdSN against ExpCmdSN/MaxCmdSN here if 1024 * the Immediate Bit is not set, and no Immediate 1025 * Data is attached. 1026 * 1027 * A PDU/CmdSN carrying Immediate Data can only 1028 * be processed after the DataCRC has passed. 1029 * If the DataCRC fails, the CmdSN MUST NOT 1030 * be acknowledged. (See below) 1031 */ 1032 if (!cmd->immediate_data) { 1033 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 1034 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) 1035 return 0; 1036 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1037 return iscsit_add_reject_from_cmd( 1038 ISCSI_REASON_PROTOCOL_ERROR, 1039 1, 0, buf, cmd); 1040 } 1041 1042 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn); 1043 1044 /* 1045 * If no Immediate Data is attached, it's OK to return now. 1046 */ 1047 if (!cmd->immediate_data) { 1048 if (send_check_condition) 1049 return 0; 1050 1051 if (cmd->unsolicited_data) { 1052 iscsit_set_dataout_sequence_values(cmd); 1053 1054 spin_lock_bh(&cmd->dataout_timeout_lock); 1055 iscsit_start_dataout_timer(cmd, cmd->conn); 1056 spin_unlock_bh(&cmd->dataout_timeout_lock); 1057 } 1058 1059 return 0; 1060 } 1061 1062 /* 1063 * Early CHECK_CONDITIONs never make it to the transport processing 1064 * thread. They are processed in CmdSN order by 1065 * iscsit_check_received_cmdsn() below. 1066 */ 1067 if (send_check_condition) { 1068 immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 1069 dump_immediate_data = 1; 1070 goto after_immediate_data; 1071 } 1072 /* 1073 * Call directly into transport_generic_new_cmd() to perform 1074 * the backend memory allocation. 1075 */ 1076 ret = transport_generic_new_cmd(&cmd->se_cmd); 1077 if (ret < 0) { 1078 immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 1079 dump_immediate_data = 1; 1080 goto after_immediate_data; 1081 } 1082 1083 immed_ret = iscsit_handle_immediate_data(cmd, buf, payload_length); 1084 after_immediate_data: 1085 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 1086 /* 1087 * A PDU/CmdSN carrying Immediate Data passed 1088 * DataCRC, check against ExpCmdSN/MaxCmdSN if 1089 * Immediate Bit is not set. 1090 */ 1091 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 1092 /* 1093 * Special case for Unsupported SAM WRITE Opcodes 1094 * and ImmediateData=Yes. 1095 */ 1096 if (dump_immediate_data) { 1097 if (iscsit_dump_data_payload(conn, payload_length, 1) < 0) 1098 return -1; 1099 } else if (cmd->unsolicited_data) { 1100 iscsit_set_dataout_sequence_values(cmd); 1101 1102 spin_lock_bh(&cmd->dataout_timeout_lock); 1103 iscsit_start_dataout_timer(cmd, cmd->conn); 1104 spin_unlock_bh(&cmd->dataout_timeout_lock); 1105 } 1106 1107 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1108 return iscsit_add_reject_from_cmd( 1109 ISCSI_REASON_PROTOCOL_ERROR, 1110 1, 0, buf, cmd); 1111 1112 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 1113 /* 1114 * Immediate Data failed DataCRC and ERL>=1, 1115 * silently drop this PDU and let the initiator 1116 * plug the CmdSN gap. 1117 * 1118 * FIXME: Send Unsolicited NOPIN with reserved 1119 * TTT here to help the initiator figure out 1120 * the missing CmdSN, although they should be 1121 * intelligent enough to determine the missing 1122 * CmdSN and issue a retry to plug the sequence. 1123 */ 1124 cmd->i_state = ISTATE_REMOVE; 1125 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 1126 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 1127 return -1; 1128 1129 return 0; 1130 } 1131 1132 static u32 iscsit_do_crypto_hash_sg( 1133 struct hash_desc *hash, 1134 struct iscsi_cmd *cmd, 1135 u32 data_offset, 1136 u32 data_length, 1137 u32 padding, 1138 u8 *pad_bytes) 1139 { 1140 u32 data_crc; 1141 u32 i; 1142 struct scatterlist *sg; 1143 unsigned int page_off; 1144 1145 crypto_hash_init(hash); 1146 1147 sg = cmd->first_data_sg; 1148 page_off = cmd->first_data_sg_off; 1149 1150 i = 0; 1151 while (data_length) { 1152 u32 cur_len = min_t(u32, data_length, (sg[i].length - page_off)); 1153 1154 crypto_hash_update(hash, &sg[i], cur_len); 1155 1156 data_length -= cur_len; 1157 page_off = 0; 1158 i++; 1159 } 1160 1161 if (padding) { 1162 struct scatterlist pad_sg; 1163 1164 sg_init_one(&pad_sg, pad_bytes, padding); 1165 crypto_hash_update(hash, &pad_sg, padding); 1166 } 1167 crypto_hash_final(hash, (u8 *) &data_crc); 1168 1169 return data_crc; 1170 } 1171 1172 static void iscsit_do_crypto_hash_buf( 1173 struct hash_desc *hash, 1174 unsigned char *buf, 1175 u32 payload_length, 1176 u32 padding, 1177 u8 *pad_bytes, 1178 u8 *data_crc) 1179 { 1180 struct scatterlist sg; 1181 1182 crypto_hash_init(hash); 1183 1184 sg_init_one(&sg, buf, payload_length); 1185 crypto_hash_update(hash, &sg, payload_length); 1186 1187 if (padding) { 1188 sg_init_one(&sg, pad_bytes, padding); 1189 crypto_hash_update(hash, &sg, padding); 1190 } 1191 crypto_hash_final(hash, data_crc); 1192 } 1193 1194 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) 1195 { 1196 int iov_ret, ooo_cmdsn = 0, ret; 1197 u8 data_crc_failed = 0; 1198 u32 checksum, iov_count = 0, padding = 0, rx_got = 0; 1199 u32 rx_size = 0, payload_length; 1200 struct iscsi_cmd *cmd = NULL; 1201 struct se_cmd *se_cmd; 1202 struct iscsi_data *hdr; 1203 struct kvec *iov; 1204 unsigned long flags; 1205 1206 hdr = (struct iscsi_data *) buf; 1207 payload_length = ntoh24(hdr->dlength); 1208 hdr->itt = be32_to_cpu(hdr->itt); 1209 hdr->ttt = be32_to_cpu(hdr->ttt); 1210 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 1211 hdr->datasn = be32_to_cpu(hdr->datasn); 1212 hdr->offset = be32_to_cpu(hdr->offset); 1213 1214 if (!payload_length) { 1215 pr_err("DataOUT payload is ZERO, protocol error.\n"); 1216 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1217 buf, conn); 1218 } 1219 1220 /* iSCSI write */ 1221 spin_lock_bh(&conn->sess->session_stats_lock); 1222 conn->sess->rx_data_octets += payload_length; 1223 if (conn->sess->se_sess->se_node_acl) { 1224 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock); 1225 conn->sess->se_sess->se_node_acl->write_bytes += payload_length; 1226 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock); 1227 } 1228 spin_unlock_bh(&conn->sess->session_stats_lock); 1229 1230 if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) { 1231 pr_err("DataSegmentLength: %u is greater than" 1232 " MaxRecvDataSegmentLength: %u\n", payload_length, 1233 conn->conn_ops->MaxRecvDataSegmentLength); 1234 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1235 buf, conn); 1236 } 1237 1238 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 1239 payload_length); 1240 if (!cmd) 1241 return 0; 1242 1243 pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x," 1244 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", 1245 hdr->itt, hdr->ttt, hdr->datasn, hdr->offset, 1246 payload_length, conn->cid); 1247 1248 if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) { 1249 pr_err("Command ITT: 0x%08x received DataOUT after" 1250 " last DataOUT received, dumping payload\n", 1251 cmd->init_task_tag); 1252 return iscsit_dump_data_payload(conn, payload_length, 1); 1253 } 1254 1255 if (cmd->data_direction != DMA_TO_DEVICE) { 1256 pr_err("Command ITT: 0x%08x received DataOUT for a" 1257 " NON-WRITE command.\n", cmd->init_task_tag); 1258 return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR, 1259 1, 0, buf, cmd); 1260 } 1261 se_cmd = &cmd->se_cmd; 1262 iscsit_mod_dataout_timer(cmd); 1263 1264 if ((hdr->offset + payload_length) > cmd->se_cmd.data_length) { 1265 pr_err("DataOut Offset: %u, Length %u greater than" 1266 " iSCSI Command EDTL %u, protocol error.\n", 1267 hdr->offset, payload_length, cmd->se_cmd.data_length); 1268 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID, 1269 1, 0, buf, cmd); 1270 } 1271 1272 if (cmd->unsolicited_data) { 1273 int dump_unsolicited_data = 0; 1274 1275 if (conn->sess->sess_ops->InitialR2T) { 1276 pr_err("Received unexpected unsolicited data" 1277 " while InitialR2T=Yes, protocol error.\n"); 1278 transport_send_check_condition_and_sense(&cmd->se_cmd, 1279 TCM_UNEXPECTED_UNSOLICITED_DATA, 0); 1280 return -1; 1281 } 1282 /* 1283 * Special case for dealing with Unsolicited DataOUT 1284 * and Unsupported SAM WRITE Opcodes and SE resource allocation 1285 * failures; 1286 */ 1287 1288 /* Something's amiss if we're not in WRITE_PENDING state... */ 1289 spin_lock_irqsave(&se_cmd->t_state_lock, flags); 1290 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING); 1291 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); 1292 1293 spin_lock_irqsave(&se_cmd->t_state_lock, flags); 1294 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) || 1295 (se_cmd->se_cmd_flags & SCF_SCSI_CDB_EXCEPTION)) 1296 dump_unsolicited_data = 1; 1297 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); 1298 1299 if (dump_unsolicited_data) { 1300 /* 1301 * Check if a delayed TASK_ABORTED status needs to 1302 * be sent now if the ISCSI_FLAG_CMD_FINAL has been 1303 * received with the unsolicitied data out. 1304 */ 1305 if (hdr->flags & ISCSI_FLAG_CMD_FINAL) 1306 iscsit_stop_dataout_timer(cmd); 1307 1308 transport_check_aborted_status(se_cmd, 1309 (hdr->flags & ISCSI_FLAG_CMD_FINAL)); 1310 return iscsit_dump_data_payload(conn, payload_length, 1); 1311 } 1312 } else { 1313 /* 1314 * For the normal solicited data path: 1315 * 1316 * Check for a delayed TASK_ABORTED status and dump any 1317 * incoming data out payload if one exists. Also, when the 1318 * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current 1319 * data out sequence, we decrement outstanding_r2ts. Once 1320 * outstanding_r2ts reaches zero, go ahead and send the delayed 1321 * TASK_ABORTED status. 1322 */ 1323 if (se_cmd->transport_state & CMD_T_ABORTED) { 1324 if (hdr->flags & ISCSI_FLAG_CMD_FINAL) 1325 if (--cmd->outstanding_r2ts < 1) { 1326 iscsit_stop_dataout_timer(cmd); 1327 transport_check_aborted_status( 1328 se_cmd, 1); 1329 } 1330 1331 return iscsit_dump_data_payload(conn, payload_length, 1); 1332 } 1333 } 1334 /* 1335 * Preform DataSN, DataSequenceInOrder, DataPDUInOrder, and 1336 * within-command recovery checks before receiving the payload. 1337 */ 1338 ret = iscsit_check_pre_dataout(cmd, buf); 1339 if (ret == DATAOUT_WITHIN_COMMAND_RECOVERY) 1340 return 0; 1341 else if (ret == DATAOUT_CANNOT_RECOVER) 1342 return -1; 1343 1344 rx_size += payload_length; 1345 iov = &cmd->iov_data[0]; 1346 1347 iov_ret = iscsit_map_iovec(cmd, iov, hdr->offset, payload_length); 1348 if (iov_ret < 0) 1349 return -1; 1350 1351 iov_count += iov_ret; 1352 1353 padding = ((-payload_length) & 3); 1354 if (padding != 0) { 1355 iov[iov_count].iov_base = cmd->pad_bytes; 1356 iov[iov_count++].iov_len = padding; 1357 rx_size += padding; 1358 pr_debug("Receiving %u padding bytes.\n", padding); 1359 } 1360 1361 if (conn->conn_ops->DataDigest) { 1362 iov[iov_count].iov_base = &checksum; 1363 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 1364 rx_size += ISCSI_CRC_LEN; 1365 } 1366 1367 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); 1368 1369 iscsit_unmap_iovec(cmd); 1370 1371 if (rx_got != rx_size) 1372 return -1; 1373 1374 if (conn->conn_ops->DataDigest) { 1375 u32 data_crc; 1376 1377 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd, 1378 hdr->offset, payload_length, padding, 1379 cmd->pad_bytes); 1380 1381 if (checksum != data_crc) { 1382 pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 1383 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x" 1384 " does not match computed 0x%08x\n", 1385 hdr->itt, hdr->offset, payload_length, 1386 hdr->datasn, checksum, data_crc); 1387 data_crc_failed = 1; 1388 } else { 1389 pr_debug("Got CRC32C DataDigest 0x%08x for" 1390 " %u bytes of Data Out\n", checksum, 1391 payload_length); 1392 } 1393 } 1394 /* 1395 * Increment post receive data and CRC values or perform 1396 * within-command recovery. 1397 */ 1398 ret = iscsit_check_post_dataout(cmd, buf, data_crc_failed); 1399 if ((ret == DATAOUT_NORMAL) || (ret == DATAOUT_WITHIN_COMMAND_RECOVERY)) 1400 return 0; 1401 else if (ret == DATAOUT_SEND_R2T) { 1402 iscsit_set_dataout_sequence_values(cmd); 1403 iscsit_build_r2ts_for_cmd(cmd, conn, false); 1404 } else if (ret == DATAOUT_SEND_TO_TRANSPORT) { 1405 /* 1406 * Handle extra special case for out of order 1407 * Unsolicited Data Out. 1408 */ 1409 spin_lock_bh(&cmd->istate_lock); 1410 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN); 1411 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1412 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1413 spin_unlock_bh(&cmd->istate_lock); 1414 1415 iscsit_stop_dataout_timer(cmd); 1416 return (!ooo_cmdsn) ? transport_generic_handle_data( 1417 &cmd->se_cmd) : 0; 1418 } else /* DATAOUT_CANNOT_RECOVER */ 1419 return -1; 1420 1421 return 0; 1422 } 1423 1424 static int iscsit_handle_nop_out( 1425 struct iscsi_conn *conn, 1426 unsigned char *buf) 1427 { 1428 unsigned char *ping_data = NULL; 1429 int cmdsn_ret, niov = 0, ret = 0, rx_got, rx_size; 1430 u32 checksum, data_crc, padding = 0, payload_length; 1431 struct iscsi_cmd *cmd = NULL; 1432 struct kvec *iov = NULL; 1433 struct iscsi_nopout *hdr; 1434 1435 hdr = (struct iscsi_nopout *) buf; 1436 payload_length = ntoh24(hdr->dlength); 1437 hdr->itt = be32_to_cpu(hdr->itt); 1438 hdr->ttt = be32_to_cpu(hdr->ttt); 1439 hdr->cmdsn = be32_to_cpu(hdr->cmdsn); 1440 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 1441 1442 if ((hdr->itt == 0xFFFFFFFF) && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1443 pr_err("NOPOUT ITT is reserved, but Immediate Bit is" 1444 " not set, protocol error.\n"); 1445 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1446 buf, conn); 1447 } 1448 1449 if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) { 1450 pr_err("NOPOUT Ping Data DataSegmentLength: %u is" 1451 " greater than MaxRecvDataSegmentLength: %u, protocol" 1452 " error.\n", payload_length, 1453 conn->conn_ops->MaxRecvDataSegmentLength); 1454 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1455 buf, conn); 1456 } 1457 1458 pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%09x," 1459 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n", 1460 (hdr->itt == 0xFFFFFFFF) ? "Response" : "Request", 1461 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn, 1462 payload_length); 1463 /* 1464 * This is not a response to a Unsolicited NopIN, which means 1465 * it can either be a NOPOUT ping request (with a valid ITT), 1466 * or a NOPOUT not requesting a NOPIN (with a reserved ITT). 1467 * Either way, make sure we allocate an struct iscsi_cmd, as both 1468 * can contain ping data. 1469 */ 1470 if (hdr->ttt == 0xFFFFFFFF) { 1471 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 1472 if (!cmd) 1473 return iscsit_add_reject( 1474 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1475 1, buf, conn); 1476 1477 cmd->iscsi_opcode = ISCSI_OP_NOOP_OUT; 1478 cmd->i_state = ISTATE_SEND_NOPIN; 1479 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1480 1 : 0); 1481 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 1482 cmd->targ_xfer_tag = 0xFFFFFFFF; 1483 cmd->cmd_sn = hdr->cmdsn; 1484 cmd->exp_stat_sn = hdr->exp_statsn; 1485 cmd->data_direction = DMA_NONE; 1486 } 1487 1488 if (payload_length && (hdr->ttt == 0xFFFFFFFF)) { 1489 rx_size = payload_length; 1490 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1491 if (!ping_data) { 1492 pr_err("Unable to allocate memory for" 1493 " NOPOUT ping data.\n"); 1494 ret = -1; 1495 goto out; 1496 } 1497 1498 iov = &cmd->iov_misc[0]; 1499 iov[niov].iov_base = ping_data; 1500 iov[niov++].iov_len = payload_length; 1501 1502 padding = ((-payload_length) & 3); 1503 if (padding != 0) { 1504 pr_debug("Receiving %u additional bytes" 1505 " for padding.\n", padding); 1506 iov[niov].iov_base = &cmd->pad_bytes; 1507 iov[niov++].iov_len = padding; 1508 rx_size += padding; 1509 } 1510 if (conn->conn_ops->DataDigest) { 1511 iov[niov].iov_base = &checksum; 1512 iov[niov++].iov_len = ISCSI_CRC_LEN; 1513 rx_size += ISCSI_CRC_LEN; 1514 } 1515 1516 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size); 1517 if (rx_got != rx_size) { 1518 ret = -1; 1519 goto out; 1520 } 1521 1522 if (conn->conn_ops->DataDigest) { 1523 iscsit_do_crypto_hash_buf(&conn->conn_rx_hash, 1524 ping_data, payload_length, 1525 padding, cmd->pad_bytes, 1526 (u8 *)&data_crc); 1527 1528 if (checksum != data_crc) { 1529 pr_err("Ping data CRC32C DataDigest" 1530 " 0x%08x does not match computed 0x%08x\n", 1531 checksum, data_crc); 1532 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1533 pr_err("Unable to recover from" 1534 " NOPOUT Ping DataCRC failure while in" 1535 " ERL=0.\n"); 1536 ret = -1; 1537 goto out; 1538 } else { 1539 /* 1540 * Silently drop this PDU and let the 1541 * initiator plug the CmdSN gap. 1542 */ 1543 pr_debug("Dropping NOPOUT" 1544 " Command CmdSN: 0x%08x due to" 1545 " DataCRC error.\n", hdr->cmdsn); 1546 ret = 0; 1547 goto out; 1548 } 1549 } else { 1550 pr_debug("Got CRC32C DataDigest" 1551 " 0x%08x for %u bytes of ping data.\n", 1552 checksum, payload_length); 1553 } 1554 } 1555 1556 ping_data[payload_length] = '\0'; 1557 /* 1558 * Attach ping data to struct iscsi_cmd->buf_ptr. 1559 */ 1560 cmd->buf_ptr = ping_data; 1561 cmd->buf_ptr_size = payload_length; 1562 1563 pr_debug("Got %u bytes of NOPOUT ping" 1564 " data.\n", payload_length); 1565 pr_debug("Ping Data: \"%s\"\n", ping_data); 1566 } 1567 1568 if (hdr->itt != 0xFFFFFFFF) { 1569 if (!cmd) { 1570 pr_err("Checking CmdSN for NOPOUT," 1571 " but cmd is NULL!\n"); 1572 return -1; 1573 } 1574 /* 1575 * Initiator is expecting a NopIN ping reply, 1576 */ 1577 spin_lock_bh(&conn->cmd_lock); 1578 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 1579 spin_unlock_bh(&conn->cmd_lock); 1580 1581 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn); 1582 1583 if (hdr->opcode & ISCSI_OP_IMMEDIATE) { 1584 iscsit_add_cmd_to_response_queue(cmd, conn, 1585 cmd->i_state); 1586 return 0; 1587 } 1588 1589 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 1590 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 1591 ret = 0; 1592 goto ping_out; 1593 } 1594 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1595 return iscsit_add_reject_from_cmd( 1596 ISCSI_REASON_PROTOCOL_ERROR, 1597 1, 0, buf, cmd); 1598 1599 return 0; 1600 } 1601 1602 if (hdr->ttt != 0xFFFFFFFF) { 1603 /* 1604 * This was a response to a unsolicited NOPIN ping. 1605 */ 1606 cmd = iscsit_find_cmd_from_ttt(conn, hdr->ttt); 1607 if (!cmd) 1608 return -1; 1609 1610 iscsit_stop_nopin_response_timer(conn); 1611 1612 cmd->i_state = ISTATE_REMOVE; 1613 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 1614 iscsit_start_nopin_timer(conn); 1615 } else { 1616 /* 1617 * Initiator is not expecting a NOPIN is response. 1618 * Just ignore for now. 1619 * 1620 * iSCSI v19-91 10.18 1621 * "A NOP-OUT may also be used to confirm a changed 1622 * ExpStatSN if another PDU will not be available 1623 * for a long time." 1624 */ 1625 ret = 0; 1626 goto out; 1627 } 1628 1629 return 0; 1630 out: 1631 if (cmd) 1632 iscsit_release_cmd(cmd); 1633 ping_out: 1634 kfree(ping_data); 1635 return ret; 1636 } 1637 1638 static int iscsit_handle_task_mgt_cmd( 1639 struct iscsi_conn *conn, 1640 unsigned char *buf) 1641 { 1642 struct iscsi_cmd *cmd; 1643 struct se_tmr_req *se_tmr; 1644 struct iscsi_tmr_req *tmr_req; 1645 struct iscsi_tm *hdr; 1646 int out_of_order_cmdsn = 0; 1647 int ret; 1648 u8 function; 1649 1650 hdr = (struct iscsi_tm *) buf; 1651 hdr->itt = be32_to_cpu(hdr->itt); 1652 hdr->rtt = be32_to_cpu(hdr->rtt); 1653 hdr->cmdsn = be32_to_cpu(hdr->cmdsn); 1654 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 1655 hdr->refcmdsn = be32_to_cpu(hdr->refcmdsn); 1656 hdr->exp_datasn = be32_to_cpu(hdr->exp_datasn); 1657 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL; 1658 function = hdr->flags; 1659 1660 pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:" 1661 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:" 1662 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function, 1663 hdr->rtt, hdr->refcmdsn, conn->cid); 1664 1665 if ((function != ISCSI_TM_FUNC_ABORT_TASK) && 1666 ((function != ISCSI_TM_FUNC_TASK_REASSIGN) && 1667 (hdr->rtt != ISCSI_RESERVED_TAG))) { 1668 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n"); 1669 hdr->rtt = ISCSI_RESERVED_TAG; 1670 } 1671 1672 if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) && 1673 !(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1674 pr_err("Task Management Request TASK_REASSIGN not" 1675 " issued as immediate command, bad iSCSI Initiator" 1676 "implementation\n"); 1677 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1678 buf, conn); 1679 } 1680 if ((function != ISCSI_TM_FUNC_ABORT_TASK) && 1681 (hdr->refcmdsn != ISCSI_RESERVED_TAG)) 1682 hdr->refcmdsn = ISCSI_RESERVED_TAG; 1683 1684 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 1685 if (!cmd) 1686 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1687 1, buf, conn); 1688 1689 cmd->data_direction = DMA_NONE; 1690 1691 cmd->tmr_req = kzalloc(sizeof(struct iscsi_tmr_req), GFP_KERNEL); 1692 if (!cmd->tmr_req) { 1693 pr_err("Unable to allocate memory for" 1694 " Task Management command!\n"); 1695 return iscsit_add_reject_from_cmd( 1696 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1697 1, 1, buf, cmd); 1698 } 1699 1700 /* 1701 * TASK_REASSIGN for ERL=2 / connection stays inside of 1702 * LIO-Target $FABRIC_MOD 1703 */ 1704 if (function != ISCSI_TM_FUNC_TASK_REASSIGN) { 1705 1706 u8 tcm_function; 1707 int ret; 1708 1709 transport_init_se_cmd(&cmd->se_cmd, 1710 &lio_target_fabric_configfs->tf_ops, 1711 conn->sess->se_sess, 0, DMA_NONE, 1712 MSG_SIMPLE_TAG, &cmd->sense_buffer[0]); 1713 1714 switch (function) { 1715 case ISCSI_TM_FUNC_ABORT_TASK: 1716 tcm_function = TMR_ABORT_TASK; 1717 break; 1718 case ISCSI_TM_FUNC_ABORT_TASK_SET: 1719 tcm_function = TMR_ABORT_TASK_SET; 1720 break; 1721 case ISCSI_TM_FUNC_CLEAR_ACA: 1722 tcm_function = TMR_CLEAR_ACA; 1723 break; 1724 case ISCSI_TM_FUNC_CLEAR_TASK_SET: 1725 tcm_function = TMR_CLEAR_TASK_SET; 1726 break; 1727 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET: 1728 tcm_function = TMR_LUN_RESET; 1729 break; 1730 case ISCSI_TM_FUNC_TARGET_WARM_RESET: 1731 tcm_function = TMR_TARGET_WARM_RESET; 1732 break; 1733 case ISCSI_TM_FUNC_TARGET_COLD_RESET: 1734 tcm_function = TMR_TARGET_COLD_RESET; 1735 break; 1736 default: 1737 pr_err("Unknown iSCSI TMR Function:" 1738 " 0x%02x\n", function); 1739 return iscsit_add_reject_from_cmd( 1740 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1741 1, 1, buf, cmd); 1742 } 1743 1744 ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req, 1745 tcm_function, GFP_KERNEL); 1746 if (ret < 0) 1747 return iscsit_add_reject_from_cmd( 1748 ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1749 1, 1, buf, cmd); 1750 1751 cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req; 1752 } 1753 1754 cmd->iscsi_opcode = ISCSI_OP_SCSI_TMFUNC; 1755 cmd->i_state = ISTATE_SEND_TASKMGTRSP; 1756 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 1757 cmd->init_task_tag = hdr->itt; 1758 cmd->targ_xfer_tag = 0xFFFFFFFF; 1759 cmd->cmd_sn = hdr->cmdsn; 1760 cmd->exp_stat_sn = hdr->exp_statsn; 1761 se_tmr = cmd->se_cmd.se_tmr_req; 1762 tmr_req = cmd->tmr_req; 1763 /* 1764 * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN 1765 */ 1766 if (function != ISCSI_TM_FUNC_TASK_REASSIGN) { 1767 ret = transport_lookup_tmr_lun(&cmd->se_cmd, 1768 scsilun_to_int(&hdr->lun)); 1769 if (ret < 0) { 1770 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; 1771 se_tmr->response = ISCSI_TMF_RSP_NO_LUN; 1772 goto attach; 1773 } 1774 } 1775 1776 switch (function) { 1777 case ISCSI_TM_FUNC_ABORT_TASK: 1778 se_tmr->response = iscsit_tmr_abort_task(cmd, buf); 1779 if (se_tmr->response != ISCSI_TMF_RSP_COMPLETE) { 1780 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; 1781 goto attach; 1782 } 1783 break; 1784 case ISCSI_TM_FUNC_ABORT_TASK_SET: 1785 case ISCSI_TM_FUNC_CLEAR_ACA: 1786 case ISCSI_TM_FUNC_CLEAR_TASK_SET: 1787 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET: 1788 break; 1789 case ISCSI_TM_FUNC_TARGET_WARM_RESET: 1790 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) { 1791 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; 1792 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED; 1793 goto attach; 1794 } 1795 break; 1796 case ISCSI_TM_FUNC_TARGET_COLD_RESET: 1797 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) { 1798 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; 1799 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED; 1800 goto attach; 1801 } 1802 break; 1803 case ISCSI_TM_FUNC_TASK_REASSIGN: 1804 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf); 1805 /* 1806 * Perform sanity checks on the ExpDataSN only if the 1807 * TASK_REASSIGN was successful. 1808 */ 1809 if (se_tmr->response != ISCSI_TMF_RSP_COMPLETE) 1810 break; 1811 1812 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0) 1813 return iscsit_add_reject_from_cmd( 1814 ISCSI_REASON_BOOKMARK_INVALID, 1, 1, 1815 buf, cmd); 1816 break; 1817 default: 1818 pr_err("Unknown TMR function: 0x%02x, protocol" 1819 " error.\n", function); 1820 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; 1821 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED; 1822 goto attach; 1823 } 1824 1825 if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) && 1826 (se_tmr->response == ISCSI_TMF_RSP_COMPLETE)) 1827 se_tmr->call_transport = 1; 1828 attach: 1829 spin_lock_bh(&conn->cmd_lock); 1830 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 1831 spin_unlock_bh(&conn->cmd_lock); 1832 1833 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 1834 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 1835 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP) 1836 out_of_order_cmdsn = 1; 1837 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) 1838 return 0; 1839 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 1840 return iscsit_add_reject_from_cmd( 1841 ISCSI_REASON_PROTOCOL_ERROR, 1842 1, 0, buf, cmd); 1843 } 1844 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn); 1845 1846 if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE)) 1847 return 0; 1848 /* 1849 * Found the referenced task, send to transport for processing. 1850 */ 1851 if (se_tmr->call_transport) 1852 return transport_generic_handle_tmr(&cmd->se_cmd); 1853 1854 /* 1855 * Could not find the referenced LUN, task, or Task Management 1856 * command not authorized or supported. Change state and 1857 * let the tx_thread send the response. 1858 * 1859 * For connection recovery, this is also the default action for 1860 * TMR TASK_REASSIGN. 1861 */ 1862 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 1863 return 0; 1864 } 1865 1866 /* #warning FIXME: Support Text Command parameters besides SendTargets */ 1867 static int iscsit_handle_text_cmd( 1868 struct iscsi_conn *conn, 1869 unsigned char *buf) 1870 { 1871 char *text_ptr, *text_in; 1872 int cmdsn_ret, niov = 0, rx_got, rx_size; 1873 u32 checksum = 0, data_crc = 0, payload_length; 1874 u32 padding = 0, pad_bytes = 0, text_length = 0; 1875 struct iscsi_cmd *cmd; 1876 struct kvec iov[3]; 1877 struct iscsi_text *hdr; 1878 1879 hdr = (struct iscsi_text *) buf; 1880 payload_length = ntoh24(hdr->dlength); 1881 hdr->itt = be32_to_cpu(hdr->itt); 1882 hdr->ttt = be32_to_cpu(hdr->ttt); 1883 hdr->cmdsn = be32_to_cpu(hdr->cmdsn); 1884 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 1885 1886 if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) { 1887 pr_err("Unable to accept text parameter length: %u" 1888 "greater than MaxRecvDataSegmentLength %u.\n", 1889 payload_length, conn->conn_ops->MaxRecvDataSegmentLength); 1890 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 1891 buf, conn); 1892 } 1893 1894 pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x," 1895 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn, 1896 hdr->exp_statsn, payload_length); 1897 1898 rx_size = text_length = payload_length; 1899 if (text_length) { 1900 text_in = kzalloc(text_length, GFP_KERNEL); 1901 if (!text_in) { 1902 pr_err("Unable to allocate memory for" 1903 " incoming text parameters\n"); 1904 return -1; 1905 } 1906 1907 memset(iov, 0, 3 * sizeof(struct kvec)); 1908 iov[niov].iov_base = text_in; 1909 iov[niov++].iov_len = text_length; 1910 1911 padding = ((-payload_length) & 3); 1912 if (padding != 0) { 1913 iov[niov].iov_base = &pad_bytes; 1914 iov[niov++].iov_len = padding; 1915 rx_size += padding; 1916 pr_debug("Receiving %u additional bytes" 1917 " for padding.\n", padding); 1918 } 1919 if (conn->conn_ops->DataDigest) { 1920 iov[niov].iov_base = &checksum; 1921 iov[niov++].iov_len = ISCSI_CRC_LEN; 1922 rx_size += ISCSI_CRC_LEN; 1923 } 1924 1925 rx_got = rx_data(conn, &iov[0], niov, rx_size); 1926 if (rx_got != rx_size) { 1927 kfree(text_in); 1928 return -1; 1929 } 1930 1931 if (conn->conn_ops->DataDigest) { 1932 iscsit_do_crypto_hash_buf(&conn->conn_rx_hash, 1933 text_in, text_length, 1934 padding, (u8 *)&pad_bytes, 1935 (u8 *)&data_crc); 1936 1937 if (checksum != data_crc) { 1938 pr_err("Text data CRC32C DataDigest" 1939 " 0x%08x does not match computed" 1940 " 0x%08x\n", checksum, data_crc); 1941 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1942 pr_err("Unable to recover from" 1943 " Text Data digest failure while in" 1944 " ERL=0.\n"); 1945 kfree(text_in); 1946 return -1; 1947 } else { 1948 /* 1949 * Silently drop this PDU and let the 1950 * initiator plug the CmdSN gap. 1951 */ 1952 pr_debug("Dropping Text" 1953 " Command CmdSN: 0x%08x due to" 1954 " DataCRC error.\n", hdr->cmdsn); 1955 kfree(text_in); 1956 return 0; 1957 } 1958 } else { 1959 pr_debug("Got CRC32C DataDigest" 1960 " 0x%08x for %u bytes of text data.\n", 1961 checksum, text_length); 1962 } 1963 } 1964 text_in[text_length - 1] = '\0'; 1965 pr_debug("Successfully read %d bytes of text" 1966 " data.\n", text_length); 1967 1968 if (strncmp("SendTargets", text_in, 11) != 0) { 1969 pr_err("Received Text Data that is not" 1970 " SendTargets, cannot continue.\n"); 1971 kfree(text_in); 1972 return -1; 1973 } 1974 text_ptr = strchr(text_in, '='); 1975 if (!text_ptr) { 1976 pr_err("No \"=\" separator found in Text Data," 1977 " cannot continue.\n"); 1978 kfree(text_in); 1979 return -1; 1980 } 1981 if (strncmp("=All", text_ptr, 4) != 0) { 1982 pr_err("Unable to locate All value for" 1983 " SendTargets key, cannot continue.\n"); 1984 kfree(text_in); 1985 return -1; 1986 } 1987 /*#warning Support SendTargets=(iSCSI Target Name/Nothing) values. */ 1988 kfree(text_in); 1989 } 1990 1991 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 1992 if (!cmd) 1993 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1994 1, buf, conn); 1995 1996 cmd->iscsi_opcode = ISCSI_OP_TEXT; 1997 cmd->i_state = ISTATE_SEND_TEXTRSP; 1998 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 1999 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 2000 cmd->targ_xfer_tag = 0xFFFFFFFF; 2001 cmd->cmd_sn = hdr->cmdsn; 2002 cmd->exp_stat_sn = hdr->exp_statsn; 2003 cmd->data_direction = DMA_NONE; 2004 2005 spin_lock_bh(&conn->cmd_lock); 2006 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 2007 spin_unlock_bh(&conn->cmd_lock); 2008 2009 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn); 2010 2011 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) { 2012 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 2013 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 2014 return iscsit_add_reject_from_cmd( 2015 ISCSI_REASON_PROTOCOL_ERROR, 2016 1, 0, buf, cmd); 2017 2018 return 0; 2019 } 2020 2021 return iscsit_execute_cmd(cmd, 0); 2022 } 2023 2024 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2025 { 2026 struct iscsi_conn *conn_p; 2027 struct iscsi_session *sess = conn->sess; 2028 2029 pr_debug("Received logout request CLOSESESSION on CID: %hu" 2030 " for SID: %u.\n", conn->cid, conn->sess->sid); 2031 2032 atomic_set(&sess->session_logout, 1); 2033 atomic_set(&conn->conn_logout_remove, 1); 2034 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION; 2035 2036 iscsit_inc_conn_usage_count(conn); 2037 iscsit_inc_session_usage_count(sess); 2038 2039 spin_lock_bh(&sess->conn_lock); 2040 list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) { 2041 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN) 2042 continue; 2043 2044 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 2045 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT; 2046 } 2047 spin_unlock_bh(&sess->conn_lock); 2048 2049 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2050 2051 return 0; 2052 } 2053 2054 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2055 { 2056 struct iscsi_conn *l_conn; 2057 struct iscsi_session *sess = conn->sess; 2058 2059 pr_debug("Received logout request CLOSECONNECTION for CID:" 2060 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); 2061 2062 /* 2063 * A Logout Request with a CLOSECONNECTION reason code for a CID 2064 * can arrive on a connection with a differing CID. 2065 */ 2066 if (conn->cid == cmd->logout_cid) { 2067 spin_lock_bh(&conn->state_lock); 2068 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 2069 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT; 2070 2071 atomic_set(&conn->conn_logout_remove, 1); 2072 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION; 2073 iscsit_inc_conn_usage_count(conn); 2074 2075 spin_unlock_bh(&conn->state_lock); 2076 } else { 2077 /* 2078 * Handle all different cid CLOSECONNECTION requests in 2079 * iscsit_logout_post_handler_diffcid() as to give enough 2080 * time for any non immediate command's CmdSN to be 2081 * acknowledged on the connection in question. 2082 * 2083 * Here we simply make sure the CID is still around. 2084 */ 2085 l_conn = iscsit_get_conn_from_cid(sess, 2086 cmd->logout_cid); 2087 if (!l_conn) { 2088 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND; 2089 iscsit_add_cmd_to_response_queue(cmd, conn, 2090 cmd->i_state); 2091 return 0; 2092 } 2093 2094 iscsit_dec_conn_usage_count(l_conn); 2095 } 2096 2097 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2098 2099 return 0; 2100 } 2101 2102 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2103 { 2104 struct iscsi_session *sess = conn->sess; 2105 2106 pr_debug("Received explicit REMOVECONNFORRECOVERY logout for" 2107 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); 2108 2109 if (sess->sess_ops->ErrorRecoveryLevel != 2) { 2110 pr_err("Received Logout Request REMOVECONNFORRECOVERY" 2111 " while ERL!=2.\n"); 2112 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED; 2113 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2114 return 0; 2115 } 2116 2117 if (conn->cid == cmd->logout_cid) { 2118 pr_err("Received Logout Request REMOVECONNFORRECOVERY" 2119 " with CID: %hu on CID: %hu, implementation error.\n", 2120 cmd->logout_cid, conn->cid); 2121 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED; 2122 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2123 return 0; 2124 } 2125 2126 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); 2127 2128 return 0; 2129 } 2130 2131 static int iscsit_handle_logout_cmd( 2132 struct iscsi_conn *conn, 2133 unsigned char *buf) 2134 { 2135 int cmdsn_ret, logout_remove = 0; 2136 u8 reason_code = 0; 2137 struct iscsi_cmd *cmd; 2138 struct iscsi_logout *hdr; 2139 struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn); 2140 2141 hdr = (struct iscsi_logout *) buf; 2142 reason_code = (hdr->flags & 0x7f); 2143 hdr->itt = be32_to_cpu(hdr->itt); 2144 hdr->cid = be16_to_cpu(hdr->cid); 2145 hdr->cmdsn = be32_to_cpu(hdr->cmdsn); 2146 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 2147 2148 if (tiqn) { 2149 spin_lock(&tiqn->logout_stats.lock); 2150 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) 2151 tiqn->logout_stats.normal_logouts++; 2152 else 2153 tiqn->logout_stats.abnormal_logouts++; 2154 spin_unlock(&tiqn->logout_stats.lock); 2155 } 2156 2157 pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x" 2158 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n", 2159 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code, 2160 hdr->cid, conn->cid); 2161 2162 if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) { 2163 pr_err("Received logout request on connection that" 2164 " is not in logged in state, ignoring request.\n"); 2165 return 0; 2166 } 2167 2168 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 2169 if (!cmd) 2170 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1, 2171 buf, conn); 2172 2173 cmd->iscsi_opcode = ISCSI_OP_LOGOUT; 2174 cmd->i_state = ISTATE_SEND_LOGOUTRSP; 2175 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); 2176 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt; 2177 cmd->targ_xfer_tag = 0xFFFFFFFF; 2178 cmd->cmd_sn = hdr->cmdsn; 2179 cmd->exp_stat_sn = hdr->exp_statsn; 2180 cmd->logout_cid = hdr->cid; 2181 cmd->logout_reason = reason_code; 2182 cmd->data_direction = DMA_NONE; 2183 2184 /* 2185 * We need to sleep in these cases (by returning 1) until the Logout 2186 * Response gets sent in the tx thread. 2187 */ 2188 if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) || 2189 ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) && 2190 (hdr->cid == conn->cid))) 2191 logout_remove = 1; 2192 2193 spin_lock_bh(&conn->cmd_lock); 2194 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); 2195 spin_unlock_bh(&conn->cmd_lock); 2196 2197 if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY) 2198 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn); 2199 2200 /* 2201 * Immediate commands are executed, well, immediately. 2202 * Non-Immediate Logout Commands are executed in CmdSN order. 2203 */ 2204 if (cmd->immediate_cmd) { 2205 int ret = iscsit_execute_cmd(cmd, 0); 2206 2207 if (ret < 0) 2208 return ret; 2209 } else { 2210 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); 2211 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 2212 logout_remove = 0; 2213 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) { 2214 return iscsit_add_reject_from_cmd( 2215 ISCSI_REASON_PROTOCOL_ERROR, 2216 1, 0, buf, cmd); 2217 } 2218 } 2219 2220 return logout_remove; 2221 } 2222 2223 static int iscsit_handle_snack( 2224 struct iscsi_conn *conn, 2225 unsigned char *buf) 2226 { 2227 struct iscsi_snack *hdr; 2228 2229 hdr = (struct iscsi_snack *) buf; 2230 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL; 2231 hdr->itt = be32_to_cpu(hdr->itt); 2232 hdr->ttt = be32_to_cpu(hdr->ttt); 2233 hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn); 2234 hdr->begrun = be32_to_cpu(hdr->begrun); 2235 hdr->runlength = be32_to_cpu(hdr->runlength); 2236 2237 pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:" 2238 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x," 2239 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags, 2240 hdr->begrun, hdr->runlength, conn->cid); 2241 2242 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2243 pr_err("Initiator sent SNACK request while in" 2244 " ErrorRecoveryLevel=0.\n"); 2245 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 2246 buf, conn); 2247 } 2248 /* 2249 * SNACK_DATA and SNACK_R2T are both 0, so check which function to 2250 * call from inside iscsi_send_recovery_datain_or_r2t(). 2251 */ 2252 switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) { 2253 case 0: 2254 return iscsit_handle_recovery_datain_or_r2t(conn, buf, 2255 hdr->itt, hdr->ttt, hdr->begrun, hdr->runlength); 2256 case ISCSI_FLAG_SNACK_TYPE_STATUS: 2257 return iscsit_handle_status_snack(conn, hdr->itt, hdr->ttt, 2258 hdr->begrun, hdr->runlength); 2259 case ISCSI_FLAG_SNACK_TYPE_DATA_ACK: 2260 return iscsit_handle_data_ack(conn, hdr->ttt, hdr->begrun, 2261 hdr->runlength); 2262 case ISCSI_FLAG_SNACK_TYPE_RDATA: 2263 /* FIXME: Support R-Data SNACK */ 2264 pr_err("R-Data SNACK Not Supported.\n"); 2265 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 2266 buf, conn); 2267 default: 2268 pr_err("Unknown SNACK type 0x%02x, protocol" 2269 " error.\n", hdr->flags & 0x0f); 2270 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 2271 buf, conn); 2272 } 2273 2274 return 0; 2275 } 2276 2277 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn) 2278 { 2279 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || 2280 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { 2281 wait_for_completion_interruptible_timeout( 2282 &conn->rx_half_close_comp, 2283 ISCSI_RX_THREAD_TCP_TIMEOUT * HZ); 2284 } 2285 } 2286 2287 static int iscsit_handle_immediate_data( 2288 struct iscsi_cmd *cmd, 2289 unsigned char *buf, 2290 u32 length) 2291 { 2292 int iov_ret, rx_got = 0, rx_size = 0; 2293 u32 checksum, iov_count = 0, padding = 0; 2294 struct iscsi_conn *conn = cmd->conn; 2295 struct kvec *iov; 2296 2297 iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, cmd->write_data_done, length); 2298 if (iov_ret < 0) 2299 return IMMEDIATE_DATA_CANNOT_RECOVER; 2300 2301 rx_size = length; 2302 iov_count = iov_ret; 2303 iov = &cmd->iov_data[0]; 2304 2305 padding = ((-length) & 3); 2306 if (padding != 0) { 2307 iov[iov_count].iov_base = cmd->pad_bytes; 2308 iov[iov_count++].iov_len = padding; 2309 rx_size += padding; 2310 } 2311 2312 if (conn->conn_ops->DataDigest) { 2313 iov[iov_count].iov_base = &checksum; 2314 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 2315 rx_size += ISCSI_CRC_LEN; 2316 } 2317 2318 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); 2319 2320 iscsit_unmap_iovec(cmd); 2321 2322 if (rx_got != rx_size) { 2323 iscsit_rx_thread_wait_for_tcp(conn); 2324 return IMMEDIATE_DATA_CANNOT_RECOVER; 2325 } 2326 2327 if (conn->conn_ops->DataDigest) { 2328 u32 data_crc; 2329 2330 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd, 2331 cmd->write_data_done, length, padding, 2332 cmd->pad_bytes); 2333 2334 if (checksum != data_crc) { 2335 pr_err("ImmediateData CRC32C DataDigest 0x%08x" 2336 " does not match computed 0x%08x\n", checksum, 2337 data_crc); 2338 2339 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 2340 pr_err("Unable to recover from" 2341 " Immediate Data digest failure while" 2342 " in ERL=0.\n"); 2343 iscsit_add_reject_from_cmd( 2344 ISCSI_REASON_DATA_DIGEST_ERROR, 2345 1, 0, buf, cmd); 2346 return IMMEDIATE_DATA_CANNOT_RECOVER; 2347 } else { 2348 iscsit_add_reject_from_cmd( 2349 ISCSI_REASON_DATA_DIGEST_ERROR, 2350 0, 0, buf, cmd); 2351 return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 2352 } 2353 } else { 2354 pr_debug("Got CRC32C DataDigest 0x%08x for" 2355 " %u bytes of Immediate Data\n", checksum, 2356 length); 2357 } 2358 } 2359 2360 cmd->write_data_done += length; 2361 2362 if (cmd->write_data_done == cmd->se_cmd.data_length) { 2363 spin_lock_bh(&cmd->istate_lock); 2364 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 2365 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 2366 spin_unlock_bh(&cmd->istate_lock); 2367 } 2368 2369 return IMMEDIATE_DATA_NORMAL_OPERATION; 2370 } 2371 2372 /* 2373 * Called with sess->conn_lock held. 2374 */ 2375 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections 2376 with active network interface */ 2377 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn) 2378 { 2379 struct iscsi_cmd *cmd; 2380 struct iscsi_conn *conn_p; 2381 2382 /* 2383 * Only send a Asynchronous Message on connections whos network 2384 * interface is still functional. 2385 */ 2386 list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) { 2387 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) { 2388 iscsit_inc_conn_usage_count(conn_p); 2389 break; 2390 } 2391 } 2392 2393 if (!conn_p) 2394 return; 2395 2396 cmd = iscsit_allocate_cmd(conn_p, GFP_KERNEL); 2397 if (!cmd) { 2398 iscsit_dec_conn_usage_count(conn_p); 2399 return; 2400 } 2401 2402 cmd->logout_cid = conn->cid; 2403 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT; 2404 cmd->i_state = ISTATE_SEND_ASYNCMSG; 2405 2406 spin_lock_bh(&conn_p->cmd_lock); 2407 list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list); 2408 spin_unlock_bh(&conn_p->cmd_lock); 2409 2410 iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state); 2411 iscsit_dec_conn_usage_count(conn_p); 2412 } 2413 2414 static int iscsit_send_conn_drop_async_message( 2415 struct iscsi_cmd *cmd, 2416 struct iscsi_conn *conn) 2417 { 2418 struct iscsi_async *hdr; 2419 2420 cmd->tx_size = ISCSI_HDR_LEN; 2421 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT; 2422 2423 hdr = (struct iscsi_async *) cmd->pdu; 2424 hdr->opcode = ISCSI_OP_ASYNC_EVENT; 2425 hdr->flags = ISCSI_FLAG_CMD_FINAL; 2426 cmd->init_task_tag = 0xFFFFFFFF; 2427 cmd->targ_xfer_tag = 0xFFFFFFFF; 2428 put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]); 2429 cmd->stat_sn = conn->stat_sn++; 2430 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2431 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2432 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2433 hdr->async_event = ISCSI_ASYNC_MSG_DROPPING_CONNECTION; 2434 hdr->param1 = cpu_to_be16(cmd->logout_cid); 2435 hdr->param2 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait); 2436 hdr->param3 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain); 2437 2438 if (conn->conn_ops->HeaderDigest) { 2439 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2440 2441 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2442 (unsigned char *)hdr, ISCSI_HDR_LEN, 2443 0, NULL, (u8 *)header_digest); 2444 2445 cmd->tx_size += ISCSI_CRC_LEN; 2446 pr_debug("Attaching CRC32C HeaderDigest to" 2447 " Async Message 0x%08x\n", *header_digest); 2448 } 2449 2450 cmd->iov_misc[0].iov_base = cmd->pdu; 2451 cmd->iov_misc[0].iov_len = cmd->tx_size; 2452 cmd->iov_misc_count = 1; 2453 2454 pr_debug("Sending Connection Dropped Async Message StatSN:" 2455 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn, 2456 cmd->logout_cid, conn->cid); 2457 return 0; 2458 } 2459 2460 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn) 2461 { 2462 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || 2463 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { 2464 wait_for_completion_interruptible_timeout( 2465 &conn->tx_half_close_comp, 2466 ISCSI_TX_THREAD_TCP_TIMEOUT * HZ); 2467 } 2468 } 2469 2470 static int iscsit_send_data_in( 2471 struct iscsi_cmd *cmd, 2472 struct iscsi_conn *conn) 2473 { 2474 int iov_ret = 0, set_statsn = 0; 2475 u32 iov_count = 0, tx_size = 0; 2476 struct iscsi_datain datain; 2477 struct iscsi_datain_req *dr; 2478 struct iscsi_data_rsp *hdr; 2479 struct kvec *iov; 2480 int eodr = 0; 2481 int ret; 2482 2483 memset(&datain, 0, sizeof(struct iscsi_datain)); 2484 dr = iscsit_get_datain_values(cmd, &datain); 2485 if (!dr) { 2486 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n", 2487 cmd->init_task_tag); 2488 return -1; 2489 } 2490 2491 /* 2492 * Be paranoid and double check the logic for now. 2493 */ 2494 if ((datain.offset + datain.length) > cmd->se_cmd.data_length) { 2495 pr_err("Command ITT: 0x%08x, datain.offset: %u and" 2496 " datain.length: %u exceeds cmd->data_length: %u\n", 2497 cmd->init_task_tag, datain.offset, datain.length, 2498 cmd->se_cmd.data_length); 2499 return -1; 2500 } 2501 2502 spin_lock_bh(&conn->sess->session_stats_lock); 2503 conn->sess->tx_data_octets += datain.length; 2504 if (conn->sess->se_sess->se_node_acl) { 2505 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock); 2506 conn->sess->se_sess->se_node_acl->read_bytes += datain.length; 2507 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock); 2508 } 2509 spin_unlock_bh(&conn->sess->session_stats_lock); 2510 /* 2511 * Special case for successfully execution w/ both DATAIN 2512 * and Sense Data. 2513 */ 2514 if ((datain.flags & ISCSI_FLAG_DATA_STATUS) && 2515 (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)) 2516 datain.flags &= ~ISCSI_FLAG_DATA_STATUS; 2517 else { 2518 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) || 2519 (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) { 2520 iscsit_increment_maxcmdsn(cmd, conn->sess); 2521 cmd->stat_sn = conn->stat_sn++; 2522 set_statsn = 1; 2523 } else if (dr->dr_complete == 2524 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY) 2525 set_statsn = 1; 2526 } 2527 2528 hdr = (struct iscsi_data_rsp *) cmd->pdu; 2529 memset(hdr, 0, ISCSI_HDR_LEN); 2530 hdr->opcode = ISCSI_OP_SCSI_DATA_IN; 2531 hdr->flags = datain.flags; 2532 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) { 2533 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { 2534 hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW; 2535 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 2536 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { 2537 hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW; 2538 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 2539 } 2540 } 2541 hton24(hdr->dlength, datain.length); 2542 if (hdr->flags & ISCSI_FLAG_DATA_ACK) 2543 int_to_scsilun(cmd->se_cmd.orig_fe_lun, 2544 (struct scsi_lun *)&hdr->lun); 2545 else 2546 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); 2547 2548 hdr->itt = cpu_to_be32(cmd->init_task_tag); 2549 hdr->ttt = (hdr->flags & ISCSI_FLAG_DATA_ACK) ? 2550 cpu_to_be32(cmd->targ_xfer_tag) : 2551 0xFFFFFFFF; 2552 hdr->statsn = (set_statsn) ? cpu_to_be32(cmd->stat_sn) : 2553 0xFFFFFFFF; 2554 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2555 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2556 hdr->datasn = cpu_to_be32(datain.data_sn); 2557 hdr->offset = cpu_to_be32(datain.offset); 2558 2559 iov = &cmd->iov_data[0]; 2560 iov[iov_count].iov_base = cmd->pdu; 2561 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 2562 tx_size += ISCSI_HDR_LEN; 2563 2564 if (conn->conn_ops->HeaderDigest) { 2565 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2566 2567 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2568 (unsigned char *)hdr, ISCSI_HDR_LEN, 2569 0, NULL, (u8 *)header_digest); 2570 2571 iov[0].iov_len += ISCSI_CRC_LEN; 2572 tx_size += ISCSI_CRC_LEN; 2573 2574 pr_debug("Attaching CRC32 HeaderDigest" 2575 " for DataIN PDU 0x%08x\n", *header_digest); 2576 } 2577 2578 iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], datain.offset, datain.length); 2579 if (iov_ret < 0) 2580 return -1; 2581 2582 iov_count += iov_ret; 2583 tx_size += datain.length; 2584 2585 cmd->padding = ((-datain.length) & 3); 2586 if (cmd->padding) { 2587 iov[iov_count].iov_base = cmd->pad_bytes; 2588 iov[iov_count++].iov_len = cmd->padding; 2589 tx_size += cmd->padding; 2590 2591 pr_debug("Attaching %u padding bytes\n", 2592 cmd->padding); 2593 } 2594 if (conn->conn_ops->DataDigest) { 2595 cmd->data_crc = iscsit_do_crypto_hash_sg(&conn->conn_tx_hash, cmd, 2596 datain.offset, datain.length, cmd->padding, cmd->pad_bytes); 2597 2598 iov[iov_count].iov_base = &cmd->data_crc; 2599 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 2600 tx_size += ISCSI_CRC_LEN; 2601 2602 pr_debug("Attached CRC32C DataDigest %d bytes, crc" 2603 " 0x%08x\n", datain.length+cmd->padding, cmd->data_crc); 2604 } 2605 2606 cmd->iov_data_count = iov_count; 2607 cmd->tx_size = tx_size; 2608 2609 pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x," 2610 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", 2611 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn), 2612 ntohl(hdr->offset), datain.length, conn->cid); 2613 2614 /* sendpage is preferred but can't insert markers */ 2615 if (!conn->conn_ops->IFMarker) 2616 ret = iscsit_fe_sendpage_sg(cmd, conn); 2617 else 2618 ret = iscsit_send_tx_data(cmd, conn, 0); 2619 2620 iscsit_unmap_iovec(cmd); 2621 2622 if (ret < 0) { 2623 iscsit_tx_thread_wait_for_tcp(conn); 2624 return ret; 2625 } 2626 2627 if (dr->dr_complete) { 2628 eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ? 2629 2 : 1; 2630 iscsit_free_datain_req(cmd, dr); 2631 } 2632 2633 return eodr; 2634 } 2635 2636 static int iscsit_send_logout_response( 2637 struct iscsi_cmd *cmd, 2638 struct iscsi_conn *conn) 2639 { 2640 int niov = 0, tx_size; 2641 struct iscsi_conn *logout_conn = NULL; 2642 struct iscsi_conn_recovery *cr = NULL; 2643 struct iscsi_session *sess = conn->sess; 2644 struct kvec *iov; 2645 struct iscsi_logout_rsp *hdr; 2646 /* 2647 * The actual shutting down of Sessions and/or Connections 2648 * for CLOSESESSION and CLOSECONNECTION Logout Requests 2649 * is done in scsi_logout_post_handler(). 2650 */ 2651 switch (cmd->logout_reason) { 2652 case ISCSI_LOGOUT_REASON_CLOSE_SESSION: 2653 pr_debug("iSCSI session logout successful, setting" 2654 " logout response to ISCSI_LOGOUT_SUCCESS.\n"); 2655 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2656 break; 2657 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: 2658 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND) 2659 break; 2660 /* 2661 * For CLOSECONNECTION logout requests carrying 2662 * a matching logout CID -> local CID, the reference 2663 * for the local CID will have been incremented in 2664 * iscsi_logout_closeconnection(). 2665 * 2666 * For CLOSECONNECTION logout requests carrying 2667 * a different CID than the connection it arrived 2668 * on, the connection responding to cmd->logout_cid 2669 * is stopped in iscsit_logout_post_handler_diffcid(). 2670 */ 2671 2672 pr_debug("iSCSI CID: %hu logout on CID: %hu" 2673 " successful.\n", cmd->logout_cid, conn->cid); 2674 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2675 break; 2676 case ISCSI_LOGOUT_REASON_RECOVERY: 2677 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) || 2678 (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED)) 2679 break; 2680 /* 2681 * If the connection is still active from our point of view 2682 * force connection recovery to occur. 2683 */ 2684 logout_conn = iscsit_get_conn_from_cid_rcfr(sess, 2685 cmd->logout_cid); 2686 if ((logout_conn)) { 2687 iscsit_connection_reinstatement_rcfr(logout_conn); 2688 iscsit_dec_conn_usage_count(logout_conn); 2689 } 2690 2691 cr = iscsit_get_inactive_connection_recovery_entry( 2692 conn->sess, cmd->logout_cid); 2693 if (!cr) { 2694 pr_err("Unable to locate CID: %hu for" 2695 " REMOVECONNFORRECOVERY Logout Request.\n", 2696 cmd->logout_cid); 2697 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND; 2698 break; 2699 } 2700 2701 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn); 2702 2703 pr_debug("iSCSI REMOVECONNFORRECOVERY logout" 2704 " for recovery for CID: %hu on CID: %hu successful.\n", 2705 cmd->logout_cid, conn->cid); 2706 cmd->logout_response = ISCSI_LOGOUT_SUCCESS; 2707 break; 2708 default: 2709 pr_err("Unknown cmd->logout_reason: 0x%02x\n", 2710 cmd->logout_reason); 2711 return -1; 2712 } 2713 2714 tx_size = ISCSI_HDR_LEN; 2715 hdr = (struct iscsi_logout_rsp *)cmd->pdu; 2716 memset(hdr, 0, ISCSI_HDR_LEN); 2717 hdr->opcode = ISCSI_OP_LOGOUT_RSP; 2718 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2719 hdr->response = cmd->logout_response; 2720 hdr->itt = cpu_to_be32(cmd->init_task_tag); 2721 cmd->stat_sn = conn->stat_sn++; 2722 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2723 2724 iscsit_increment_maxcmdsn(cmd, conn->sess); 2725 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2726 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2727 2728 iov = &cmd->iov_misc[0]; 2729 iov[niov].iov_base = cmd->pdu; 2730 iov[niov++].iov_len = ISCSI_HDR_LEN; 2731 2732 if (conn->conn_ops->HeaderDigest) { 2733 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2734 2735 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2736 (unsigned char *)hdr, ISCSI_HDR_LEN, 2737 0, NULL, (u8 *)header_digest); 2738 2739 iov[0].iov_len += ISCSI_CRC_LEN; 2740 tx_size += ISCSI_CRC_LEN; 2741 pr_debug("Attaching CRC32C HeaderDigest to" 2742 " Logout Response 0x%08x\n", *header_digest); 2743 } 2744 cmd->iov_misc_count = niov; 2745 cmd->tx_size = tx_size; 2746 2747 pr_debug("Sending Logout Response ITT: 0x%08x StatSN:" 2748 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n", 2749 cmd->init_task_tag, cmd->stat_sn, hdr->response, 2750 cmd->logout_cid, conn->cid); 2751 2752 return 0; 2753 } 2754 2755 /* 2756 * Unsolicited NOPIN, either requesting a response or not. 2757 */ 2758 static int iscsit_send_unsolicited_nopin( 2759 struct iscsi_cmd *cmd, 2760 struct iscsi_conn *conn, 2761 int want_response) 2762 { 2763 int tx_size = ISCSI_HDR_LEN; 2764 struct iscsi_nopin *hdr; 2765 int ret; 2766 2767 hdr = (struct iscsi_nopin *) cmd->pdu; 2768 memset(hdr, 0, ISCSI_HDR_LEN); 2769 hdr->opcode = ISCSI_OP_NOOP_IN; 2770 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2771 hdr->itt = cpu_to_be32(cmd->init_task_tag); 2772 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 2773 cmd->stat_sn = conn->stat_sn; 2774 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2775 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2776 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2777 2778 if (conn->conn_ops->HeaderDigest) { 2779 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2780 2781 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2782 (unsigned char *)hdr, ISCSI_HDR_LEN, 2783 0, NULL, (u8 *)header_digest); 2784 2785 tx_size += ISCSI_CRC_LEN; 2786 pr_debug("Attaching CRC32C HeaderDigest to" 2787 " NopIN 0x%08x\n", *header_digest); 2788 } 2789 2790 cmd->iov_misc[0].iov_base = cmd->pdu; 2791 cmd->iov_misc[0].iov_len = tx_size; 2792 cmd->iov_misc_count = 1; 2793 cmd->tx_size = tx_size; 2794 2795 pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:" 2796 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid); 2797 2798 ret = iscsit_send_tx_data(cmd, conn, 1); 2799 if (ret < 0) { 2800 iscsit_tx_thread_wait_for_tcp(conn); 2801 return ret; 2802 } 2803 2804 spin_lock_bh(&cmd->istate_lock); 2805 cmd->i_state = want_response ? 2806 ISTATE_SENT_NOPIN_WANT_RESPONSE : ISTATE_SENT_STATUS; 2807 spin_unlock_bh(&cmd->istate_lock); 2808 2809 return 0; 2810 } 2811 2812 static int iscsit_send_nopin_response( 2813 struct iscsi_cmd *cmd, 2814 struct iscsi_conn *conn) 2815 { 2816 int niov = 0, tx_size; 2817 u32 padding = 0; 2818 struct kvec *iov; 2819 struct iscsi_nopin *hdr; 2820 2821 tx_size = ISCSI_HDR_LEN; 2822 hdr = (struct iscsi_nopin *) cmd->pdu; 2823 memset(hdr, 0, ISCSI_HDR_LEN); 2824 hdr->opcode = ISCSI_OP_NOOP_IN; 2825 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2826 hton24(hdr->dlength, cmd->buf_ptr_size); 2827 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); 2828 hdr->itt = cpu_to_be32(cmd->init_task_tag); 2829 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 2830 cmd->stat_sn = conn->stat_sn++; 2831 hdr->statsn = cpu_to_be32(cmd->stat_sn); 2832 2833 iscsit_increment_maxcmdsn(cmd, conn->sess); 2834 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2835 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2836 2837 iov = &cmd->iov_misc[0]; 2838 iov[niov].iov_base = cmd->pdu; 2839 iov[niov++].iov_len = ISCSI_HDR_LEN; 2840 2841 if (conn->conn_ops->HeaderDigest) { 2842 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2843 2844 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2845 (unsigned char *)hdr, ISCSI_HDR_LEN, 2846 0, NULL, (u8 *)header_digest); 2847 2848 iov[0].iov_len += ISCSI_CRC_LEN; 2849 tx_size += ISCSI_CRC_LEN; 2850 pr_debug("Attaching CRC32C HeaderDigest" 2851 " to NopIn 0x%08x\n", *header_digest); 2852 } 2853 2854 /* 2855 * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr. 2856 * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size. 2857 */ 2858 if (cmd->buf_ptr_size) { 2859 iov[niov].iov_base = cmd->buf_ptr; 2860 iov[niov++].iov_len = cmd->buf_ptr_size; 2861 tx_size += cmd->buf_ptr_size; 2862 2863 pr_debug("Echoing back %u bytes of ping" 2864 " data.\n", cmd->buf_ptr_size); 2865 2866 padding = ((-cmd->buf_ptr_size) & 3); 2867 if (padding != 0) { 2868 iov[niov].iov_base = &cmd->pad_bytes; 2869 iov[niov++].iov_len = padding; 2870 tx_size += padding; 2871 pr_debug("Attaching %u additional" 2872 " padding bytes.\n", padding); 2873 } 2874 if (conn->conn_ops->DataDigest) { 2875 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2876 cmd->buf_ptr, cmd->buf_ptr_size, 2877 padding, (u8 *)&cmd->pad_bytes, 2878 (u8 *)&cmd->data_crc); 2879 2880 iov[niov].iov_base = &cmd->data_crc; 2881 iov[niov++].iov_len = ISCSI_CRC_LEN; 2882 tx_size += ISCSI_CRC_LEN; 2883 pr_debug("Attached DataDigest for %u" 2884 " bytes of ping data, CRC 0x%08x\n", 2885 cmd->buf_ptr_size, cmd->data_crc); 2886 } 2887 } 2888 2889 cmd->iov_misc_count = niov; 2890 cmd->tx_size = tx_size; 2891 2892 pr_debug("Sending NOPIN Response ITT: 0x%08x, TTT:" 2893 " 0x%08x, StatSN: 0x%08x, Length %u\n", cmd->init_task_tag, 2894 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size); 2895 2896 return 0; 2897 } 2898 2899 static int iscsit_send_r2t( 2900 struct iscsi_cmd *cmd, 2901 struct iscsi_conn *conn) 2902 { 2903 int tx_size = 0; 2904 struct iscsi_r2t *r2t; 2905 struct iscsi_r2t_rsp *hdr; 2906 int ret; 2907 2908 r2t = iscsit_get_r2t_from_list(cmd); 2909 if (!r2t) 2910 return -1; 2911 2912 hdr = (struct iscsi_r2t_rsp *) cmd->pdu; 2913 memset(hdr, 0, ISCSI_HDR_LEN); 2914 hdr->opcode = ISCSI_OP_R2T; 2915 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 2916 int_to_scsilun(cmd->se_cmd.orig_fe_lun, 2917 (struct scsi_lun *)&hdr->lun); 2918 hdr->itt = cpu_to_be32(cmd->init_task_tag); 2919 spin_lock_bh(&conn->sess->ttt_lock); 2920 r2t->targ_xfer_tag = conn->sess->targ_xfer_tag++; 2921 if (r2t->targ_xfer_tag == 0xFFFFFFFF) 2922 r2t->targ_xfer_tag = conn->sess->targ_xfer_tag++; 2923 spin_unlock_bh(&conn->sess->ttt_lock); 2924 hdr->ttt = cpu_to_be32(r2t->targ_xfer_tag); 2925 hdr->statsn = cpu_to_be32(conn->stat_sn); 2926 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 2927 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 2928 hdr->r2tsn = cpu_to_be32(r2t->r2t_sn); 2929 hdr->data_offset = cpu_to_be32(r2t->offset); 2930 hdr->data_length = cpu_to_be32(r2t->xfer_len); 2931 2932 cmd->iov_misc[0].iov_base = cmd->pdu; 2933 cmd->iov_misc[0].iov_len = ISCSI_HDR_LEN; 2934 tx_size += ISCSI_HDR_LEN; 2935 2936 if (conn->conn_ops->HeaderDigest) { 2937 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 2938 2939 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 2940 (unsigned char *)hdr, ISCSI_HDR_LEN, 2941 0, NULL, (u8 *)header_digest); 2942 2943 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN; 2944 tx_size += ISCSI_CRC_LEN; 2945 pr_debug("Attaching CRC32 HeaderDigest for R2T" 2946 " PDU 0x%08x\n", *header_digest); 2947 } 2948 2949 pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:" 2950 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n", 2951 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag, 2952 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn, 2953 r2t->offset, r2t->xfer_len, conn->cid); 2954 2955 cmd->iov_misc_count = 1; 2956 cmd->tx_size = tx_size; 2957 2958 spin_lock_bh(&cmd->r2t_lock); 2959 r2t->sent_r2t = 1; 2960 spin_unlock_bh(&cmd->r2t_lock); 2961 2962 ret = iscsit_send_tx_data(cmd, conn, 1); 2963 if (ret < 0) { 2964 iscsit_tx_thread_wait_for_tcp(conn); 2965 return ret; 2966 } 2967 2968 spin_lock_bh(&cmd->dataout_timeout_lock); 2969 iscsit_start_dataout_timer(cmd, conn); 2970 spin_unlock_bh(&cmd->dataout_timeout_lock); 2971 2972 return 0; 2973 } 2974 2975 /* 2976 * @recovery: If called from iscsi_task_reassign_complete_write() for 2977 * connection recovery. 2978 */ 2979 int iscsit_build_r2ts_for_cmd( 2980 struct iscsi_cmd *cmd, 2981 struct iscsi_conn *conn, 2982 bool recovery) 2983 { 2984 int first_r2t = 1; 2985 u32 offset = 0, xfer_len = 0; 2986 2987 spin_lock_bh(&cmd->r2t_lock); 2988 if (cmd->cmd_flags & ICF_SENT_LAST_R2T) { 2989 spin_unlock_bh(&cmd->r2t_lock); 2990 return 0; 2991 } 2992 2993 if (conn->sess->sess_ops->DataSequenceInOrder && 2994 !recovery) 2995 cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done); 2996 2997 while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) { 2998 if (conn->sess->sess_ops->DataSequenceInOrder) { 2999 offset = cmd->r2t_offset; 3000 3001 if (first_r2t && recovery) { 3002 int new_data_end = offset + 3003 conn->sess->sess_ops->MaxBurstLength - 3004 cmd->next_burst_len; 3005 3006 if (new_data_end > cmd->se_cmd.data_length) 3007 xfer_len = cmd->se_cmd.data_length - offset; 3008 else 3009 xfer_len = 3010 conn->sess->sess_ops->MaxBurstLength - 3011 cmd->next_burst_len; 3012 } else { 3013 int new_data_end = offset + 3014 conn->sess->sess_ops->MaxBurstLength; 3015 3016 if (new_data_end > cmd->se_cmd.data_length) 3017 xfer_len = cmd->se_cmd.data_length - offset; 3018 else 3019 xfer_len = conn->sess->sess_ops->MaxBurstLength; 3020 } 3021 cmd->r2t_offset += xfer_len; 3022 3023 if (cmd->r2t_offset == cmd->se_cmd.data_length) 3024 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 3025 } else { 3026 struct iscsi_seq *seq; 3027 3028 seq = iscsit_get_seq_holder_for_r2t(cmd); 3029 if (!seq) { 3030 spin_unlock_bh(&cmd->r2t_lock); 3031 return -1; 3032 } 3033 3034 offset = seq->offset; 3035 xfer_len = seq->xfer_len; 3036 3037 if (cmd->seq_send_order == cmd->seq_count) 3038 cmd->cmd_flags |= ICF_SENT_LAST_R2T; 3039 } 3040 cmd->outstanding_r2ts++; 3041 first_r2t = 0; 3042 3043 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) { 3044 spin_unlock_bh(&cmd->r2t_lock); 3045 return -1; 3046 } 3047 3048 if (cmd->cmd_flags & ICF_SENT_LAST_R2T) 3049 break; 3050 } 3051 spin_unlock_bh(&cmd->r2t_lock); 3052 3053 return 0; 3054 } 3055 3056 static int iscsit_send_status( 3057 struct iscsi_cmd *cmd, 3058 struct iscsi_conn *conn) 3059 { 3060 u8 iov_count = 0, recovery; 3061 u32 padding = 0, tx_size = 0; 3062 struct iscsi_scsi_rsp *hdr; 3063 struct kvec *iov; 3064 3065 recovery = (cmd->i_state != ISTATE_SEND_STATUS); 3066 if (!recovery) 3067 cmd->stat_sn = conn->stat_sn++; 3068 3069 spin_lock_bh(&conn->sess->session_stats_lock); 3070 conn->sess->rsp_pdus++; 3071 spin_unlock_bh(&conn->sess->session_stats_lock); 3072 3073 hdr = (struct iscsi_scsi_rsp *) cmd->pdu; 3074 memset(hdr, 0, ISCSI_HDR_LEN); 3075 hdr->opcode = ISCSI_OP_SCSI_CMD_RSP; 3076 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3077 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { 3078 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW; 3079 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 3080 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { 3081 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW; 3082 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); 3083 } 3084 hdr->response = cmd->iscsi_response; 3085 hdr->cmd_status = cmd->se_cmd.scsi_status; 3086 hdr->itt = cpu_to_be32(cmd->init_task_tag); 3087 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3088 3089 iscsit_increment_maxcmdsn(cmd, conn->sess); 3090 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3091 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 3092 3093 iov = &cmd->iov_misc[0]; 3094 iov[iov_count].iov_base = cmd->pdu; 3095 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3096 tx_size += ISCSI_HDR_LEN; 3097 3098 /* 3099 * Attach SENSE DATA payload to iSCSI Response PDU 3100 */ 3101 if (cmd->se_cmd.sense_buffer && 3102 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 3103 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 3104 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 3105 hton24(hdr->dlength, cmd->se_cmd.scsi_sense_length); 3106 iov[iov_count].iov_base = cmd->se_cmd.sense_buffer; 3107 iov[iov_count++].iov_len = 3108 (cmd->se_cmd.scsi_sense_length + padding); 3109 tx_size += cmd->se_cmd.scsi_sense_length; 3110 3111 if (padding) { 3112 memset(cmd->se_cmd.sense_buffer + 3113 cmd->se_cmd.scsi_sense_length, 0, padding); 3114 tx_size += padding; 3115 pr_debug("Adding %u bytes of padding to" 3116 " SENSE.\n", padding); 3117 } 3118 3119 if (conn->conn_ops->DataDigest) { 3120 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3121 cmd->se_cmd.sense_buffer, 3122 (cmd->se_cmd.scsi_sense_length + padding), 3123 0, NULL, (u8 *)&cmd->data_crc); 3124 3125 iov[iov_count].iov_base = &cmd->data_crc; 3126 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 3127 tx_size += ISCSI_CRC_LEN; 3128 3129 pr_debug("Attaching CRC32 DataDigest for" 3130 " SENSE, %u bytes CRC 0x%08x\n", 3131 (cmd->se_cmd.scsi_sense_length + padding), 3132 cmd->data_crc); 3133 } 3134 3135 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI" 3136 " Response PDU\n", 3137 cmd->se_cmd.scsi_sense_length); 3138 } 3139 3140 if (conn->conn_ops->HeaderDigest) { 3141 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3142 3143 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3144 (unsigned char *)hdr, ISCSI_HDR_LEN, 3145 0, NULL, (u8 *)header_digest); 3146 3147 iov[0].iov_len += ISCSI_CRC_LEN; 3148 tx_size += ISCSI_CRC_LEN; 3149 pr_debug("Attaching CRC32 HeaderDigest for Response" 3150 " PDU 0x%08x\n", *header_digest); 3151 } 3152 3153 cmd->iov_misc_count = iov_count; 3154 cmd->tx_size = tx_size; 3155 3156 pr_debug("Built %sSCSI Response, ITT: 0x%08x, StatSN: 0x%08x," 3157 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n", 3158 (!recovery) ? "" : "Recovery ", cmd->init_task_tag, 3159 cmd->stat_sn, 0x00, cmd->se_cmd.scsi_status, conn->cid); 3160 3161 return 0; 3162 } 3163 3164 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr) 3165 { 3166 switch (se_tmr->response) { 3167 case TMR_FUNCTION_COMPLETE: 3168 return ISCSI_TMF_RSP_COMPLETE; 3169 case TMR_TASK_DOES_NOT_EXIST: 3170 return ISCSI_TMF_RSP_NO_TASK; 3171 case TMR_LUN_DOES_NOT_EXIST: 3172 return ISCSI_TMF_RSP_NO_LUN; 3173 case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED: 3174 return ISCSI_TMF_RSP_NOT_SUPPORTED; 3175 case TMR_FUNCTION_AUTHORIZATION_FAILED: 3176 return ISCSI_TMF_RSP_AUTH_FAILED; 3177 case TMR_FUNCTION_REJECTED: 3178 default: 3179 return ISCSI_TMF_RSP_REJECTED; 3180 } 3181 } 3182 3183 static int iscsit_send_task_mgt_rsp( 3184 struct iscsi_cmd *cmd, 3185 struct iscsi_conn *conn) 3186 { 3187 struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req; 3188 struct iscsi_tm_rsp *hdr; 3189 u32 tx_size = 0; 3190 3191 hdr = (struct iscsi_tm_rsp *) cmd->pdu; 3192 memset(hdr, 0, ISCSI_HDR_LEN); 3193 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP; 3194 hdr->flags = ISCSI_FLAG_CMD_FINAL; 3195 hdr->response = iscsit_convert_tcm_tmr_rsp(se_tmr); 3196 hdr->itt = cpu_to_be32(cmd->init_task_tag); 3197 cmd->stat_sn = conn->stat_sn++; 3198 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3199 3200 iscsit_increment_maxcmdsn(cmd, conn->sess); 3201 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3202 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 3203 3204 cmd->iov_misc[0].iov_base = cmd->pdu; 3205 cmd->iov_misc[0].iov_len = ISCSI_HDR_LEN; 3206 tx_size += ISCSI_HDR_LEN; 3207 3208 if (conn->conn_ops->HeaderDigest) { 3209 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3210 3211 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3212 (unsigned char *)hdr, ISCSI_HDR_LEN, 3213 0, NULL, (u8 *)header_digest); 3214 3215 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN; 3216 tx_size += ISCSI_CRC_LEN; 3217 pr_debug("Attaching CRC32 HeaderDigest for Task" 3218 " Mgmt Response PDU 0x%08x\n", *header_digest); 3219 } 3220 3221 cmd->iov_misc_count = 1; 3222 cmd->tx_size = tx_size; 3223 3224 pr_debug("Built Task Management Response ITT: 0x%08x," 3225 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n", 3226 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid); 3227 3228 return 0; 3229 } 3230 3231 static bool iscsit_check_inaddr_any(struct iscsi_np *np) 3232 { 3233 bool ret = false; 3234 3235 if (np->np_sockaddr.ss_family == AF_INET6) { 3236 const struct sockaddr_in6 sin6 = { 3237 .sin6_addr = IN6ADDR_ANY_INIT }; 3238 struct sockaddr_in6 *sock_in6 = 3239 (struct sockaddr_in6 *)&np->np_sockaddr; 3240 3241 if (!memcmp(sock_in6->sin6_addr.s6_addr, 3242 sin6.sin6_addr.s6_addr, 16)) 3243 ret = true; 3244 } else { 3245 struct sockaddr_in * sock_in = 3246 (struct sockaddr_in *)&np->np_sockaddr; 3247 3248 if (sock_in->sin_addr.s_addr == INADDR_ANY) 3249 ret = true; 3250 } 3251 3252 return ret; 3253 } 3254 3255 #define SENDTARGETS_BUF_LIMIT 32768U 3256 3257 static int iscsit_build_sendtargets_response(struct iscsi_cmd *cmd) 3258 { 3259 char *payload = NULL; 3260 struct iscsi_conn *conn = cmd->conn; 3261 struct iscsi_portal_group *tpg; 3262 struct iscsi_tiqn *tiqn; 3263 struct iscsi_tpg_np *tpg_np; 3264 int buffer_len, end_of_buf = 0, len = 0, payload_len = 0; 3265 unsigned char buf[ISCSI_IQN_LEN+12]; /* iqn + "TargetName=" + \0 */ 3266 3267 buffer_len = max(conn->conn_ops->MaxRecvDataSegmentLength, 3268 SENDTARGETS_BUF_LIMIT); 3269 3270 payload = kzalloc(buffer_len, GFP_KERNEL); 3271 if (!payload) { 3272 pr_err("Unable to allocate memory for sendtargets" 3273 " response.\n"); 3274 return -ENOMEM; 3275 } 3276 3277 spin_lock(&tiqn_lock); 3278 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) { 3279 len = sprintf(buf, "TargetName=%s", tiqn->tiqn); 3280 len += 1; 3281 3282 if ((len + payload_len) > buffer_len) { 3283 spin_unlock(&tiqn->tiqn_tpg_lock); 3284 end_of_buf = 1; 3285 goto eob; 3286 } 3287 memcpy(payload + payload_len, buf, len); 3288 payload_len += len; 3289 3290 spin_lock(&tiqn->tiqn_tpg_lock); 3291 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) { 3292 3293 spin_lock(&tpg->tpg_state_lock); 3294 if ((tpg->tpg_state == TPG_STATE_FREE) || 3295 (tpg->tpg_state == TPG_STATE_INACTIVE)) { 3296 spin_unlock(&tpg->tpg_state_lock); 3297 continue; 3298 } 3299 spin_unlock(&tpg->tpg_state_lock); 3300 3301 spin_lock(&tpg->tpg_np_lock); 3302 list_for_each_entry(tpg_np, &tpg->tpg_gnp_list, 3303 tpg_np_list) { 3304 struct iscsi_np *np = tpg_np->tpg_np; 3305 bool inaddr_any = iscsit_check_inaddr_any(np); 3306 3307 len = sprintf(buf, "TargetAddress=" 3308 "%s%s%s:%hu,%hu", 3309 (np->np_sockaddr.ss_family == AF_INET6) ? 3310 "[" : "", (inaddr_any == false) ? 3311 np->np_ip : conn->local_ip, 3312 (np->np_sockaddr.ss_family == AF_INET6) ? 3313 "]" : "", (inaddr_any == false) ? 3314 np->np_port : conn->local_port, 3315 tpg->tpgt); 3316 len += 1; 3317 3318 if ((len + payload_len) > buffer_len) { 3319 spin_unlock(&tpg->tpg_np_lock); 3320 spin_unlock(&tiqn->tiqn_tpg_lock); 3321 end_of_buf = 1; 3322 goto eob; 3323 } 3324 memcpy(payload + payload_len, buf, len); 3325 payload_len += len; 3326 } 3327 spin_unlock(&tpg->tpg_np_lock); 3328 } 3329 spin_unlock(&tiqn->tiqn_tpg_lock); 3330 eob: 3331 if (end_of_buf) 3332 break; 3333 } 3334 spin_unlock(&tiqn_lock); 3335 3336 cmd->buf_ptr = payload; 3337 3338 return payload_len; 3339 } 3340 3341 /* 3342 * FIXME: Add support for F_BIT and C_BIT when the length is longer than 3343 * MaxRecvDataSegmentLength. 3344 */ 3345 static int iscsit_send_text_rsp( 3346 struct iscsi_cmd *cmd, 3347 struct iscsi_conn *conn) 3348 { 3349 struct iscsi_text_rsp *hdr; 3350 struct kvec *iov; 3351 u32 padding = 0, tx_size = 0; 3352 int text_length, iov_count = 0; 3353 3354 text_length = iscsit_build_sendtargets_response(cmd); 3355 if (text_length < 0) 3356 return text_length; 3357 3358 padding = ((-text_length) & 3); 3359 if (padding != 0) { 3360 memset(cmd->buf_ptr + text_length, 0, padding); 3361 pr_debug("Attaching %u additional bytes for" 3362 " padding.\n", padding); 3363 } 3364 3365 hdr = (struct iscsi_text_rsp *) cmd->pdu; 3366 memset(hdr, 0, ISCSI_HDR_LEN); 3367 hdr->opcode = ISCSI_OP_TEXT_RSP; 3368 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3369 hton24(hdr->dlength, text_length); 3370 hdr->itt = cpu_to_be32(cmd->init_task_tag); 3371 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); 3372 cmd->stat_sn = conn->stat_sn++; 3373 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3374 3375 iscsit_increment_maxcmdsn(cmd, conn->sess); 3376 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3377 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 3378 3379 iov = &cmd->iov_misc[0]; 3380 3381 iov[iov_count].iov_base = cmd->pdu; 3382 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3383 iov[iov_count].iov_base = cmd->buf_ptr; 3384 iov[iov_count++].iov_len = text_length + padding; 3385 3386 tx_size += (ISCSI_HDR_LEN + text_length + padding); 3387 3388 if (conn->conn_ops->HeaderDigest) { 3389 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3390 3391 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3392 (unsigned char *)hdr, ISCSI_HDR_LEN, 3393 0, NULL, (u8 *)header_digest); 3394 3395 iov[0].iov_len += ISCSI_CRC_LEN; 3396 tx_size += ISCSI_CRC_LEN; 3397 pr_debug("Attaching CRC32 HeaderDigest for" 3398 " Text Response PDU 0x%08x\n", *header_digest); 3399 } 3400 3401 if (conn->conn_ops->DataDigest) { 3402 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3403 cmd->buf_ptr, (text_length + padding), 3404 0, NULL, (u8 *)&cmd->data_crc); 3405 3406 iov[iov_count].iov_base = &cmd->data_crc; 3407 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 3408 tx_size += ISCSI_CRC_LEN; 3409 3410 pr_debug("Attaching DataDigest for %u bytes of text" 3411 " data, CRC 0x%08x\n", (text_length + padding), 3412 cmd->data_crc); 3413 } 3414 3415 cmd->iov_misc_count = iov_count; 3416 cmd->tx_size = tx_size; 3417 3418 pr_debug("Built Text Response: ITT: 0x%08x, StatSN: 0x%08x," 3419 " Length: %u, CID: %hu\n", cmd->init_task_tag, cmd->stat_sn, 3420 text_length, conn->cid); 3421 return 0; 3422 } 3423 3424 static int iscsit_send_reject( 3425 struct iscsi_cmd *cmd, 3426 struct iscsi_conn *conn) 3427 { 3428 u32 iov_count = 0, tx_size = 0; 3429 struct iscsi_reject *hdr; 3430 struct kvec *iov; 3431 3432 hdr = (struct iscsi_reject *) cmd->pdu; 3433 hdr->opcode = ISCSI_OP_REJECT; 3434 hdr->flags |= ISCSI_FLAG_CMD_FINAL; 3435 hton24(hdr->dlength, ISCSI_HDR_LEN); 3436 cmd->stat_sn = conn->stat_sn++; 3437 hdr->statsn = cpu_to_be32(cmd->stat_sn); 3438 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); 3439 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); 3440 3441 iov = &cmd->iov_misc[0]; 3442 3443 iov[iov_count].iov_base = cmd->pdu; 3444 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3445 iov[iov_count].iov_base = cmd->buf_ptr; 3446 iov[iov_count++].iov_len = ISCSI_HDR_LEN; 3447 3448 tx_size = (ISCSI_HDR_LEN + ISCSI_HDR_LEN); 3449 3450 if (conn->conn_ops->HeaderDigest) { 3451 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; 3452 3453 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3454 (unsigned char *)hdr, ISCSI_HDR_LEN, 3455 0, NULL, (u8 *)header_digest); 3456 3457 iov[0].iov_len += ISCSI_CRC_LEN; 3458 tx_size += ISCSI_CRC_LEN; 3459 pr_debug("Attaching CRC32 HeaderDigest for" 3460 " REJECT PDU 0x%08x\n", *header_digest); 3461 } 3462 3463 if (conn->conn_ops->DataDigest) { 3464 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, 3465 (unsigned char *)cmd->buf_ptr, ISCSI_HDR_LEN, 3466 0, NULL, (u8 *)&cmd->data_crc); 3467 3468 iov[iov_count].iov_base = &cmd->data_crc; 3469 iov[iov_count++].iov_len = ISCSI_CRC_LEN; 3470 tx_size += ISCSI_CRC_LEN; 3471 pr_debug("Attaching CRC32 DataDigest for REJECT" 3472 " PDU 0x%08x\n", cmd->data_crc); 3473 } 3474 3475 cmd->iov_misc_count = iov_count; 3476 cmd->tx_size = tx_size; 3477 3478 pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x," 3479 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid); 3480 3481 return 0; 3482 } 3483 3484 void iscsit_thread_get_cpumask(struct iscsi_conn *conn) 3485 { 3486 struct iscsi_thread_set *ts = conn->thread_set; 3487 int ord, cpu; 3488 /* 3489 * thread_id is assigned from iscsit_global->ts_bitmap from 3490 * within iscsi_thread_set.c:iscsi_allocate_thread_sets() 3491 * 3492 * Here we use thread_id to determine which CPU that this 3493 * iSCSI connection's iscsi_thread_set will be scheduled to 3494 * execute upon. 3495 */ 3496 ord = ts->thread_id % cpumask_weight(cpu_online_mask); 3497 for_each_online_cpu(cpu) { 3498 if (ord-- == 0) { 3499 cpumask_set_cpu(cpu, conn->conn_cpumask); 3500 return; 3501 } 3502 } 3503 /* 3504 * This should never be reached.. 3505 */ 3506 dump_stack(); 3507 cpumask_setall(conn->conn_cpumask); 3508 } 3509 3510 static inline void iscsit_thread_check_cpumask( 3511 struct iscsi_conn *conn, 3512 struct task_struct *p, 3513 int mode) 3514 { 3515 char buf[128]; 3516 /* 3517 * mode == 1 signals iscsi_target_tx_thread() usage. 3518 * mode == 0 signals iscsi_target_rx_thread() usage. 3519 */ 3520 if (mode == 1) { 3521 if (!conn->conn_tx_reset_cpumask) 3522 return; 3523 conn->conn_tx_reset_cpumask = 0; 3524 } else { 3525 if (!conn->conn_rx_reset_cpumask) 3526 return; 3527 conn->conn_rx_reset_cpumask = 0; 3528 } 3529 /* 3530 * Update the CPU mask for this single kthread so that 3531 * both TX and RX kthreads are scheduled to run on the 3532 * same CPU. 3533 */ 3534 memset(buf, 0, 128); 3535 cpumask_scnprintf(buf, 128, conn->conn_cpumask); 3536 set_cpus_allowed_ptr(p, conn->conn_cpumask); 3537 } 3538 3539 static int handle_immediate_queue(struct iscsi_conn *conn) 3540 { 3541 struct iscsi_queue_req *qr; 3542 struct iscsi_cmd *cmd; 3543 u8 state; 3544 int ret; 3545 3546 while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) { 3547 atomic_set(&conn->check_immediate_queue, 0); 3548 cmd = qr->cmd; 3549 state = qr->state; 3550 kmem_cache_free(lio_qr_cache, qr); 3551 3552 switch (state) { 3553 case ISTATE_SEND_R2T: 3554 ret = iscsit_send_r2t(cmd, conn); 3555 if (ret < 0) 3556 goto err; 3557 break; 3558 case ISTATE_REMOVE: 3559 if (cmd->data_direction == DMA_TO_DEVICE) 3560 iscsit_stop_dataout_timer(cmd); 3561 3562 spin_lock_bh(&conn->cmd_lock); 3563 list_del(&cmd->i_conn_node); 3564 spin_unlock_bh(&conn->cmd_lock); 3565 3566 iscsit_free_cmd(cmd); 3567 continue; 3568 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 3569 iscsit_mod_nopin_response_timer(conn); 3570 ret = iscsit_send_unsolicited_nopin(cmd, 3571 conn, 1); 3572 if (ret < 0) 3573 goto err; 3574 break; 3575 case ISTATE_SEND_NOPIN_NO_RESPONSE: 3576 ret = iscsit_send_unsolicited_nopin(cmd, 3577 conn, 0); 3578 if (ret < 0) 3579 goto err; 3580 break; 3581 default: 3582 pr_err("Unknown Opcode: 0x%02x ITT:" 3583 " 0x%08x, i_state: %d on CID: %hu\n", 3584 cmd->iscsi_opcode, cmd->init_task_tag, state, 3585 conn->cid); 3586 goto err; 3587 } 3588 } 3589 3590 return 0; 3591 3592 err: 3593 return -1; 3594 } 3595 3596 static int handle_response_queue(struct iscsi_conn *conn) 3597 { 3598 struct iscsi_queue_req *qr; 3599 struct iscsi_cmd *cmd; 3600 u8 state; 3601 int ret; 3602 3603 while ((qr = iscsit_get_cmd_from_response_queue(conn))) { 3604 cmd = qr->cmd; 3605 state = qr->state; 3606 kmem_cache_free(lio_qr_cache, qr); 3607 3608 check_rsp_state: 3609 switch (state) { 3610 case ISTATE_SEND_DATAIN: 3611 ret = iscsit_send_data_in(cmd, conn); 3612 if (ret < 0) 3613 goto err; 3614 else if (!ret) 3615 /* more drs */ 3616 goto check_rsp_state; 3617 else if (ret == 1) { 3618 /* all done */ 3619 spin_lock_bh(&cmd->istate_lock); 3620 cmd->i_state = ISTATE_SENT_STATUS; 3621 spin_unlock_bh(&cmd->istate_lock); 3622 continue; 3623 } else if (ret == 2) { 3624 /* Still must send status, 3625 SCF_TRANSPORT_TASK_SENSE was set */ 3626 spin_lock_bh(&cmd->istate_lock); 3627 cmd->i_state = ISTATE_SEND_STATUS; 3628 spin_unlock_bh(&cmd->istate_lock); 3629 state = ISTATE_SEND_STATUS; 3630 goto check_rsp_state; 3631 } 3632 3633 break; 3634 case ISTATE_SEND_STATUS: 3635 case ISTATE_SEND_STATUS_RECOVERY: 3636 ret = iscsit_send_status(cmd, conn); 3637 break; 3638 case ISTATE_SEND_LOGOUTRSP: 3639 ret = iscsit_send_logout_response(cmd, conn); 3640 break; 3641 case ISTATE_SEND_ASYNCMSG: 3642 ret = iscsit_send_conn_drop_async_message( 3643 cmd, conn); 3644 break; 3645 case ISTATE_SEND_NOPIN: 3646 ret = iscsit_send_nopin_response(cmd, conn); 3647 break; 3648 case ISTATE_SEND_REJECT: 3649 ret = iscsit_send_reject(cmd, conn); 3650 break; 3651 case ISTATE_SEND_TASKMGTRSP: 3652 ret = iscsit_send_task_mgt_rsp(cmd, conn); 3653 if (ret != 0) 3654 break; 3655 ret = iscsit_tmr_post_handler(cmd, conn); 3656 if (ret != 0) 3657 iscsit_fall_back_to_erl0(conn->sess); 3658 break; 3659 case ISTATE_SEND_TEXTRSP: 3660 ret = iscsit_send_text_rsp(cmd, conn); 3661 break; 3662 default: 3663 pr_err("Unknown Opcode: 0x%02x ITT:" 3664 " 0x%08x, i_state: %d on CID: %hu\n", 3665 cmd->iscsi_opcode, cmd->init_task_tag, 3666 state, conn->cid); 3667 goto err; 3668 } 3669 if (ret < 0) 3670 goto err; 3671 3672 if (iscsit_send_tx_data(cmd, conn, 1) < 0) { 3673 iscsit_tx_thread_wait_for_tcp(conn); 3674 iscsit_unmap_iovec(cmd); 3675 goto err; 3676 } 3677 iscsit_unmap_iovec(cmd); 3678 3679 switch (state) { 3680 case ISTATE_SEND_LOGOUTRSP: 3681 if (!iscsit_logout_post_handler(cmd, conn)) 3682 goto restart; 3683 /* fall through */ 3684 case ISTATE_SEND_STATUS: 3685 case ISTATE_SEND_ASYNCMSG: 3686 case ISTATE_SEND_NOPIN: 3687 case ISTATE_SEND_STATUS_RECOVERY: 3688 case ISTATE_SEND_TEXTRSP: 3689 case ISTATE_SEND_TASKMGTRSP: 3690 spin_lock_bh(&cmd->istate_lock); 3691 cmd->i_state = ISTATE_SENT_STATUS; 3692 spin_unlock_bh(&cmd->istate_lock); 3693 break; 3694 case ISTATE_SEND_REJECT: 3695 if (cmd->cmd_flags & ICF_REJECT_FAIL_CONN) { 3696 cmd->cmd_flags &= ~ICF_REJECT_FAIL_CONN; 3697 complete(&cmd->reject_comp); 3698 goto err; 3699 } 3700 complete(&cmd->reject_comp); 3701 break; 3702 default: 3703 pr_err("Unknown Opcode: 0x%02x ITT:" 3704 " 0x%08x, i_state: %d on CID: %hu\n", 3705 cmd->iscsi_opcode, cmd->init_task_tag, 3706 cmd->i_state, conn->cid); 3707 goto err; 3708 } 3709 3710 if (atomic_read(&conn->check_immediate_queue)) 3711 break; 3712 } 3713 3714 return 0; 3715 3716 err: 3717 return -1; 3718 restart: 3719 return -EAGAIN; 3720 } 3721 3722 int iscsi_target_tx_thread(void *arg) 3723 { 3724 int ret = 0; 3725 struct iscsi_conn *conn; 3726 struct iscsi_thread_set *ts = arg; 3727 /* 3728 * Allow ourselves to be interrupted by SIGINT so that a 3729 * connection recovery / failure event can be triggered externally. 3730 */ 3731 allow_signal(SIGINT); 3732 3733 restart: 3734 conn = iscsi_tx_thread_pre_handler(ts); 3735 if (!conn) 3736 goto out; 3737 3738 ret = 0; 3739 3740 while (!kthread_should_stop()) { 3741 /* 3742 * Ensure that both TX and RX per connection kthreads 3743 * are scheduled to run on the same CPU. 3744 */ 3745 iscsit_thread_check_cpumask(conn, current, 1); 3746 3747 schedule_timeout_interruptible(MAX_SCHEDULE_TIMEOUT); 3748 3749 if ((ts->status == ISCSI_THREAD_SET_RESET) || 3750 signal_pending(current)) 3751 goto transport_err; 3752 3753 ret = handle_immediate_queue(conn); 3754 if (ret < 0) 3755 goto transport_err; 3756 3757 ret = handle_response_queue(conn); 3758 if (ret == -EAGAIN) 3759 goto restart; 3760 else if (ret < 0) 3761 goto transport_err; 3762 } 3763 3764 transport_err: 3765 iscsit_take_action_for_connection_exit(conn); 3766 goto restart; 3767 out: 3768 return 0; 3769 } 3770 3771 int iscsi_target_rx_thread(void *arg) 3772 { 3773 int ret; 3774 u8 buffer[ISCSI_HDR_LEN], opcode; 3775 u32 checksum = 0, digest = 0; 3776 struct iscsi_conn *conn = NULL; 3777 struct iscsi_thread_set *ts = arg; 3778 struct kvec iov; 3779 /* 3780 * Allow ourselves to be interrupted by SIGINT so that a 3781 * connection recovery / failure event can be triggered externally. 3782 */ 3783 allow_signal(SIGINT); 3784 3785 restart: 3786 conn = iscsi_rx_thread_pre_handler(ts); 3787 if (!conn) 3788 goto out; 3789 3790 while (!kthread_should_stop()) { 3791 /* 3792 * Ensure that both TX and RX per connection kthreads 3793 * are scheduled to run on the same CPU. 3794 */ 3795 iscsit_thread_check_cpumask(conn, current, 0); 3796 3797 memset(buffer, 0, ISCSI_HDR_LEN); 3798 memset(&iov, 0, sizeof(struct kvec)); 3799 3800 iov.iov_base = buffer; 3801 iov.iov_len = ISCSI_HDR_LEN; 3802 3803 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN); 3804 if (ret != ISCSI_HDR_LEN) { 3805 iscsit_rx_thread_wait_for_tcp(conn); 3806 goto transport_err; 3807 } 3808 3809 /* 3810 * Set conn->bad_hdr for use with REJECT PDUs. 3811 */ 3812 memcpy(&conn->bad_hdr, &buffer, ISCSI_HDR_LEN); 3813 3814 if (conn->conn_ops->HeaderDigest) { 3815 iov.iov_base = &digest; 3816 iov.iov_len = ISCSI_CRC_LEN; 3817 3818 ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN); 3819 if (ret != ISCSI_CRC_LEN) { 3820 iscsit_rx_thread_wait_for_tcp(conn); 3821 goto transport_err; 3822 } 3823 3824 iscsit_do_crypto_hash_buf(&conn->conn_rx_hash, 3825 buffer, ISCSI_HDR_LEN, 3826 0, NULL, (u8 *)&checksum); 3827 3828 if (digest != checksum) { 3829 pr_err("HeaderDigest CRC32C failed," 3830 " received 0x%08x, computed 0x%08x\n", 3831 digest, checksum); 3832 /* 3833 * Set the PDU to 0xff so it will intentionally 3834 * hit default in the switch below. 3835 */ 3836 memset(buffer, 0xff, ISCSI_HDR_LEN); 3837 spin_lock_bh(&conn->sess->session_stats_lock); 3838 conn->sess->conn_digest_errors++; 3839 spin_unlock_bh(&conn->sess->session_stats_lock); 3840 } else { 3841 pr_debug("Got HeaderDigest CRC32C" 3842 " 0x%08x\n", checksum); 3843 } 3844 } 3845 3846 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 3847 goto transport_err; 3848 3849 opcode = buffer[0] & ISCSI_OPCODE_MASK; 3850 3851 if (conn->sess->sess_ops->SessionType && 3852 ((!(opcode & ISCSI_OP_TEXT)) || 3853 (!(opcode & ISCSI_OP_LOGOUT)))) { 3854 pr_err("Received illegal iSCSI Opcode: 0x%02x" 3855 " while in Discovery Session, rejecting.\n", opcode); 3856 iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, 3857 buffer, conn); 3858 goto transport_err; 3859 } 3860 3861 switch (opcode) { 3862 case ISCSI_OP_SCSI_CMD: 3863 if (iscsit_handle_scsi_cmd(conn, buffer) < 0) 3864 goto transport_err; 3865 break; 3866 case ISCSI_OP_SCSI_DATA_OUT: 3867 if (iscsit_handle_data_out(conn, buffer) < 0) 3868 goto transport_err; 3869 break; 3870 case ISCSI_OP_NOOP_OUT: 3871 if (iscsit_handle_nop_out(conn, buffer) < 0) 3872 goto transport_err; 3873 break; 3874 case ISCSI_OP_SCSI_TMFUNC: 3875 if (iscsit_handle_task_mgt_cmd(conn, buffer) < 0) 3876 goto transport_err; 3877 break; 3878 case ISCSI_OP_TEXT: 3879 if (iscsit_handle_text_cmd(conn, buffer) < 0) 3880 goto transport_err; 3881 break; 3882 case ISCSI_OP_LOGOUT: 3883 ret = iscsit_handle_logout_cmd(conn, buffer); 3884 if (ret > 0) { 3885 wait_for_completion_timeout(&conn->conn_logout_comp, 3886 SECONDS_FOR_LOGOUT_COMP * HZ); 3887 goto transport_err; 3888 } else if (ret < 0) 3889 goto transport_err; 3890 break; 3891 case ISCSI_OP_SNACK: 3892 if (iscsit_handle_snack(conn, buffer) < 0) 3893 goto transport_err; 3894 break; 3895 default: 3896 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", 3897 opcode); 3898 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 3899 pr_err("Cannot recover from unknown" 3900 " opcode while ERL=0, closing iSCSI connection" 3901 ".\n"); 3902 goto transport_err; 3903 } 3904 if (!conn->conn_ops->OFMarker) { 3905 pr_err("Unable to recover from unknown" 3906 " opcode while OFMarker=No, closing iSCSI" 3907 " connection.\n"); 3908 goto transport_err; 3909 } 3910 if (iscsit_recover_from_unknown_opcode(conn) < 0) { 3911 pr_err("Unable to recover from unknown" 3912 " opcode, closing iSCSI connection.\n"); 3913 goto transport_err; 3914 } 3915 break; 3916 } 3917 } 3918 3919 transport_err: 3920 if (!signal_pending(current)) 3921 atomic_set(&conn->transport_failed, 1); 3922 iscsit_take_action_for_connection_exit(conn); 3923 goto restart; 3924 out: 3925 return 0; 3926 } 3927 3928 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn) 3929 { 3930 struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL; 3931 struct iscsi_session *sess = conn->sess; 3932 /* 3933 * We expect this function to only ever be called from either RX or TX 3934 * thread context via iscsit_close_connection() once the other context 3935 * has been reset -> returned sleeping pre-handler state. 3936 */ 3937 spin_lock_bh(&conn->cmd_lock); 3938 list_for_each_entry_safe(cmd, cmd_tmp, &conn->conn_cmd_list, i_conn_node) { 3939 3940 list_del(&cmd->i_conn_node); 3941 spin_unlock_bh(&conn->cmd_lock); 3942 3943 iscsit_increment_maxcmdsn(cmd, sess); 3944 3945 iscsit_free_cmd(cmd); 3946 3947 spin_lock_bh(&conn->cmd_lock); 3948 } 3949 spin_unlock_bh(&conn->cmd_lock); 3950 } 3951 3952 static void iscsit_stop_timers_for_cmds( 3953 struct iscsi_conn *conn) 3954 { 3955 struct iscsi_cmd *cmd; 3956 3957 spin_lock_bh(&conn->cmd_lock); 3958 list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) { 3959 if (cmd->data_direction == DMA_TO_DEVICE) 3960 iscsit_stop_dataout_timer(cmd); 3961 } 3962 spin_unlock_bh(&conn->cmd_lock); 3963 } 3964 3965 int iscsit_close_connection( 3966 struct iscsi_conn *conn) 3967 { 3968 int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT); 3969 struct iscsi_session *sess = conn->sess; 3970 3971 pr_debug("Closing iSCSI connection CID %hu on SID:" 3972 " %u\n", conn->cid, sess->sid); 3973 /* 3974 * Always up conn_logout_comp just in case the RX Thread is sleeping 3975 * and the logout response never got sent because the connection 3976 * failed. 3977 */ 3978 complete(&conn->conn_logout_comp); 3979 3980 iscsi_release_thread_set(conn); 3981 3982 iscsit_stop_timers_for_cmds(conn); 3983 iscsit_stop_nopin_response_timer(conn); 3984 iscsit_stop_nopin_timer(conn); 3985 iscsit_free_queue_reqs_for_conn(conn); 3986 3987 /* 3988 * During Connection recovery drop unacknowledged out of order 3989 * commands for this connection, and prepare the other commands 3990 * for realligence. 3991 * 3992 * During normal operation clear the out of order commands (but 3993 * do not free the struct iscsi_ooo_cmdsn's) and release all 3994 * struct iscsi_cmds. 3995 */ 3996 if (atomic_read(&conn->connection_recovery)) { 3997 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn); 3998 iscsit_prepare_cmds_for_realligance(conn); 3999 } else { 4000 iscsit_clear_ooo_cmdsns_for_conn(conn); 4001 iscsit_release_commands_from_conn(conn); 4002 } 4003 4004 /* 4005 * Handle decrementing session or connection usage count if 4006 * a logout response was not able to be sent because the 4007 * connection failed. Fall back to Session Recovery here. 4008 */ 4009 if (atomic_read(&conn->conn_logout_remove)) { 4010 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) { 4011 iscsit_dec_conn_usage_count(conn); 4012 iscsit_dec_session_usage_count(sess); 4013 } 4014 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) 4015 iscsit_dec_conn_usage_count(conn); 4016 4017 atomic_set(&conn->conn_logout_remove, 0); 4018 atomic_set(&sess->session_reinstatement, 0); 4019 atomic_set(&sess->session_fall_back_to_erl0, 1); 4020 } 4021 4022 spin_lock_bh(&sess->conn_lock); 4023 list_del(&conn->conn_list); 4024 4025 /* 4026 * Attempt to let the Initiator know this connection failed by 4027 * sending an Connection Dropped Async Message on another 4028 * active connection. 4029 */ 4030 if (atomic_read(&conn->connection_recovery)) 4031 iscsit_build_conn_drop_async_message(conn); 4032 4033 spin_unlock_bh(&sess->conn_lock); 4034 4035 /* 4036 * If connection reinstatement is being performed on this connection, 4037 * up the connection reinstatement semaphore that is being blocked on 4038 * in iscsit_cause_connection_reinstatement(). 4039 */ 4040 spin_lock_bh(&conn->state_lock); 4041 if (atomic_read(&conn->sleep_on_conn_wait_comp)) { 4042 spin_unlock_bh(&conn->state_lock); 4043 complete(&conn->conn_wait_comp); 4044 wait_for_completion(&conn->conn_post_wait_comp); 4045 spin_lock_bh(&conn->state_lock); 4046 } 4047 4048 /* 4049 * If connection reinstatement is being performed on this connection 4050 * by receiving a REMOVECONNFORRECOVERY logout request, up the 4051 * connection wait rcfr semaphore that is being blocked on 4052 * an iscsit_connection_reinstatement_rcfr(). 4053 */ 4054 if (atomic_read(&conn->connection_wait_rcfr)) { 4055 spin_unlock_bh(&conn->state_lock); 4056 complete(&conn->conn_wait_rcfr_comp); 4057 wait_for_completion(&conn->conn_post_wait_comp); 4058 spin_lock_bh(&conn->state_lock); 4059 } 4060 atomic_set(&conn->connection_reinstatement, 1); 4061 spin_unlock_bh(&conn->state_lock); 4062 4063 /* 4064 * If any other processes are accessing this connection pointer we 4065 * must wait until they have completed. 4066 */ 4067 iscsit_check_conn_usage_count(conn); 4068 4069 if (conn->conn_rx_hash.tfm) 4070 crypto_free_hash(conn->conn_rx_hash.tfm); 4071 if (conn->conn_tx_hash.tfm) 4072 crypto_free_hash(conn->conn_tx_hash.tfm); 4073 4074 if (conn->conn_cpumask) 4075 free_cpumask_var(conn->conn_cpumask); 4076 4077 kfree(conn->conn_ops); 4078 conn->conn_ops = NULL; 4079 4080 if (conn->sock) { 4081 if (conn->conn_flags & CONNFLAG_SCTP_STRUCT_FILE) { 4082 kfree(conn->sock->file); 4083 conn->sock->file = NULL; 4084 } 4085 sock_release(conn->sock); 4086 } 4087 conn->thread_set = NULL; 4088 4089 pr_debug("Moving to TARG_CONN_STATE_FREE.\n"); 4090 conn->conn_state = TARG_CONN_STATE_FREE; 4091 kfree(conn); 4092 4093 spin_lock_bh(&sess->conn_lock); 4094 atomic_dec(&sess->nconn); 4095 pr_debug("Decremented iSCSI connection count to %hu from node:" 4096 " %s\n", atomic_read(&sess->nconn), 4097 sess->sess_ops->InitiatorName); 4098 /* 4099 * Make sure that if one connection fails in an non ERL=2 iSCSI 4100 * Session that they all fail. 4101 */ 4102 if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout && 4103 !atomic_read(&sess->session_logout)) 4104 atomic_set(&sess->session_fall_back_to_erl0, 1); 4105 4106 /* 4107 * If this was not the last connection in the session, and we are 4108 * performing session reinstatement or falling back to ERL=0, call 4109 * iscsit_stop_session() without sleeping to shutdown the other 4110 * active connections. 4111 */ 4112 if (atomic_read(&sess->nconn)) { 4113 if (!atomic_read(&sess->session_reinstatement) && 4114 !atomic_read(&sess->session_fall_back_to_erl0)) { 4115 spin_unlock_bh(&sess->conn_lock); 4116 return 0; 4117 } 4118 if (!atomic_read(&sess->session_stop_active)) { 4119 atomic_set(&sess->session_stop_active, 1); 4120 spin_unlock_bh(&sess->conn_lock); 4121 iscsit_stop_session(sess, 0, 0); 4122 return 0; 4123 } 4124 spin_unlock_bh(&sess->conn_lock); 4125 return 0; 4126 } 4127 4128 /* 4129 * If this was the last connection in the session and one of the 4130 * following is occurring: 4131 * 4132 * Session Reinstatement is not being performed, and are falling back 4133 * to ERL=0 call iscsit_close_session(). 4134 * 4135 * Session Logout was requested. iscsit_close_session() will be called 4136 * elsewhere. 4137 * 4138 * Session Continuation is not being performed, start the Time2Retain 4139 * handler and check if sleep_on_sess_wait_sem is active. 4140 */ 4141 if (!atomic_read(&sess->session_reinstatement) && 4142 atomic_read(&sess->session_fall_back_to_erl0)) { 4143 spin_unlock_bh(&sess->conn_lock); 4144 target_put_session(sess->se_sess); 4145 4146 return 0; 4147 } else if (atomic_read(&sess->session_logout)) { 4148 pr_debug("Moving to TARG_SESS_STATE_FREE.\n"); 4149 sess->session_state = TARG_SESS_STATE_FREE; 4150 spin_unlock_bh(&sess->conn_lock); 4151 4152 if (atomic_read(&sess->sleep_on_sess_wait_comp)) 4153 complete(&sess->session_wait_comp); 4154 4155 return 0; 4156 } else { 4157 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); 4158 sess->session_state = TARG_SESS_STATE_FAILED; 4159 4160 if (!atomic_read(&sess->session_continuation)) { 4161 spin_unlock_bh(&sess->conn_lock); 4162 iscsit_start_time2retain_handler(sess); 4163 } else 4164 spin_unlock_bh(&sess->conn_lock); 4165 4166 if (atomic_read(&sess->sleep_on_sess_wait_comp)) 4167 complete(&sess->session_wait_comp); 4168 4169 return 0; 4170 } 4171 spin_unlock_bh(&sess->conn_lock); 4172 4173 return 0; 4174 } 4175 4176 int iscsit_close_session(struct iscsi_session *sess) 4177 { 4178 struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess); 4179 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; 4180 4181 if (atomic_read(&sess->nconn)) { 4182 pr_err("%d connection(s) still exist for iSCSI session" 4183 " to %s\n", atomic_read(&sess->nconn), 4184 sess->sess_ops->InitiatorName); 4185 BUG(); 4186 } 4187 4188 spin_lock_bh(&se_tpg->session_lock); 4189 atomic_set(&sess->session_logout, 1); 4190 atomic_set(&sess->session_reinstatement, 1); 4191 iscsit_stop_time2retain_timer(sess); 4192 spin_unlock_bh(&se_tpg->session_lock); 4193 4194 /* 4195 * transport_deregister_session_configfs() will clear the 4196 * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context 4197 * can be setting it again with __transport_register_session() in 4198 * iscsi_post_login_handler() again after the iscsit_stop_session() 4199 * completes in iscsi_np context. 4200 */ 4201 transport_deregister_session_configfs(sess->se_sess); 4202 4203 /* 4204 * If any other processes are accessing this session pointer we must 4205 * wait until they have completed. If we are in an interrupt (the 4206 * time2retain handler) and contain and active session usage count we 4207 * restart the timer and exit. 4208 */ 4209 if (!in_interrupt()) { 4210 if (iscsit_check_session_usage_count(sess) == 1) 4211 iscsit_stop_session(sess, 1, 1); 4212 } else { 4213 if (iscsit_check_session_usage_count(sess) == 2) { 4214 atomic_set(&sess->session_logout, 0); 4215 iscsit_start_time2retain_handler(sess); 4216 return 0; 4217 } 4218 } 4219 4220 transport_deregister_session(sess->se_sess); 4221 4222 if (sess->sess_ops->ErrorRecoveryLevel == 2) 4223 iscsit_free_connection_recovery_entires(sess); 4224 4225 iscsit_free_all_ooo_cmdsns(sess); 4226 4227 spin_lock_bh(&se_tpg->session_lock); 4228 pr_debug("Moving to TARG_SESS_STATE_FREE.\n"); 4229 sess->session_state = TARG_SESS_STATE_FREE; 4230 pr_debug("Released iSCSI session from node: %s\n", 4231 sess->sess_ops->InitiatorName); 4232 tpg->nsessions--; 4233 if (tpg->tpg_tiqn) 4234 tpg->tpg_tiqn->tiqn_nsessions--; 4235 4236 pr_debug("Decremented number of active iSCSI Sessions on" 4237 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions); 4238 4239 spin_lock(&sess_idr_lock); 4240 idr_remove(&sess_idr, sess->session_index); 4241 spin_unlock(&sess_idr_lock); 4242 4243 kfree(sess->sess_ops); 4244 sess->sess_ops = NULL; 4245 spin_unlock_bh(&se_tpg->session_lock); 4246 4247 kfree(sess); 4248 return 0; 4249 } 4250 4251 static void iscsit_logout_post_handler_closesession( 4252 struct iscsi_conn *conn) 4253 { 4254 struct iscsi_session *sess = conn->sess; 4255 4256 iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD); 4257 iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD); 4258 4259 atomic_set(&conn->conn_logout_remove, 0); 4260 complete(&conn->conn_logout_comp); 4261 4262 iscsit_dec_conn_usage_count(conn); 4263 iscsit_stop_session(sess, 1, 1); 4264 iscsit_dec_session_usage_count(sess); 4265 target_put_session(sess->se_sess); 4266 } 4267 4268 static void iscsit_logout_post_handler_samecid( 4269 struct iscsi_conn *conn) 4270 { 4271 iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD); 4272 iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD); 4273 4274 atomic_set(&conn->conn_logout_remove, 0); 4275 complete(&conn->conn_logout_comp); 4276 4277 iscsit_cause_connection_reinstatement(conn, 1); 4278 iscsit_dec_conn_usage_count(conn); 4279 } 4280 4281 static void iscsit_logout_post_handler_diffcid( 4282 struct iscsi_conn *conn, 4283 u16 cid) 4284 { 4285 struct iscsi_conn *l_conn; 4286 struct iscsi_session *sess = conn->sess; 4287 4288 if (!sess) 4289 return; 4290 4291 spin_lock_bh(&sess->conn_lock); 4292 list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) { 4293 if (l_conn->cid == cid) { 4294 iscsit_inc_conn_usage_count(l_conn); 4295 break; 4296 } 4297 } 4298 spin_unlock_bh(&sess->conn_lock); 4299 4300 if (!l_conn) 4301 return; 4302 4303 if (l_conn->sock) 4304 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN); 4305 4306 spin_lock_bh(&l_conn->state_lock); 4307 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n"); 4308 l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT; 4309 spin_unlock_bh(&l_conn->state_lock); 4310 4311 iscsit_cause_connection_reinstatement(l_conn, 1); 4312 iscsit_dec_conn_usage_count(l_conn); 4313 } 4314 4315 /* 4316 * Return of 0 causes the TX thread to restart. 4317 */ 4318 static int iscsit_logout_post_handler( 4319 struct iscsi_cmd *cmd, 4320 struct iscsi_conn *conn) 4321 { 4322 int ret = 0; 4323 4324 switch (cmd->logout_reason) { 4325 case ISCSI_LOGOUT_REASON_CLOSE_SESSION: 4326 switch (cmd->logout_response) { 4327 case ISCSI_LOGOUT_SUCCESS: 4328 case ISCSI_LOGOUT_CLEANUP_FAILED: 4329 default: 4330 iscsit_logout_post_handler_closesession(conn); 4331 break; 4332 } 4333 ret = 0; 4334 break; 4335 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: 4336 if (conn->cid == cmd->logout_cid) { 4337 switch (cmd->logout_response) { 4338 case ISCSI_LOGOUT_SUCCESS: 4339 case ISCSI_LOGOUT_CLEANUP_FAILED: 4340 default: 4341 iscsit_logout_post_handler_samecid(conn); 4342 break; 4343 } 4344 ret = 0; 4345 } else { 4346 switch (cmd->logout_response) { 4347 case ISCSI_LOGOUT_SUCCESS: 4348 iscsit_logout_post_handler_diffcid(conn, 4349 cmd->logout_cid); 4350 break; 4351 case ISCSI_LOGOUT_CID_NOT_FOUND: 4352 case ISCSI_LOGOUT_CLEANUP_FAILED: 4353 default: 4354 break; 4355 } 4356 ret = 1; 4357 } 4358 break; 4359 case ISCSI_LOGOUT_REASON_RECOVERY: 4360 switch (cmd->logout_response) { 4361 case ISCSI_LOGOUT_SUCCESS: 4362 case ISCSI_LOGOUT_CID_NOT_FOUND: 4363 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED: 4364 case ISCSI_LOGOUT_CLEANUP_FAILED: 4365 default: 4366 break; 4367 } 4368 ret = 1; 4369 break; 4370 default: 4371 break; 4372 4373 } 4374 return ret; 4375 } 4376 4377 void iscsit_fail_session(struct iscsi_session *sess) 4378 { 4379 struct iscsi_conn *conn; 4380 4381 spin_lock_bh(&sess->conn_lock); 4382 list_for_each_entry(conn, &sess->sess_conn_list, conn_list) { 4383 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n"); 4384 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT; 4385 } 4386 spin_unlock_bh(&sess->conn_lock); 4387 4388 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); 4389 sess->session_state = TARG_SESS_STATE_FAILED; 4390 } 4391 4392 int iscsit_free_session(struct iscsi_session *sess) 4393 { 4394 u16 conn_count = atomic_read(&sess->nconn); 4395 struct iscsi_conn *conn, *conn_tmp = NULL; 4396 int is_last; 4397 4398 spin_lock_bh(&sess->conn_lock); 4399 atomic_set(&sess->sleep_on_sess_wait_comp, 1); 4400 4401 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list, 4402 conn_list) { 4403 if (conn_count == 0) 4404 break; 4405 4406 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) { 4407 is_last = 1; 4408 } else { 4409 iscsit_inc_conn_usage_count(conn_tmp); 4410 is_last = 0; 4411 } 4412 iscsit_inc_conn_usage_count(conn); 4413 4414 spin_unlock_bh(&sess->conn_lock); 4415 iscsit_cause_connection_reinstatement(conn, 1); 4416 spin_lock_bh(&sess->conn_lock); 4417 4418 iscsit_dec_conn_usage_count(conn); 4419 if (is_last == 0) 4420 iscsit_dec_conn_usage_count(conn_tmp); 4421 4422 conn_count--; 4423 } 4424 4425 if (atomic_read(&sess->nconn)) { 4426 spin_unlock_bh(&sess->conn_lock); 4427 wait_for_completion(&sess->session_wait_comp); 4428 } else 4429 spin_unlock_bh(&sess->conn_lock); 4430 4431 target_put_session(sess->se_sess); 4432 return 0; 4433 } 4434 4435 void iscsit_stop_session( 4436 struct iscsi_session *sess, 4437 int session_sleep, 4438 int connection_sleep) 4439 { 4440 u16 conn_count = atomic_read(&sess->nconn); 4441 struct iscsi_conn *conn, *conn_tmp = NULL; 4442 int is_last; 4443 4444 spin_lock_bh(&sess->conn_lock); 4445 if (session_sleep) 4446 atomic_set(&sess->sleep_on_sess_wait_comp, 1); 4447 4448 if (connection_sleep) { 4449 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list, 4450 conn_list) { 4451 if (conn_count == 0) 4452 break; 4453 4454 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) { 4455 is_last = 1; 4456 } else { 4457 iscsit_inc_conn_usage_count(conn_tmp); 4458 is_last = 0; 4459 } 4460 iscsit_inc_conn_usage_count(conn); 4461 4462 spin_unlock_bh(&sess->conn_lock); 4463 iscsit_cause_connection_reinstatement(conn, 1); 4464 spin_lock_bh(&sess->conn_lock); 4465 4466 iscsit_dec_conn_usage_count(conn); 4467 if (is_last == 0) 4468 iscsit_dec_conn_usage_count(conn_tmp); 4469 conn_count--; 4470 } 4471 } else { 4472 list_for_each_entry(conn, &sess->sess_conn_list, conn_list) 4473 iscsit_cause_connection_reinstatement(conn, 0); 4474 } 4475 4476 if (session_sleep && atomic_read(&sess->nconn)) { 4477 spin_unlock_bh(&sess->conn_lock); 4478 wait_for_completion(&sess->session_wait_comp); 4479 } else 4480 spin_unlock_bh(&sess->conn_lock); 4481 } 4482 4483 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force) 4484 { 4485 struct iscsi_session *sess; 4486 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; 4487 struct se_session *se_sess, *se_sess_tmp; 4488 int session_count = 0; 4489 4490 spin_lock_bh(&se_tpg->session_lock); 4491 if (tpg->nsessions && !force) { 4492 spin_unlock_bh(&se_tpg->session_lock); 4493 return -1; 4494 } 4495 4496 list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list, 4497 sess_list) { 4498 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr; 4499 4500 spin_lock(&sess->conn_lock); 4501 if (atomic_read(&sess->session_fall_back_to_erl0) || 4502 atomic_read(&sess->session_logout) || 4503 (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) { 4504 spin_unlock(&sess->conn_lock); 4505 continue; 4506 } 4507 atomic_set(&sess->session_reinstatement, 1); 4508 spin_unlock(&sess->conn_lock); 4509 spin_unlock_bh(&se_tpg->session_lock); 4510 4511 iscsit_free_session(sess); 4512 spin_lock_bh(&se_tpg->session_lock); 4513 4514 session_count++; 4515 } 4516 spin_unlock_bh(&se_tpg->session_lock); 4517 4518 pr_debug("Released %d iSCSI Session(s) from Target Portal" 4519 " Group: %hu\n", session_count, tpg->tpgt); 4520 return 0; 4521 } 4522 4523 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure"); 4524 MODULE_VERSION("4.1.x"); 4525 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 4526 MODULE_LICENSE("GPL"); 4527 4528 module_init(iscsi_target_init_module); 4529 module_exit(iscsi_target_cleanup_module); 4530