1 /******************************************************************************* 2 * Filename: target_core_transport.c 3 * 4 * This file contains the Generic Target Engine Core. 5 * 6 * (c) Copyright 2002-2013 Datera, Inc. 7 * 8 * Nicholas A. Bellinger <nab@kernel.org> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License as published by 12 * the Free Software Foundation; either version 2 of the License, or 13 * (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 * 20 * You should have received a copy of the GNU General Public License 21 * along with this program; if not, write to the Free Software 22 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 23 * 24 ******************************************************************************/ 25 26 #include <linux/net.h> 27 #include <linux/delay.h> 28 #include <linux/string.h> 29 #include <linux/timer.h> 30 #include <linux/slab.h> 31 #include <linux/spinlock.h> 32 #include <linux/kthread.h> 33 #include <linux/in.h> 34 #include <linux/cdrom.h> 35 #include <linux/module.h> 36 #include <linux/ratelimit.h> 37 #include <linux/vmalloc.h> 38 #include <asm/unaligned.h> 39 #include <net/sock.h> 40 #include <net/tcp.h> 41 #include <scsi/scsi_proto.h> 42 #include <scsi/scsi_common.h> 43 44 #include <target/target_core_base.h> 45 #include <target/target_core_backend.h> 46 #include <target/target_core_fabric.h> 47 48 #include "target_core_internal.h" 49 #include "target_core_alua.h" 50 #include "target_core_pr.h" 51 #include "target_core_ua.h" 52 53 #define CREATE_TRACE_POINTS 54 #include <trace/events/target.h> 55 56 static struct workqueue_struct *target_completion_wq; 57 static struct kmem_cache *se_sess_cache; 58 struct kmem_cache *se_ua_cache; 59 struct kmem_cache *t10_pr_reg_cache; 60 struct kmem_cache *t10_alua_lu_gp_cache; 61 struct kmem_cache *t10_alua_lu_gp_mem_cache; 62 struct kmem_cache *t10_alua_tg_pt_gp_cache; 63 struct kmem_cache *t10_alua_lba_map_cache; 64 struct kmem_cache *t10_alua_lba_map_mem_cache; 65 66 static void transport_complete_task_attr(struct se_cmd *cmd); 67 static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason); 68 static void transport_handle_queue_full(struct se_cmd *cmd, 69 struct se_device *dev, int err, bool write_pending); 70 static void target_complete_ok_work(struct work_struct *work); 71 72 int init_se_kmem_caches(void) 73 { 74 se_sess_cache = kmem_cache_create("se_sess_cache", 75 sizeof(struct se_session), __alignof__(struct se_session), 76 0, NULL); 77 if (!se_sess_cache) { 78 pr_err("kmem_cache_create() for struct se_session" 79 " failed\n"); 80 goto out; 81 } 82 se_ua_cache = kmem_cache_create("se_ua_cache", 83 sizeof(struct se_ua), __alignof__(struct se_ua), 84 0, NULL); 85 if (!se_ua_cache) { 86 pr_err("kmem_cache_create() for struct se_ua failed\n"); 87 goto out_free_sess_cache; 88 } 89 t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache", 90 sizeof(struct t10_pr_registration), 91 __alignof__(struct t10_pr_registration), 0, NULL); 92 if (!t10_pr_reg_cache) { 93 pr_err("kmem_cache_create() for struct t10_pr_registration" 94 " failed\n"); 95 goto out_free_ua_cache; 96 } 97 t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache", 98 sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp), 99 0, NULL); 100 if (!t10_alua_lu_gp_cache) { 101 pr_err("kmem_cache_create() for t10_alua_lu_gp_cache" 102 " failed\n"); 103 goto out_free_pr_reg_cache; 104 } 105 t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache", 106 sizeof(struct t10_alua_lu_gp_member), 107 __alignof__(struct t10_alua_lu_gp_member), 0, NULL); 108 if (!t10_alua_lu_gp_mem_cache) { 109 pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_" 110 "cache failed\n"); 111 goto out_free_lu_gp_cache; 112 } 113 t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache", 114 sizeof(struct t10_alua_tg_pt_gp), 115 __alignof__(struct t10_alua_tg_pt_gp), 0, NULL); 116 if (!t10_alua_tg_pt_gp_cache) { 117 pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_" 118 "cache failed\n"); 119 goto out_free_lu_gp_mem_cache; 120 } 121 t10_alua_lba_map_cache = kmem_cache_create( 122 "t10_alua_lba_map_cache", 123 sizeof(struct t10_alua_lba_map), 124 __alignof__(struct t10_alua_lba_map), 0, NULL); 125 if (!t10_alua_lba_map_cache) { 126 pr_err("kmem_cache_create() for t10_alua_lba_map_" 127 "cache failed\n"); 128 goto out_free_tg_pt_gp_cache; 129 } 130 t10_alua_lba_map_mem_cache = kmem_cache_create( 131 "t10_alua_lba_map_mem_cache", 132 sizeof(struct t10_alua_lba_map_member), 133 __alignof__(struct t10_alua_lba_map_member), 0, NULL); 134 if (!t10_alua_lba_map_mem_cache) { 135 pr_err("kmem_cache_create() for t10_alua_lba_map_mem_" 136 "cache failed\n"); 137 goto out_free_lba_map_cache; 138 } 139 140 target_completion_wq = alloc_workqueue("target_completion", 141 WQ_MEM_RECLAIM, 0); 142 if (!target_completion_wq) 143 goto out_free_lba_map_mem_cache; 144 145 return 0; 146 147 out_free_lba_map_mem_cache: 148 kmem_cache_destroy(t10_alua_lba_map_mem_cache); 149 out_free_lba_map_cache: 150 kmem_cache_destroy(t10_alua_lba_map_cache); 151 out_free_tg_pt_gp_cache: 152 kmem_cache_destroy(t10_alua_tg_pt_gp_cache); 153 out_free_lu_gp_mem_cache: 154 kmem_cache_destroy(t10_alua_lu_gp_mem_cache); 155 out_free_lu_gp_cache: 156 kmem_cache_destroy(t10_alua_lu_gp_cache); 157 out_free_pr_reg_cache: 158 kmem_cache_destroy(t10_pr_reg_cache); 159 out_free_ua_cache: 160 kmem_cache_destroy(se_ua_cache); 161 out_free_sess_cache: 162 kmem_cache_destroy(se_sess_cache); 163 out: 164 return -ENOMEM; 165 } 166 167 void release_se_kmem_caches(void) 168 { 169 destroy_workqueue(target_completion_wq); 170 kmem_cache_destroy(se_sess_cache); 171 kmem_cache_destroy(se_ua_cache); 172 kmem_cache_destroy(t10_pr_reg_cache); 173 kmem_cache_destroy(t10_alua_lu_gp_cache); 174 kmem_cache_destroy(t10_alua_lu_gp_mem_cache); 175 kmem_cache_destroy(t10_alua_tg_pt_gp_cache); 176 kmem_cache_destroy(t10_alua_lba_map_cache); 177 kmem_cache_destroy(t10_alua_lba_map_mem_cache); 178 } 179 180 /* This code ensures unique mib indexes are handed out. */ 181 static DEFINE_SPINLOCK(scsi_mib_index_lock); 182 static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX]; 183 184 /* 185 * Allocate a new row index for the entry type specified 186 */ 187 u32 scsi_get_new_index(scsi_index_t type) 188 { 189 u32 new_index; 190 191 BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX)); 192 193 spin_lock(&scsi_mib_index_lock); 194 new_index = ++scsi_mib_index[type]; 195 spin_unlock(&scsi_mib_index_lock); 196 197 return new_index; 198 } 199 200 void transport_subsystem_check_init(void) 201 { 202 int ret; 203 static int sub_api_initialized; 204 205 if (sub_api_initialized) 206 return; 207 208 ret = request_module("target_core_iblock"); 209 if (ret != 0) 210 pr_err("Unable to load target_core_iblock\n"); 211 212 ret = request_module("target_core_file"); 213 if (ret != 0) 214 pr_err("Unable to load target_core_file\n"); 215 216 ret = request_module("target_core_pscsi"); 217 if (ret != 0) 218 pr_err("Unable to load target_core_pscsi\n"); 219 220 ret = request_module("target_core_user"); 221 if (ret != 0) 222 pr_err("Unable to load target_core_user\n"); 223 224 sub_api_initialized = 1; 225 } 226 227 struct se_session *transport_init_session(enum target_prot_op sup_prot_ops) 228 { 229 struct se_session *se_sess; 230 231 se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL); 232 if (!se_sess) { 233 pr_err("Unable to allocate struct se_session from" 234 " se_sess_cache\n"); 235 return ERR_PTR(-ENOMEM); 236 } 237 INIT_LIST_HEAD(&se_sess->sess_list); 238 INIT_LIST_HEAD(&se_sess->sess_acl_list); 239 INIT_LIST_HEAD(&se_sess->sess_cmd_list); 240 INIT_LIST_HEAD(&se_sess->sess_wait_list); 241 spin_lock_init(&se_sess->sess_cmd_lock); 242 se_sess->sup_prot_ops = sup_prot_ops; 243 244 return se_sess; 245 } 246 EXPORT_SYMBOL(transport_init_session); 247 248 int transport_alloc_session_tags(struct se_session *se_sess, 249 unsigned int tag_num, unsigned int tag_size) 250 { 251 int rc; 252 253 se_sess->sess_cmd_map = kzalloc(tag_num * tag_size, 254 GFP_KERNEL | __GFP_NOWARN | __GFP_RETRY_MAYFAIL); 255 if (!se_sess->sess_cmd_map) { 256 se_sess->sess_cmd_map = vzalloc(tag_num * tag_size); 257 if (!se_sess->sess_cmd_map) { 258 pr_err("Unable to allocate se_sess->sess_cmd_map\n"); 259 return -ENOMEM; 260 } 261 } 262 263 rc = percpu_ida_init(&se_sess->sess_tag_pool, tag_num); 264 if (rc < 0) { 265 pr_err("Unable to init se_sess->sess_tag_pool," 266 " tag_num: %u\n", tag_num); 267 kvfree(se_sess->sess_cmd_map); 268 se_sess->sess_cmd_map = NULL; 269 return -ENOMEM; 270 } 271 272 return 0; 273 } 274 EXPORT_SYMBOL(transport_alloc_session_tags); 275 276 struct se_session *transport_init_session_tags(unsigned int tag_num, 277 unsigned int tag_size, 278 enum target_prot_op sup_prot_ops) 279 { 280 struct se_session *se_sess; 281 int rc; 282 283 if (tag_num != 0 && !tag_size) { 284 pr_err("init_session_tags called with percpu-ida tag_num:" 285 " %u, but zero tag_size\n", tag_num); 286 return ERR_PTR(-EINVAL); 287 } 288 if (!tag_num && tag_size) { 289 pr_err("init_session_tags called with percpu-ida tag_size:" 290 " %u, but zero tag_num\n", tag_size); 291 return ERR_PTR(-EINVAL); 292 } 293 294 se_sess = transport_init_session(sup_prot_ops); 295 if (IS_ERR(se_sess)) 296 return se_sess; 297 298 rc = transport_alloc_session_tags(se_sess, tag_num, tag_size); 299 if (rc < 0) { 300 transport_free_session(se_sess); 301 return ERR_PTR(-ENOMEM); 302 } 303 304 return se_sess; 305 } 306 EXPORT_SYMBOL(transport_init_session_tags); 307 308 /* 309 * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called. 310 */ 311 void __transport_register_session( 312 struct se_portal_group *se_tpg, 313 struct se_node_acl *se_nacl, 314 struct se_session *se_sess, 315 void *fabric_sess_ptr) 316 { 317 const struct target_core_fabric_ops *tfo = se_tpg->se_tpg_tfo; 318 unsigned char buf[PR_REG_ISID_LEN]; 319 320 se_sess->se_tpg = se_tpg; 321 se_sess->fabric_sess_ptr = fabric_sess_ptr; 322 /* 323 * Used by struct se_node_acl's under ConfigFS to locate active se_session-t 324 * 325 * Only set for struct se_session's that will actually be moving I/O. 326 * eg: *NOT* discovery sessions. 327 */ 328 if (se_nacl) { 329 /* 330 * 331 * Determine if fabric allows for T10-PI feature bits exposed to 332 * initiators for device backends with !dev->dev_attrib.pi_prot_type. 333 * 334 * If so, then always save prot_type on a per se_node_acl node 335 * basis and re-instate the previous sess_prot_type to avoid 336 * disabling PI from below any previously initiator side 337 * registered LUNs. 338 */ 339 if (se_nacl->saved_prot_type) 340 se_sess->sess_prot_type = se_nacl->saved_prot_type; 341 else if (tfo->tpg_check_prot_fabric_only) 342 se_sess->sess_prot_type = se_nacl->saved_prot_type = 343 tfo->tpg_check_prot_fabric_only(se_tpg); 344 /* 345 * If the fabric module supports an ISID based TransportID, 346 * save this value in binary from the fabric I_T Nexus now. 347 */ 348 if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) { 349 memset(&buf[0], 0, PR_REG_ISID_LEN); 350 se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess, 351 &buf[0], PR_REG_ISID_LEN); 352 se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]); 353 } 354 355 spin_lock_irq(&se_nacl->nacl_sess_lock); 356 /* 357 * The se_nacl->nacl_sess pointer will be set to the 358 * last active I_T Nexus for each struct se_node_acl. 359 */ 360 se_nacl->nacl_sess = se_sess; 361 362 list_add_tail(&se_sess->sess_acl_list, 363 &se_nacl->acl_sess_list); 364 spin_unlock_irq(&se_nacl->nacl_sess_lock); 365 } 366 list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list); 367 368 pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n", 369 se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr); 370 } 371 EXPORT_SYMBOL(__transport_register_session); 372 373 void transport_register_session( 374 struct se_portal_group *se_tpg, 375 struct se_node_acl *se_nacl, 376 struct se_session *se_sess, 377 void *fabric_sess_ptr) 378 { 379 unsigned long flags; 380 381 spin_lock_irqsave(&se_tpg->session_lock, flags); 382 __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr); 383 spin_unlock_irqrestore(&se_tpg->session_lock, flags); 384 } 385 EXPORT_SYMBOL(transport_register_session); 386 387 struct se_session * 388 target_alloc_session(struct se_portal_group *tpg, 389 unsigned int tag_num, unsigned int tag_size, 390 enum target_prot_op prot_op, 391 const char *initiatorname, void *private, 392 int (*callback)(struct se_portal_group *, 393 struct se_session *, void *)) 394 { 395 struct se_session *sess; 396 397 /* 398 * If the fabric driver is using percpu-ida based pre allocation 399 * of I/O descriptor tags, go ahead and perform that setup now.. 400 */ 401 if (tag_num != 0) 402 sess = transport_init_session_tags(tag_num, tag_size, prot_op); 403 else 404 sess = transport_init_session(prot_op); 405 406 if (IS_ERR(sess)) 407 return sess; 408 409 sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg, 410 (unsigned char *)initiatorname); 411 if (!sess->se_node_acl) { 412 transport_free_session(sess); 413 return ERR_PTR(-EACCES); 414 } 415 /* 416 * Go ahead and perform any remaining fabric setup that is 417 * required before transport_register_session(). 418 */ 419 if (callback != NULL) { 420 int rc = callback(tpg, sess, private); 421 if (rc) { 422 transport_free_session(sess); 423 return ERR_PTR(rc); 424 } 425 } 426 427 transport_register_session(tpg, sess->se_node_acl, sess, private); 428 return sess; 429 } 430 EXPORT_SYMBOL(target_alloc_session); 431 432 ssize_t target_show_dynamic_sessions(struct se_portal_group *se_tpg, char *page) 433 { 434 struct se_session *se_sess; 435 ssize_t len = 0; 436 437 spin_lock_bh(&se_tpg->session_lock); 438 list_for_each_entry(se_sess, &se_tpg->tpg_sess_list, sess_list) { 439 if (!se_sess->se_node_acl) 440 continue; 441 if (!se_sess->se_node_acl->dynamic_node_acl) 442 continue; 443 if (strlen(se_sess->se_node_acl->initiatorname) + 1 + len > PAGE_SIZE) 444 break; 445 446 len += snprintf(page + len, PAGE_SIZE - len, "%s\n", 447 se_sess->se_node_acl->initiatorname); 448 len += 1; /* Include NULL terminator */ 449 } 450 spin_unlock_bh(&se_tpg->session_lock); 451 452 return len; 453 } 454 EXPORT_SYMBOL(target_show_dynamic_sessions); 455 456 static void target_complete_nacl(struct kref *kref) 457 { 458 struct se_node_acl *nacl = container_of(kref, 459 struct se_node_acl, acl_kref); 460 struct se_portal_group *se_tpg = nacl->se_tpg; 461 462 if (!nacl->dynamic_stop) { 463 complete(&nacl->acl_free_comp); 464 return; 465 } 466 467 mutex_lock(&se_tpg->acl_node_mutex); 468 list_del_init(&nacl->acl_list); 469 mutex_unlock(&se_tpg->acl_node_mutex); 470 471 core_tpg_wait_for_nacl_pr_ref(nacl); 472 core_free_device_list_for_node(nacl, se_tpg); 473 kfree(nacl); 474 } 475 476 void target_put_nacl(struct se_node_acl *nacl) 477 { 478 kref_put(&nacl->acl_kref, target_complete_nacl); 479 } 480 EXPORT_SYMBOL(target_put_nacl); 481 482 void transport_deregister_session_configfs(struct se_session *se_sess) 483 { 484 struct se_node_acl *se_nacl; 485 unsigned long flags; 486 /* 487 * Used by struct se_node_acl's under ConfigFS to locate active struct se_session 488 */ 489 se_nacl = se_sess->se_node_acl; 490 if (se_nacl) { 491 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags); 492 if (!list_empty(&se_sess->sess_acl_list)) 493 list_del_init(&se_sess->sess_acl_list); 494 /* 495 * If the session list is empty, then clear the pointer. 496 * Otherwise, set the struct se_session pointer from the tail 497 * element of the per struct se_node_acl active session list. 498 */ 499 if (list_empty(&se_nacl->acl_sess_list)) 500 se_nacl->nacl_sess = NULL; 501 else { 502 se_nacl->nacl_sess = container_of( 503 se_nacl->acl_sess_list.prev, 504 struct se_session, sess_acl_list); 505 } 506 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags); 507 } 508 } 509 EXPORT_SYMBOL(transport_deregister_session_configfs); 510 511 void transport_free_session(struct se_session *se_sess) 512 { 513 struct se_node_acl *se_nacl = se_sess->se_node_acl; 514 515 /* 516 * Drop the se_node_acl->nacl_kref obtained from within 517 * core_tpg_get_initiator_node_acl(). 518 */ 519 if (se_nacl) { 520 struct se_portal_group *se_tpg = se_nacl->se_tpg; 521 const struct target_core_fabric_ops *se_tfo = se_tpg->se_tpg_tfo; 522 unsigned long flags; 523 524 se_sess->se_node_acl = NULL; 525 526 /* 527 * Also determine if we need to drop the extra ->cmd_kref if 528 * it had been previously dynamically generated, and 529 * the endpoint is not caching dynamic ACLs. 530 */ 531 mutex_lock(&se_tpg->acl_node_mutex); 532 if (se_nacl->dynamic_node_acl && 533 !se_tfo->tpg_check_demo_mode_cache(se_tpg)) { 534 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags); 535 if (list_empty(&se_nacl->acl_sess_list)) 536 se_nacl->dynamic_stop = true; 537 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags); 538 539 if (se_nacl->dynamic_stop) 540 list_del_init(&se_nacl->acl_list); 541 } 542 mutex_unlock(&se_tpg->acl_node_mutex); 543 544 if (se_nacl->dynamic_stop) 545 target_put_nacl(se_nacl); 546 547 target_put_nacl(se_nacl); 548 } 549 if (se_sess->sess_cmd_map) { 550 percpu_ida_destroy(&se_sess->sess_tag_pool); 551 kvfree(se_sess->sess_cmd_map); 552 } 553 kmem_cache_free(se_sess_cache, se_sess); 554 } 555 EXPORT_SYMBOL(transport_free_session); 556 557 void transport_deregister_session(struct se_session *se_sess) 558 { 559 struct se_portal_group *se_tpg = se_sess->se_tpg; 560 unsigned long flags; 561 562 if (!se_tpg) { 563 transport_free_session(se_sess); 564 return; 565 } 566 567 spin_lock_irqsave(&se_tpg->session_lock, flags); 568 list_del(&se_sess->sess_list); 569 se_sess->se_tpg = NULL; 570 se_sess->fabric_sess_ptr = NULL; 571 spin_unlock_irqrestore(&se_tpg->session_lock, flags); 572 573 pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n", 574 se_tpg->se_tpg_tfo->get_fabric_name()); 575 /* 576 * If last kref is dropping now for an explicit NodeACL, awake sleeping 577 * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group 578 * removal context from within transport_free_session() code. 579 * 580 * For dynamic ACL, target_put_nacl() uses target_complete_nacl() 581 * to release all remaining generate_node_acl=1 created ACL resources. 582 */ 583 584 transport_free_session(se_sess); 585 } 586 EXPORT_SYMBOL(transport_deregister_session); 587 588 static void target_remove_from_state_list(struct se_cmd *cmd) 589 { 590 struct se_device *dev = cmd->se_dev; 591 unsigned long flags; 592 593 if (!dev) 594 return; 595 596 spin_lock_irqsave(&dev->execute_task_lock, flags); 597 if (cmd->state_active) { 598 list_del(&cmd->state_list); 599 cmd->state_active = false; 600 } 601 spin_unlock_irqrestore(&dev->execute_task_lock, flags); 602 } 603 604 static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd) 605 { 606 unsigned long flags; 607 608 target_remove_from_state_list(cmd); 609 610 /* 611 * Clear struct se_cmd->se_lun before the handoff to FE. 612 */ 613 cmd->se_lun = NULL; 614 615 spin_lock_irqsave(&cmd->t_state_lock, flags); 616 /* 617 * Determine if frontend context caller is requesting the stopping of 618 * this command for frontend exceptions. 619 */ 620 if (cmd->transport_state & CMD_T_STOP) { 621 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 622 __func__, __LINE__, cmd->tag); 623 624 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 625 626 complete_all(&cmd->t_transport_stop_comp); 627 return 1; 628 } 629 cmd->transport_state &= ~CMD_T_ACTIVE; 630 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 631 632 /* 633 * Some fabric modules like tcm_loop can release their internally 634 * allocated I/O reference and struct se_cmd now. 635 * 636 * Fabric modules are expected to return '1' here if the se_cmd being 637 * passed is released at this point, or zero if not being released. 638 */ 639 return cmd->se_tfo->check_stop_free(cmd); 640 } 641 642 static void transport_lun_remove_cmd(struct se_cmd *cmd) 643 { 644 struct se_lun *lun = cmd->se_lun; 645 646 if (!lun) 647 return; 648 649 if (cmpxchg(&cmd->lun_ref_active, true, false)) 650 percpu_ref_put(&lun->lun_ref); 651 } 652 653 int transport_cmd_finish_abort(struct se_cmd *cmd, int remove) 654 { 655 bool ack_kref = (cmd->se_cmd_flags & SCF_ACK_KREF); 656 int ret = 0; 657 658 if (cmd->se_cmd_flags & SCF_SE_LUN_CMD) 659 transport_lun_remove_cmd(cmd); 660 /* 661 * Allow the fabric driver to unmap any resources before 662 * releasing the descriptor via TFO->release_cmd() 663 */ 664 if (remove) 665 cmd->se_tfo->aborted_task(cmd); 666 667 if (transport_cmd_check_stop_to_fabric(cmd)) 668 return 1; 669 if (remove && ack_kref) 670 ret = target_put_sess_cmd(cmd); 671 672 return ret; 673 } 674 675 static void target_complete_failure_work(struct work_struct *work) 676 { 677 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 678 679 transport_generic_request_failure(cmd, 680 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); 681 } 682 683 /* 684 * Used when asking transport to copy Sense Data from the underlying 685 * Linux/SCSI struct scsi_cmnd 686 */ 687 static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd) 688 { 689 struct se_device *dev = cmd->se_dev; 690 691 WARN_ON(!cmd->se_lun); 692 693 if (!dev) 694 return NULL; 695 696 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) 697 return NULL; 698 699 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 700 701 pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n", 702 dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status); 703 return cmd->sense_buffer; 704 } 705 706 void transport_copy_sense_to_cmd(struct se_cmd *cmd, unsigned char *sense) 707 { 708 unsigned char *cmd_sense_buf; 709 unsigned long flags; 710 711 spin_lock_irqsave(&cmd->t_state_lock, flags); 712 cmd_sense_buf = transport_get_sense_buffer(cmd); 713 if (!cmd_sense_buf) { 714 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 715 return; 716 } 717 718 cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE; 719 memcpy(cmd_sense_buf, sense, cmd->scsi_sense_length); 720 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 721 } 722 EXPORT_SYMBOL(transport_copy_sense_to_cmd); 723 724 void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) 725 { 726 struct se_device *dev = cmd->se_dev; 727 int success; 728 unsigned long flags; 729 730 cmd->scsi_status = scsi_status; 731 732 spin_lock_irqsave(&cmd->t_state_lock, flags); 733 switch (cmd->scsi_status) { 734 case SAM_STAT_CHECK_CONDITION: 735 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) 736 success = 1; 737 else 738 success = 0; 739 break; 740 default: 741 success = 1; 742 break; 743 } 744 745 /* 746 * Check for case where an explicit ABORT_TASK has been received 747 * and transport_wait_for_tasks() will be waiting for completion.. 748 */ 749 if (cmd->transport_state & CMD_T_ABORTED || 750 cmd->transport_state & CMD_T_STOP) { 751 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 752 /* 753 * If COMPARE_AND_WRITE was stopped by __transport_wait_for_tasks(), 754 * release se_device->caw_sem obtained by sbc_compare_and_write() 755 * since target_complete_ok_work() or target_complete_failure_work() 756 * won't be called to invoke the normal CAW completion callbacks. 757 */ 758 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) { 759 up(&dev->caw_sem); 760 } 761 complete_all(&cmd->t_transport_stop_comp); 762 return; 763 } else if (!success) { 764 INIT_WORK(&cmd->work, target_complete_failure_work); 765 } else { 766 INIT_WORK(&cmd->work, target_complete_ok_work); 767 } 768 769 cmd->t_state = TRANSPORT_COMPLETE; 770 cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE); 771 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 772 773 if (cmd->se_cmd_flags & SCF_USE_CPUID) 774 queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work); 775 else 776 queue_work(target_completion_wq, &cmd->work); 777 } 778 EXPORT_SYMBOL(target_complete_cmd); 779 780 void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length) 781 { 782 if (scsi_status == SAM_STAT_GOOD && length < cmd->data_length) { 783 if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { 784 cmd->residual_count += cmd->data_length - length; 785 } else { 786 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 787 cmd->residual_count = cmd->data_length - length; 788 } 789 790 cmd->data_length = length; 791 } 792 793 target_complete_cmd(cmd, scsi_status); 794 } 795 EXPORT_SYMBOL(target_complete_cmd_with_length); 796 797 static void target_add_to_state_list(struct se_cmd *cmd) 798 { 799 struct se_device *dev = cmd->se_dev; 800 unsigned long flags; 801 802 spin_lock_irqsave(&dev->execute_task_lock, flags); 803 if (!cmd->state_active) { 804 list_add_tail(&cmd->state_list, &dev->state_list); 805 cmd->state_active = true; 806 } 807 spin_unlock_irqrestore(&dev->execute_task_lock, flags); 808 } 809 810 /* 811 * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status 812 */ 813 static void transport_write_pending_qf(struct se_cmd *cmd); 814 static void transport_complete_qf(struct se_cmd *cmd); 815 816 void target_qf_do_work(struct work_struct *work) 817 { 818 struct se_device *dev = container_of(work, struct se_device, 819 qf_work_queue); 820 LIST_HEAD(qf_cmd_list); 821 struct se_cmd *cmd, *cmd_tmp; 822 823 spin_lock_irq(&dev->qf_cmd_lock); 824 list_splice_init(&dev->qf_cmd_list, &qf_cmd_list); 825 spin_unlock_irq(&dev->qf_cmd_lock); 826 827 list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) { 828 list_del(&cmd->se_qf_node); 829 atomic_dec_mb(&dev->dev_qf_count); 830 831 pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue" 832 " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd, 833 (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" : 834 (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING" 835 : "UNKNOWN"); 836 837 if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) 838 transport_write_pending_qf(cmd); 839 else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK || 840 cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) 841 transport_complete_qf(cmd); 842 } 843 } 844 845 unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd) 846 { 847 switch (cmd->data_direction) { 848 case DMA_NONE: 849 return "NONE"; 850 case DMA_FROM_DEVICE: 851 return "READ"; 852 case DMA_TO_DEVICE: 853 return "WRITE"; 854 case DMA_BIDIRECTIONAL: 855 return "BIDI"; 856 default: 857 break; 858 } 859 860 return "UNKNOWN"; 861 } 862 863 void transport_dump_dev_state( 864 struct se_device *dev, 865 char *b, 866 int *bl) 867 { 868 *bl += sprintf(b + *bl, "Status: "); 869 if (dev->export_count) 870 *bl += sprintf(b + *bl, "ACTIVATED"); 871 else 872 *bl += sprintf(b + *bl, "DEACTIVATED"); 873 874 *bl += sprintf(b + *bl, " Max Queue Depth: %d", dev->queue_depth); 875 *bl += sprintf(b + *bl, " SectorSize: %u HwMaxSectors: %u\n", 876 dev->dev_attrib.block_size, 877 dev->dev_attrib.hw_max_sectors); 878 *bl += sprintf(b + *bl, " "); 879 } 880 881 void transport_dump_vpd_proto_id( 882 struct t10_vpd *vpd, 883 unsigned char *p_buf, 884 int p_buf_len) 885 { 886 unsigned char buf[VPD_TMP_BUF_SIZE]; 887 int len; 888 889 memset(buf, 0, VPD_TMP_BUF_SIZE); 890 len = sprintf(buf, "T10 VPD Protocol Identifier: "); 891 892 switch (vpd->protocol_identifier) { 893 case 0x00: 894 sprintf(buf+len, "Fibre Channel\n"); 895 break; 896 case 0x10: 897 sprintf(buf+len, "Parallel SCSI\n"); 898 break; 899 case 0x20: 900 sprintf(buf+len, "SSA\n"); 901 break; 902 case 0x30: 903 sprintf(buf+len, "IEEE 1394\n"); 904 break; 905 case 0x40: 906 sprintf(buf+len, "SCSI Remote Direct Memory Access" 907 " Protocol\n"); 908 break; 909 case 0x50: 910 sprintf(buf+len, "Internet SCSI (iSCSI)\n"); 911 break; 912 case 0x60: 913 sprintf(buf+len, "SAS Serial SCSI Protocol\n"); 914 break; 915 case 0x70: 916 sprintf(buf+len, "Automation/Drive Interface Transport" 917 " Protocol\n"); 918 break; 919 case 0x80: 920 sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n"); 921 break; 922 default: 923 sprintf(buf+len, "Unknown 0x%02x\n", 924 vpd->protocol_identifier); 925 break; 926 } 927 928 if (p_buf) 929 strncpy(p_buf, buf, p_buf_len); 930 else 931 pr_debug("%s", buf); 932 } 933 934 void 935 transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83) 936 { 937 /* 938 * Check if the Protocol Identifier Valid (PIV) bit is set.. 939 * 940 * from spc3r23.pdf section 7.5.1 941 */ 942 if (page_83[1] & 0x80) { 943 vpd->protocol_identifier = (page_83[0] & 0xf0); 944 vpd->protocol_identifier_set = 1; 945 transport_dump_vpd_proto_id(vpd, NULL, 0); 946 } 947 } 948 EXPORT_SYMBOL(transport_set_vpd_proto_id); 949 950 int transport_dump_vpd_assoc( 951 struct t10_vpd *vpd, 952 unsigned char *p_buf, 953 int p_buf_len) 954 { 955 unsigned char buf[VPD_TMP_BUF_SIZE]; 956 int ret = 0; 957 int len; 958 959 memset(buf, 0, VPD_TMP_BUF_SIZE); 960 len = sprintf(buf, "T10 VPD Identifier Association: "); 961 962 switch (vpd->association) { 963 case 0x00: 964 sprintf(buf+len, "addressed logical unit\n"); 965 break; 966 case 0x10: 967 sprintf(buf+len, "target port\n"); 968 break; 969 case 0x20: 970 sprintf(buf+len, "SCSI target device\n"); 971 break; 972 default: 973 sprintf(buf+len, "Unknown 0x%02x\n", vpd->association); 974 ret = -EINVAL; 975 break; 976 } 977 978 if (p_buf) 979 strncpy(p_buf, buf, p_buf_len); 980 else 981 pr_debug("%s", buf); 982 983 return ret; 984 } 985 986 int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83) 987 { 988 /* 989 * The VPD identification association.. 990 * 991 * from spc3r23.pdf Section 7.6.3.1 Table 297 992 */ 993 vpd->association = (page_83[1] & 0x30); 994 return transport_dump_vpd_assoc(vpd, NULL, 0); 995 } 996 EXPORT_SYMBOL(transport_set_vpd_assoc); 997 998 int transport_dump_vpd_ident_type( 999 struct t10_vpd *vpd, 1000 unsigned char *p_buf, 1001 int p_buf_len) 1002 { 1003 unsigned char buf[VPD_TMP_BUF_SIZE]; 1004 int ret = 0; 1005 int len; 1006 1007 memset(buf, 0, VPD_TMP_BUF_SIZE); 1008 len = sprintf(buf, "T10 VPD Identifier Type: "); 1009 1010 switch (vpd->device_identifier_type) { 1011 case 0x00: 1012 sprintf(buf+len, "Vendor specific\n"); 1013 break; 1014 case 0x01: 1015 sprintf(buf+len, "T10 Vendor ID based\n"); 1016 break; 1017 case 0x02: 1018 sprintf(buf+len, "EUI-64 based\n"); 1019 break; 1020 case 0x03: 1021 sprintf(buf+len, "NAA\n"); 1022 break; 1023 case 0x04: 1024 sprintf(buf+len, "Relative target port identifier\n"); 1025 break; 1026 case 0x08: 1027 sprintf(buf+len, "SCSI name string\n"); 1028 break; 1029 default: 1030 sprintf(buf+len, "Unsupported: 0x%02x\n", 1031 vpd->device_identifier_type); 1032 ret = -EINVAL; 1033 break; 1034 } 1035 1036 if (p_buf) { 1037 if (p_buf_len < strlen(buf)+1) 1038 return -EINVAL; 1039 strncpy(p_buf, buf, p_buf_len); 1040 } else { 1041 pr_debug("%s", buf); 1042 } 1043 1044 return ret; 1045 } 1046 1047 int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83) 1048 { 1049 /* 1050 * The VPD identifier type.. 1051 * 1052 * from spc3r23.pdf Section 7.6.3.1 Table 298 1053 */ 1054 vpd->device_identifier_type = (page_83[1] & 0x0f); 1055 return transport_dump_vpd_ident_type(vpd, NULL, 0); 1056 } 1057 EXPORT_SYMBOL(transport_set_vpd_ident_type); 1058 1059 int transport_dump_vpd_ident( 1060 struct t10_vpd *vpd, 1061 unsigned char *p_buf, 1062 int p_buf_len) 1063 { 1064 unsigned char buf[VPD_TMP_BUF_SIZE]; 1065 int ret = 0; 1066 1067 memset(buf, 0, VPD_TMP_BUF_SIZE); 1068 1069 switch (vpd->device_identifier_code_set) { 1070 case 0x01: /* Binary */ 1071 snprintf(buf, sizeof(buf), 1072 "T10 VPD Binary Device Identifier: %s\n", 1073 &vpd->device_identifier[0]); 1074 break; 1075 case 0x02: /* ASCII */ 1076 snprintf(buf, sizeof(buf), 1077 "T10 VPD ASCII Device Identifier: %s\n", 1078 &vpd->device_identifier[0]); 1079 break; 1080 case 0x03: /* UTF-8 */ 1081 snprintf(buf, sizeof(buf), 1082 "T10 VPD UTF-8 Device Identifier: %s\n", 1083 &vpd->device_identifier[0]); 1084 break; 1085 default: 1086 sprintf(buf, "T10 VPD Device Identifier encoding unsupported:" 1087 " 0x%02x", vpd->device_identifier_code_set); 1088 ret = -EINVAL; 1089 break; 1090 } 1091 1092 if (p_buf) 1093 strncpy(p_buf, buf, p_buf_len); 1094 else 1095 pr_debug("%s", buf); 1096 1097 return ret; 1098 } 1099 1100 int 1101 transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83) 1102 { 1103 static const char hex_str[] = "0123456789abcdef"; 1104 int j = 0, i = 4; /* offset to start of the identifier */ 1105 1106 /* 1107 * The VPD Code Set (encoding) 1108 * 1109 * from spc3r23.pdf Section 7.6.3.1 Table 296 1110 */ 1111 vpd->device_identifier_code_set = (page_83[0] & 0x0f); 1112 switch (vpd->device_identifier_code_set) { 1113 case 0x01: /* Binary */ 1114 vpd->device_identifier[j++] = 1115 hex_str[vpd->device_identifier_type]; 1116 while (i < (4 + page_83[3])) { 1117 vpd->device_identifier[j++] = 1118 hex_str[(page_83[i] & 0xf0) >> 4]; 1119 vpd->device_identifier[j++] = 1120 hex_str[page_83[i] & 0x0f]; 1121 i++; 1122 } 1123 break; 1124 case 0x02: /* ASCII */ 1125 case 0x03: /* UTF-8 */ 1126 while (i < (4 + page_83[3])) 1127 vpd->device_identifier[j++] = page_83[i++]; 1128 break; 1129 default: 1130 break; 1131 } 1132 1133 return transport_dump_vpd_ident(vpd, NULL, 0); 1134 } 1135 EXPORT_SYMBOL(transport_set_vpd_ident); 1136 1137 static sense_reason_t 1138 target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev, 1139 unsigned int size) 1140 { 1141 u32 mtl; 1142 1143 if (!cmd->se_tfo->max_data_sg_nents) 1144 return TCM_NO_SENSE; 1145 /* 1146 * Check if fabric enforced maximum SGL entries per I/O descriptor 1147 * exceeds se_cmd->data_length. If true, set SCF_UNDERFLOW_BIT + 1148 * residual_count and reduce original cmd->data_length to maximum 1149 * length based on single PAGE_SIZE entry scatter-lists. 1150 */ 1151 mtl = (cmd->se_tfo->max_data_sg_nents * PAGE_SIZE); 1152 if (cmd->data_length > mtl) { 1153 /* 1154 * If an existing CDB overflow is present, calculate new residual 1155 * based on CDB size minus fabric maximum transfer length. 1156 * 1157 * If an existing CDB underflow is present, calculate new residual 1158 * based on original cmd->data_length minus fabric maximum transfer 1159 * length. 1160 * 1161 * Otherwise, set the underflow residual based on cmd->data_length 1162 * minus fabric maximum transfer length. 1163 */ 1164 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) { 1165 cmd->residual_count = (size - mtl); 1166 } else if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { 1167 u32 orig_dl = size + cmd->residual_count; 1168 cmd->residual_count = (orig_dl - mtl); 1169 } else { 1170 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 1171 cmd->residual_count = (cmd->data_length - mtl); 1172 } 1173 cmd->data_length = mtl; 1174 /* 1175 * Reset sbc_check_prot() calculated protection payload 1176 * length based upon the new smaller MTL. 1177 */ 1178 if (cmd->prot_length) { 1179 u32 sectors = (mtl / dev->dev_attrib.block_size); 1180 cmd->prot_length = dev->prot_length * sectors; 1181 } 1182 } 1183 return TCM_NO_SENSE; 1184 } 1185 1186 sense_reason_t 1187 target_cmd_size_check(struct se_cmd *cmd, unsigned int size) 1188 { 1189 struct se_device *dev = cmd->se_dev; 1190 1191 if (cmd->unknown_data_length) { 1192 cmd->data_length = size; 1193 } else if (size != cmd->data_length) { 1194 pr_warn_ratelimited("TARGET_CORE[%s]: Expected Transfer Length:" 1195 " %u does not match SCSI CDB Length: %u for SAM Opcode:" 1196 " 0x%02x\n", cmd->se_tfo->get_fabric_name(), 1197 cmd->data_length, size, cmd->t_task_cdb[0]); 1198 1199 if (cmd->data_direction == DMA_TO_DEVICE) { 1200 if (cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) { 1201 pr_err_ratelimited("Rejecting underflow/overflow" 1202 " for WRITE data CDB\n"); 1203 return TCM_INVALID_CDB_FIELD; 1204 } 1205 /* 1206 * Some fabric drivers like iscsi-target still expect to 1207 * always reject overflow writes. Reject this case until 1208 * full fabric driver level support for overflow writes 1209 * is introduced tree-wide. 1210 */ 1211 if (size > cmd->data_length) { 1212 pr_err_ratelimited("Rejecting overflow for" 1213 " WRITE control CDB\n"); 1214 return TCM_INVALID_CDB_FIELD; 1215 } 1216 } 1217 /* 1218 * Reject READ_* or WRITE_* with overflow/underflow for 1219 * type SCF_SCSI_DATA_CDB. 1220 */ 1221 if (dev->dev_attrib.block_size != 512) { 1222 pr_err("Failing OVERFLOW/UNDERFLOW for LBA op" 1223 " CDB on non 512-byte sector setup subsystem" 1224 " plugin: %s\n", dev->transport->name); 1225 /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */ 1226 return TCM_INVALID_CDB_FIELD; 1227 } 1228 /* 1229 * For the overflow case keep the existing fabric provided 1230 * ->data_length. Otherwise for the underflow case, reset 1231 * ->data_length to the smaller SCSI expected data transfer 1232 * length. 1233 */ 1234 if (size > cmd->data_length) { 1235 cmd->se_cmd_flags |= SCF_OVERFLOW_BIT; 1236 cmd->residual_count = (size - cmd->data_length); 1237 } else { 1238 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; 1239 cmd->residual_count = (cmd->data_length - size); 1240 cmd->data_length = size; 1241 } 1242 } 1243 1244 return target_check_max_data_sg_nents(cmd, dev, size); 1245 1246 } 1247 1248 /* 1249 * Used by fabric modules containing a local struct se_cmd within their 1250 * fabric dependent per I/O descriptor. 1251 * 1252 * Preserves the value of @cmd->tag. 1253 */ 1254 void transport_init_se_cmd( 1255 struct se_cmd *cmd, 1256 const struct target_core_fabric_ops *tfo, 1257 struct se_session *se_sess, 1258 u32 data_length, 1259 int data_direction, 1260 int task_attr, 1261 unsigned char *sense_buffer) 1262 { 1263 INIT_LIST_HEAD(&cmd->se_delayed_node); 1264 INIT_LIST_HEAD(&cmd->se_qf_node); 1265 INIT_LIST_HEAD(&cmd->se_cmd_list); 1266 INIT_LIST_HEAD(&cmd->state_list); 1267 init_completion(&cmd->t_transport_stop_comp); 1268 init_completion(&cmd->cmd_wait_comp); 1269 spin_lock_init(&cmd->t_state_lock); 1270 INIT_WORK(&cmd->work, NULL); 1271 kref_init(&cmd->cmd_kref); 1272 1273 cmd->se_tfo = tfo; 1274 cmd->se_sess = se_sess; 1275 cmd->data_length = data_length; 1276 cmd->data_direction = data_direction; 1277 cmd->sam_task_attr = task_attr; 1278 cmd->sense_buffer = sense_buffer; 1279 1280 cmd->state_active = false; 1281 } 1282 EXPORT_SYMBOL(transport_init_se_cmd); 1283 1284 static sense_reason_t 1285 transport_check_alloc_task_attr(struct se_cmd *cmd) 1286 { 1287 struct se_device *dev = cmd->se_dev; 1288 1289 /* 1290 * Check if SAM Task Attribute emulation is enabled for this 1291 * struct se_device storage object 1292 */ 1293 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1294 return 0; 1295 1296 if (cmd->sam_task_attr == TCM_ACA_TAG) { 1297 pr_debug("SAM Task Attribute ACA" 1298 " emulation is not supported\n"); 1299 return TCM_INVALID_CDB_FIELD; 1300 } 1301 1302 return 0; 1303 } 1304 1305 sense_reason_t 1306 target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) 1307 { 1308 struct se_device *dev = cmd->se_dev; 1309 sense_reason_t ret; 1310 1311 /* 1312 * Ensure that the received CDB is less than the max (252 + 8) bytes 1313 * for VARIABLE_LENGTH_CMD 1314 */ 1315 if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) { 1316 pr_err("Received SCSI CDB with command_size: %d that" 1317 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", 1318 scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE); 1319 return TCM_INVALID_CDB_FIELD; 1320 } 1321 /* 1322 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE, 1323 * allocate the additional extended CDB buffer now.. Otherwise 1324 * setup the pointer from __t_task_cdb to t_task_cdb. 1325 */ 1326 if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) { 1327 cmd->t_task_cdb = kzalloc(scsi_command_size(cdb), 1328 GFP_KERNEL); 1329 if (!cmd->t_task_cdb) { 1330 pr_err("Unable to allocate cmd->t_task_cdb" 1331 " %u > sizeof(cmd->__t_task_cdb): %lu ops\n", 1332 scsi_command_size(cdb), 1333 (unsigned long)sizeof(cmd->__t_task_cdb)); 1334 return TCM_OUT_OF_RESOURCES; 1335 } 1336 } else 1337 cmd->t_task_cdb = &cmd->__t_task_cdb[0]; 1338 /* 1339 * Copy the original CDB into cmd-> 1340 */ 1341 memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb)); 1342 1343 trace_target_sequencer_start(cmd); 1344 1345 ret = dev->transport->parse_cdb(cmd); 1346 if (ret == TCM_UNSUPPORTED_SCSI_OPCODE) 1347 pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n", 1348 cmd->se_tfo->get_fabric_name(), 1349 cmd->se_sess->se_node_acl->initiatorname, 1350 cmd->t_task_cdb[0]); 1351 if (ret) 1352 return ret; 1353 1354 ret = transport_check_alloc_task_attr(cmd); 1355 if (ret) 1356 return ret; 1357 1358 cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE; 1359 atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus); 1360 return 0; 1361 } 1362 EXPORT_SYMBOL(target_setup_cmd_from_cdb); 1363 1364 /* 1365 * Used by fabric module frontends to queue tasks directly. 1366 * May only be used from process context. 1367 */ 1368 int transport_handle_cdb_direct( 1369 struct se_cmd *cmd) 1370 { 1371 sense_reason_t ret; 1372 1373 if (!cmd->se_lun) { 1374 dump_stack(); 1375 pr_err("cmd->se_lun is NULL\n"); 1376 return -EINVAL; 1377 } 1378 if (in_interrupt()) { 1379 dump_stack(); 1380 pr_err("transport_generic_handle_cdb cannot be called" 1381 " from interrupt context\n"); 1382 return -EINVAL; 1383 } 1384 /* 1385 * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that 1386 * outstanding descriptors are handled correctly during shutdown via 1387 * transport_wait_for_tasks() 1388 * 1389 * Also, we don't take cmd->t_state_lock here as we only expect 1390 * this to be called for initial descriptor submission. 1391 */ 1392 cmd->t_state = TRANSPORT_NEW_CMD; 1393 cmd->transport_state |= CMD_T_ACTIVE; 1394 1395 /* 1396 * transport_generic_new_cmd() is already handling QUEUE_FULL, 1397 * so follow TRANSPORT_NEW_CMD processing thread context usage 1398 * and call transport_generic_request_failure() if necessary.. 1399 */ 1400 ret = transport_generic_new_cmd(cmd); 1401 if (ret) 1402 transport_generic_request_failure(cmd, ret); 1403 return 0; 1404 } 1405 EXPORT_SYMBOL(transport_handle_cdb_direct); 1406 1407 sense_reason_t 1408 transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl, 1409 u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count) 1410 { 1411 if (!sgl || !sgl_count) 1412 return 0; 1413 1414 /* 1415 * Reject SCSI data overflow with map_mem_to_cmd() as incoming 1416 * scatterlists already have been set to follow what the fabric 1417 * passes for the original expected data transfer length. 1418 */ 1419 if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) { 1420 pr_warn("Rejecting SCSI DATA overflow for fabric using" 1421 " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n"); 1422 return TCM_INVALID_CDB_FIELD; 1423 } 1424 1425 cmd->t_data_sg = sgl; 1426 cmd->t_data_nents = sgl_count; 1427 cmd->t_bidi_data_sg = sgl_bidi; 1428 cmd->t_bidi_data_nents = sgl_bidi_count; 1429 1430 cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1431 return 0; 1432 } 1433 1434 /* 1435 * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized 1436 * se_cmd + use pre-allocated SGL memory. 1437 * 1438 * @se_cmd: command descriptor to submit 1439 * @se_sess: associated se_sess for endpoint 1440 * @cdb: pointer to SCSI CDB 1441 * @sense: pointer to SCSI sense buffer 1442 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1443 * @data_length: fabric expected data transfer length 1444 * @task_addr: SAM task attribute 1445 * @data_dir: DMA data direction 1446 * @flags: flags for command submission from target_sc_flags_tables 1447 * @sgl: struct scatterlist memory for unidirectional mapping 1448 * @sgl_count: scatterlist count for unidirectional mapping 1449 * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping 1450 * @sgl_bidi_count: scatterlist count for bidirectional READ mapping 1451 * @sgl_prot: struct scatterlist memory protection information 1452 * @sgl_prot_count: scatterlist count for protection information 1453 * 1454 * Task tags are supported if the caller has set @se_cmd->tag. 1455 * 1456 * Returns non zero to signal active I/O shutdown failure. All other 1457 * setup exceptions will be returned as a SCSI CHECK_CONDITION response, 1458 * but still return zero here. 1459 * 1460 * This may only be called from process context, and also currently 1461 * assumes internal allocation of fabric payload buffer by target-core. 1462 */ 1463 int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess, 1464 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, 1465 u32 data_length, int task_attr, int data_dir, int flags, 1466 struct scatterlist *sgl, u32 sgl_count, 1467 struct scatterlist *sgl_bidi, u32 sgl_bidi_count, 1468 struct scatterlist *sgl_prot, u32 sgl_prot_count) 1469 { 1470 struct se_portal_group *se_tpg; 1471 sense_reason_t rc; 1472 int ret; 1473 1474 se_tpg = se_sess->se_tpg; 1475 BUG_ON(!se_tpg); 1476 BUG_ON(se_cmd->se_tfo || se_cmd->se_sess); 1477 BUG_ON(in_interrupt()); 1478 /* 1479 * Initialize se_cmd for target operation. From this point 1480 * exceptions are handled by sending exception status via 1481 * target_core_fabric_ops->queue_status() callback 1482 */ 1483 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, 1484 data_length, data_dir, task_attr, sense); 1485 1486 if (flags & TARGET_SCF_USE_CPUID) 1487 se_cmd->se_cmd_flags |= SCF_USE_CPUID; 1488 else 1489 se_cmd->cpuid = WORK_CPU_UNBOUND; 1490 1491 if (flags & TARGET_SCF_UNKNOWN_SIZE) 1492 se_cmd->unknown_data_length = 1; 1493 /* 1494 * Obtain struct se_cmd->cmd_kref reference and add new cmd to 1495 * se_sess->sess_cmd_list. A second kref_get here is necessary 1496 * for fabrics using TARGET_SCF_ACK_KREF that expect a second 1497 * kref_put() to happen during fabric packet acknowledgement. 1498 */ 1499 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); 1500 if (ret) 1501 return ret; 1502 /* 1503 * Signal bidirectional data payloads to target-core 1504 */ 1505 if (flags & TARGET_SCF_BIDI_OP) 1506 se_cmd->se_cmd_flags |= SCF_BIDI; 1507 /* 1508 * Locate se_lun pointer and attach it to struct se_cmd 1509 */ 1510 rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun); 1511 if (rc) { 1512 transport_send_check_condition_and_sense(se_cmd, rc, 0); 1513 target_put_sess_cmd(se_cmd); 1514 return 0; 1515 } 1516 1517 rc = target_setup_cmd_from_cdb(se_cmd, cdb); 1518 if (rc != 0) { 1519 transport_generic_request_failure(se_cmd, rc); 1520 return 0; 1521 } 1522 1523 /* 1524 * Save pointers for SGLs containing protection information, 1525 * if present. 1526 */ 1527 if (sgl_prot_count) { 1528 se_cmd->t_prot_sg = sgl_prot; 1529 se_cmd->t_prot_nents = sgl_prot_count; 1530 se_cmd->se_cmd_flags |= SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC; 1531 } 1532 1533 /* 1534 * When a non zero sgl_count has been passed perform SGL passthrough 1535 * mapping for pre-allocated fabric memory instead of having target 1536 * core perform an internal SGL allocation.. 1537 */ 1538 if (sgl_count != 0) { 1539 BUG_ON(!sgl); 1540 1541 /* 1542 * A work-around for tcm_loop as some userspace code via 1543 * scsi-generic do not memset their associated read buffers, 1544 * so go ahead and do that here for type non-data CDBs. Also 1545 * note that this is currently guaranteed to be a single SGL 1546 * for this case by target core in target_setup_cmd_from_cdb() 1547 * -> transport_generic_cmd_sequencer(). 1548 */ 1549 if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) && 1550 se_cmd->data_direction == DMA_FROM_DEVICE) { 1551 unsigned char *buf = NULL; 1552 1553 if (sgl) 1554 buf = kmap(sg_page(sgl)) + sgl->offset; 1555 1556 if (buf) { 1557 memset(buf, 0, sgl->length); 1558 kunmap(sg_page(sgl)); 1559 } 1560 } 1561 1562 rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count, 1563 sgl_bidi, sgl_bidi_count); 1564 if (rc != 0) { 1565 transport_generic_request_failure(se_cmd, rc); 1566 return 0; 1567 } 1568 } 1569 1570 /* 1571 * Check if we need to delay processing because of ALUA 1572 * Active/NonOptimized primary access state.. 1573 */ 1574 core_alua_check_nonop_delay(se_cmd); 1575 1576 transport_handle_cdb_direct(se_cmd); 1577 return 0; 1578 } 1579 EXPORT_SYMBOL(target_submit_cmd_map_sgls); 1580 1581 /* 1582 * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd 1583 * 1584 * @se_cmd: command descriptor to submit 1585 * @se_sess: associated se_sess for endpoint 1586 * @cdb: pointer to SCSI CDB 1587 * @sense: pointer to SCSI sense buffer 1588 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1589 * @data_length: fabric expected data transfer length 1590 * @task_addr: SAM task attribute 1591 * @data_dir: DMA data direction 1592 * @flags: flags for command submission from target_sc_flags_tables 1593 * 1594 * Task tags are supported if the caller has set @se_cmd->tag. 1595 * 1596 * Returns non zero to signal active I/O shutdown failure. All other 1597 * setup exceptions will be returned as a SCSI CHECK_CONDITION response, 1598 * but still return zero here. 1599 * 1600 * This may only be called from process context, and also currently 1601 * assumes internal allocation of fabric payload buffer by target-core. 1602 * 1603 * It also assumes interal target core SGL memory allocation. 1604 */ 1605 int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, 1606 unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, 1607 u32 data_length, int task_attr, int data_dir, int flags) 1608 { 1609 return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense, 1610 unpacked_lun, data_length, task_attr, data_dir, 1611 flags, NULL, 0, NULL, 0, NULL, 0); 1612 } 1613 EXPORT_SYMBOL(target_submit_cmd); 1614 1615 static void target_complete_tmr_failure(struct work_struct *work) 1616 { 1617 struct se_cmd *se_cmd = container_of(work, struct se_cmd, work); 1618 1619 se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST; 1620 se_cmd->se_tfo->queue_tm_rsp(se_cmd); 1621 1622 transport_lun_remove_cmd(se_cmd); 1623 transport_cmd_check_stop_to_fabric(se_cmd); 1624 } 1625 1626 static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag, 1627 u64 *unpacked_lun) 1628 { 1629 struct se_cmd *se_cmd; 1630 unsigned long flags; 1631 bool ret = false; 1632 1633 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 1634 list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) { 1635 if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) 1636 continue; 1637 1638 if (se_cmd->tag == tag) { 1639 *unpacked_lun = se_cmd->orig_fe_lun; 1640 ret = true; 1641 break; 1642 } 1643 } 1644 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 1645 1646 return ret; 1647 } 1648 1649 /** 1650 * target_submit_tmr - lookup unpacked lun and submit uninitialized se_cmd 1651 * for TMR CDBs 1652 * 1653 * @se_cmd: command descriptor to submit 1654 * @se_sess: associated se_sess for endpoint 1655 * @sense: pointer to SCSI sense buffer 1656 * @unpacked_lun: unpacked LUN to reference for struct se_lun 1657 * @fabric_context: fabric context for TMR req 1658 * @tm_type: Type of TM request 1659 * @gfp: gfp type for caller 1660 * @tag: referenced task tag for TMR_ABORT_TASK 1661 * @flags: submit cmd flags 1662 * 1663 * Callable from all contexts. 1664 **/ 1665 1666 int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, 1667 unsigned char *sense, u64 unpacked_lun, 1668 void *fabric_tmr_ptr, unsigned char tm_type, 1669 gfp_t gfp, u64 tag, int flags) 1670 { 1671 struct se_portal_group *se_tpg; 1672 int ret; 1673 1674 se_tpg = se_sess->se_tpg; 1675 BUG_ON(!se_tpg); 1676 1677 transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, 1678 0, DMA_NONE, TCM_SIMPLE_TAG, sense); 1679 /* 1680 * FIXME: Currently expect caller to handle se_cmd->se_tmr_req 1681 * allocation failure. 1682 */ 1683 ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp); 1684 if (ret < 0) 1685 return -ENOMEM; 1686 1687 if (tm_type == TMR_ABORT_TASK) 1688 se_cmd->se_tmr_req->ref_task_tag = tag; 1689 1690 /* See target_submit_cmd for commentary */ 1691 ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); 1692 if (ret) { 1693 core_tmr_release_req(se_cmd->se_tmr_req); 1694 return ret; 1695 } 1696 /* 1697 * If this is ABORT_TASK with no explicit fabric provided LUN, 1698 * go ahead and search active session tags for a match to figure 1699 * out unpacked_lun for the original se_cmd. 1700 */ 1701 if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) { 1702 if (!target_lookup_lun_from_tag(se_sess, tag, &unpacked_lun)) 1703 goto failure; 1704 } 1705 1706 ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun); 1707 if (ret) 1708 goto failure; 1709 1710 transport_generic_handle_tmr(se_cmd); 1711 return 0; 1712 1713 /* 1714 * For callback during failure handling, push this work off 1715 * to process context with TMR_LUN_DOES_NOT_EXIST status. 1716 */ 1717 failure: 1718 INIT_WORK(&se_cmd->work, target_complete_tmr_failure); 1719 schedule_work(&se_cmd->work); 1720 return 0; 1721 } 1722 EXPORT_SYMBOL(target_submit_tmr); 1723 1724 /* 1725 * Handle SAM-esque emulation for generic transport request failures. 1726 */ 1727 void transport_generic_request_failure(struct se_cmd *cmd, 1728 sense_reason_t sense_reason) 1729 { 1730 int ret = 0, post_ret = 0; 1731 1732 pr_debug("-----[ Storage Engine Exception; sense_reason %d\n", 1733 sense_reason); 1734 target_show_cmd("-----[ ", cmd); 1735 1736 /* 1737 * For SAM Task Attribute emulation for failed struct se_cmd 1738 */ 1739 transport_complete_task_attr(cmd); 1740 1741 /* 1742 * Handle special case for COMPARE_AND_WRITE failure, where the 1743 * callback is expected to drop the per device ->caw_sem. 1744 */ 1745 if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) && 1746 cmd->transport_complete_callback) 1747 cmd->transport_complete_callback(cmd, false, &post_ret); 1748 1749 if (transport_check_aborted_status(cmd, 1)) 1750 return; 1751 1752 switch (sense_reason) { 1753 case TCM_NON_EXISTENT_LUN: 1754 case TCM_UNSUPPORTED_SCSI_OPCODE: 1755 case TCM_INVALID_CDB_FIELD: 1756 case TCM_INVALID_PARAMETER_LIST: 1757 case TCM_PARAMETER_LIST_LENGTH_ERROR: 1758 case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE: 1759 case TCM_UNKNOWN_MODE_PAGE: 1760 case TCM_WRITE_PROTECTED: 1761 case TCM_ADDRESS_OUT_OF_RANGE: 1762 case TCM_CHECK_CONDITION_ABORT_CMD: 1763 case TCM_CHECK_CONDITION_UNIT_ATTENTION: 1764 case TCM_CHECK_CONDITION_NOT_READY: 1765 case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED: 1766 case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED: 1767 case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED: 1768 case TCM_COPY_TARGET_DEVICE_NOT_REACHABLE: 1769 case TCM_TOO_MANY_TARGET_DESCS: 1770 case TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE: 1771 case TCM_TOO_MANY_SEGMENT_DESCS: 1772 case TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE: 1773 break; 1774 case TCM_OUT_OF_RESOURCES: 1775 cmd->scsi_status = SAM_STAT_TASK_SET_FULL; 1776 goto queue_status; 1777 case TCM_RESERVATION_CONFLICT: 1778 /* 1779 * No SENSE Data payload for this case, set SCSI Status 1780 * and queue the response to $FABRIC_MOD. 1781 * 1782 * Uses linux/include/scsi/scsi.h SAM status codes defs 1783 */ 1784 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT; 1785 /* 1786 * For UA Interlock Code 11b, a RESERVATION CONFLICT will 1787 * establish a UNIT ATTENTION with PREVIOUS RESERVATION 1788 * CONFLICT STATUS. 1789 * 1790 * See spc4r17, section 7.4.6 Control Mode Page, Table 349 1791 */ 1792 if (cmd->se_sess && 1793 cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) { 1794 target_ua_allocate_lun(cmd->se_sess->se_node_acl, 1795 cmd->orig_fe_lun, 0x2C, 1796 ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS); 1797 } 1798 1799 goto queue_status; 1800 default: 1801 pr_err("Unknown transport error for CDB 0x%02x: %d\n", 1802 cmd->t_task_cdb[0], sense_reason); 1803 sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE; 1804 break; 1805 } 1806 1807 ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0); 1808 if (ret) 1809 goto queue_full; 1810 1811 check_stop: 1812 transport_lun_remove_cmd(cmd); 1813 transport_cmd_check_stop_to_fabric(cmd); 1814 return; 1815 1816 queue_status: 1817 trace_target_cmd_complete(cmd); 1818 ret = cmd->se_tfo->queue_status(cmd); 1819 if (!ret) 1820 goto check_stop; 1821 queue_full: 1822 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 1823 } 1824 EXPORT_SYMBOL(transport_generic_request_failure); 1825 1826 void __target_execute_cmd(struct se_cmd *cmd, bool do_checks) 1827 { 1828 sense_reason_t ret; 1829 1830 if (!cmd->execute_cmd) { 1831 ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 1832 goto err; 1833 } 1834 if (do_checks) { 1835 /* 1836 * Check for an existing UNIT ATTENTION condition after 1837 * target_handle_task_attr() has done SAM task attr 1838 * checking, and possibly have already defered execution 1839 * out to target_restart_delayed_cmds() context. 1840 */ 1841 ret = target_scsi3_ua_check(cmd); 1842 if (ret) 1843 goto err; 1844 1845 ret = target_alua_state_check(cmd); 1846 if (ret) 1847 goto err; 1848 1849 ret = target_check_reservation(cmd); 1850 if (ret) { 1851 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT; 1852 goto err; 1853 } 1854 } 1855 1856 ret = cmd->execute_cmd(cmd); 1857 if (!ret) 1858 return; 1859 err: 1860 spin_lock_irq(&cmd->t_state_lock); 1861 cmd->transport_state &= ~CMD_T_SENT; 1862 spin_unlock_irq(&cmd->t_state_lock); 1863 1864 transport_generic_request_failure(cmd, ret); 1865 } 1866 1867 static int target_write_prot_action(struct se_cmd *cmd) 1868 { 1869 u32 sectors; 1870 /* 1871 * Perform WRITE_INSERT of PI using software emulation when backend 1872 * device has PI enabled, if the transport has not already generated 1873 * PI using hardware WRITE_INSERT offload. 1874 */ 1875 switch (cmd->prot_op) { 1876 case TARGET_PROT_DOUT_INSERT: 1877 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_INSERT)) 1878 sbc_dif_generate(cmd); 1879 break; 1880 case TARGET_PROT_DOUT_STRIP: 1881 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_STRIP) 1882 break; 1883 1884 sectors = cmd->data_length >> ilog2(cmd->se_dev->dev_attrib.block_size); 1885 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba, 1886 sectors, 0, cmd->t_prot_sg, 0); 1887 if (unlikely(cmd->pi_err)) { 1888 spin_lock_irq(&cmd->t_state_lock); 1889 cmd->transport_state &= ~CMD_T_SENT; 1890 spin_unlock_irq(&cmd->t_state_lock); 1891 transport_generic_request_failure(cmd, cmd->pi_err); 1892 return -1; 1893 } 1894 break; 1895 default: 1896 break; 1897 } 1898 1899 return 0; 1900 } 1901 1902 static bool target_handle_task_attr(struct se_cmd *cmd) 1903 { 1904 struct se_device *dev = cmd->se_dev; 1905 1906 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 1907 return false; 1908 1909 cmd->se_cmd_flags |= SCF_TASK_ATTR_SET; 1910 1911 /* 1912 * Check for the existence of HEAD_OF_QUEUE, and if true return 1 1913 * to allow the passed struct se_cmd list of tasks to the front of the list. 1914 */ 1915 switch (cmd->sam_task_attr) { 1916 case TCM_HEAD_TAG: 1917 pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x\n", 1918 cmd->t_task_cdb[0]); 1919 return false; 1920 case TCM_ORDERED_TAG: 1921 atomic_inc_mb(&dev->dev_ordered_sync); 1922 1923 pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n", 1924 cmd->t_task_cdb[0]); 1925 1926 /* 1927 * Execute an ORDERED command if no other older commands 1928 * exist that need to be completed first. 1929 */ 1930 if (!atomic_read(&dev->simple_cmds)) 1931 return false; 1932 break; 1933 default: 1934 /* 1935 * For SIMPLE and UNTAGGED Task Attribute commands 1936 */ 1937 atomic_inc_mb(&dev->simple_cmds); 1938 break; 1939 } 1940 1941 if (atomic_read(&dev->dev_ordered_sync) == 0) 1942 return false; 1943 1944 spin_lock(&dev->delayed_cmd_lock); 1945 list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list); 1946 spin_unlock(&dev->delayed_cmd_lock); 1947 1948 pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn", 1949 cmd->t_task_cdb[0], cmd->sam_task_attr); 1950 return true; 1951 } 1952 1953 static int __transport_check_aborted_status(struct se_cmd *, int); 1954 1955 void target_execute_cmd(struct se_cmd *cmd) 1956 { 1957 /* 1958 * Determine if frontend context caller is requesting the stopping of 1959 * this command for frontend exceptions. 1960 * 1961 * If the received CDB has aleady been aborted stop processing it here. 1962 */ 1963 spin_lock_irq(&cmd->t_state_lock); 1964 if (__transport_check_aborted_status(cmd, 1)) { 1965 spin_unlock_irq(&cmd->t_state_lock); 1966 return; 1967 } 1968 if (cmd->transport_state & CMD_T_STOP) { 1969 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 1970 __func__, __LINE__, cmd->tag); 1971 1972 spin_unlock_irq(&cmd->t_state_lock); 1973 complete_all(&cmd->t_transport_stop_comp); 1974 return; 1975 } 1976 1977 cmd->t_state = TRANSPORT_PROCESSING; 1978 cmd->transport_state &= ~CMD_T_PRE_EXECUTE; 1979 cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT; 1980 spin_unlock_irq(&cmd->t_state_lock); 1981 1982 if (target_write_prot_action(cmd)) 1983 return; 1984 1985 if (target_handle_task_attr(cmd)) { 1986 spin_lock_irq(&cmd->t_state_lock); 1987 cmd->transport_state &= ~CMD_T_SENT; 1988 spin_unlock_irq(&cmd->t_state_lock); 1989 return; 1990 } 1991 1992 __target_execute_cmd(cmd, true); 1993 } 1994 EXPORT_SYMBOL(target_execute_cmd); 1995 1996 /* 1997 * Process all commands up to the last received ORDERED task attribute which 1998 * requires another blocking boundary 1999 */ 2000 static void target_restart_delayed_cmds(struct se_device *dev) 2001 { 2002 for (;;) { 2003 struct se_cmd *cmd; 2004 2005 spin_lock(&dev->delayed_cmd_lock); 2006 if (list_empty(&dev->delayed_cmd_list)) { 2007 spin_unlock(&dev->delayed_cmd_lock); 2008 break; 2009 } 2010 2011 cmd = list_entry(dev->delayed_cmd_list.next, 2012 struct se_cmd, se_delayed_node); 2013 list_del(&cmd->se_delayed_node); 2014 spin_unlock(&dev->delayed_cmd_lock); 2015 2016 cmd->transport_state |= CMD_T_SENT; 2017 2018 __target_execute_cmd(cmd, true); 2019 2020 if (cmd->sam_task_attr == TCM_ORDERED_TAG) 2021 break; 2022 } 2023 } 2024 2025 /* 2026 * Called from I/O completion to determine which dormant/delayed 2027 * and ordered cmds need to have their tasks added to the execution queue. 2028 */ 2029 static void transport_complete_task_attr(struct se_cmd *cmd) 2030 { 2031 struct se_device *dev = cmd->se_dev; 2032 2033 if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) 2034 return; 2035 2036 if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET)) 2037 goto restart; 2038 2039 if (cmd->sam_task_attr == TCM_SIMPLE_TAG) { 2040 atomic_dec_mb(&dev->simple_cmds); 2041 dev->dev_cur_ordered_id++; 2042 } else if (cmd->sam_task_attr == TCM_HEAD_TAG) { 2043 dev->dev_cur_ordered_id++; 2044 pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n", 2045 dev->dev_cur_ordered_id); 2046 } else if (cmd->sam_task_attr == TCM_ORDERED_TAG) { 2047 atomic_dec_mb(&dev->dev_ordered_sync); 2048 2049 dev->dev_cur_ordered_id++; 2050 pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n", 2051 dev->dev_cur_ordered_id); 2052 } 2053 cmd->se_cmd_flags &= ~SCF_TASK_ATTR_SET; 2054 2055 restart: 2056 target_restart_delayed_cmds(dev); 2057 } 2058 2059 static void transport_complete_qf(struct se_cmd *cmd) 2060 { 2061 int ret = 0; 2062 2063 transport_complete_task_attr(cmd); 2064 /* 2065 * If a fabric driver ->write_pending() or ->queue_data_in() callback 2066 * has returned neither -ENOMEM or -EAGAIN, assume it's fatal and 2067 * the same callbacks should not be retried. Return CHECK_CONDITION 2068 * if a scsi_status is not already set. 2069 * 2070 * If a fabric driver ->queue_status() has returned non zero, always 2071 * keep retrying no matter what.. 2072 */ 2073 if (cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) { 2074 if (cmd->scsi_status) 2075 goto queue_status; 2076 2077 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; 2078 cmd->scsi_status = SAM_STAT_CHECK_CONDITION; 2079 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 2080 translate_sense_reason(cmd, TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); 2081 goto queue_status; 2082 } 2083 2084 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) 2085 goto queue_status; 2086 2087 switch (cmd->data_direction) { 2088 case DMA_FROM_DEVICE: 2089 if (cmd->scsi_status) 2090 goto queue_status; 2091 2092 trace_target_cmd_complete(cmd); 2093 ret = cmd->se_tfo->queue_data_in(cmd); 2094 break; 2095 case DMA_TO_DEVICE: 2096 if (cmd->se_cmd_flags & SCF_BIDI) { 2097 ret = cmd->se_tfo->queue_data_in(cmd); 2098 break; 2099 } 2100 /* fall through */ 2101 case DMA_NONE: 2102 queue_status: 2103 trace_target_cmd_complete(cmd); 2104 ret = cmd->se_tfo->queue_status(cmd); 2105 break; 2106 default: 2107 break; 2108 } 2109 2110 if (ret < 0) { 2111 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 2112 return; 2113 } 2114 transport_lun_remove_cmd(cmd); 2115 transport_cmd_check_stop_to_fabric(cmd); 2116 } 2117 2118 static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev, 2119 int err, bool write_pending) 2120 { 2121 /* 2122 * -EAGAIN or -ENOMEM signals retry of ->write_pending() and/or 2123 * ->queue_data_in() callbacks from new process context. 2124 * 2125 * Otherwise for other errors, transport_complete_qf() will send 2126 * CHECK_CONDITION via ->queue_status() instead of attempting to 2127 * retry associated fabric driver data-transfer callbacks. 2128 */ 2129 if (err == -EAGAIN || err == -ENOMEM) { 2130 cmd->t_state = (write_pending) ? TRANSPORT_COMPLETE_QF_WP : 2131 TRANSPORT_COMPLETE_QF_OK; 2132 } else { 2133 pr_warn_ratelimited("Got unknown fabric queue status: %d\n", err); 2134 cmd->t_state = TRANSPORT_COMPLETE_QF_ERR; 2135 } 2136 2137 spin_lock_irq(&dev->qf_cmd_lock); 2138 list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list); 2139 atomic_inc_mb(&dev->dev_qf_count); 2140 spin_unlock_irq(&cmd->se_dev->qf_cmd_lock); 2141 2142 schedule_work(&cmd->se_dev->qf_work_queue); 2143 } 2144 2145 static bool target_read_prot_action(struct se_cmd *cmd) 2146 { 2147 switch (cmd->prot_op) { 2148 case TARGET_PROT_DIN_STRIP: 2149 if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_STRIP)) { 2150 u32 sectors = cmd->data_length >> 2151 ilog2(cmd->se_dev->dev_attrib.block_size); 2152 2153 cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba, 2154 sectors, 0, cmd->t_prot_sg, 2155 0); 2156 if (cmd->pi_err) 2157 return true; 2158 } 2159 break; 2160 case TARGET_PROT_DIN_INSERT: 2161 if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_INSERT) 2162 break; 2163 2164 sbc_dif_generate(cmd); 2165 break; 2166 default: 2167 break; 2168 } 2169 2170 return false; 2171 } 2172 2173 static void target_complete_ok_work(struct work_struct *work) 2174 { 2175 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 2176 int ret; 2177 2178 /* 2179 * Check if we need to move delayed/dormant tasks from cmds on the 2180 * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task 2181 * Attribute. 2182 */ 2183 transport_complete_task_attr(cmd); 2184 2185 /* 2186 * Check to schedule QUEUE_FULL work, or execute an existing 2187 * cmd->transport_qf_callback() 2188 */ 2189 if (atomic_read(&cmd->se_dev->dev_qf_count) != 0) 2190 schedule_work(&cmd->se_dev->qf_work_queue); 2191 2192 /* 2193 * Check if we need to send a sense buffer from 2194 * the struct se_cmd in question. 2195 */ 2196 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) { 2197 WARN_ON(!cmd->scsi_status); 2198 ret = transport_send_check_condition_and_sense( 2199 cmd, 0, 1); 2200 if (ret) 2201 goto queue_full; 2202 2203 transport_lun_remove_cmd(cmd); 2204 transport_cmd_check_stop_to_fabric(cmd); 2205 return; 2206 } 2207 /* 2208 * Check for a callback, used by amongst other things 2209 * XDWRITE_READ_10 and COMPARE_AND_WRITE emulation. 2210 */ 2211 if (cmd->transport_complete_callback) { 2212 sense_reason_t rc; 2213 bool caw = (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE); 2214 bool zero_dl = !(cmd->data_length); 2215 int post_ret = 0; 2216 2217 rc = cmd->transport_complete_callback(cmd, true, &post_ret); 2218 if (!rc && !post_ret) { 2219 if (caw && zero_dl) 2220 goto queue_rsp; 2221 2222 return; 2223 } else if (rc) { 2224 ret = transport_send_check_condition_and_sense(cmd, 2225 rc, 0); 2226 if (ret) 2227 goto queue_full; 2228 2229 transport_lun_remove_cmd(cmd); 2230 transport_cmd_check_stop_to_fabric(cmd); 2231 return; 2232 } 2233 } 2234 2235 queue_rsp: 2236 switch (cmd->data_direction) { 2237 case DMA_FROM_DEVICE: 2238 if (cmd->scsi_status) 2239 goto queue_status; 2240 2241 atomic_long_add(cmd->data_length, 2242 &cmd->se_lun->lun_stats.tx_data_octets); 2243 /* 2244 * Perform READ_STRIP of PI using software emulation when 2245 * backend had PI enabled, if the transport will not be 2246 * performing hardware READ_STRIP offload. 2247 */ 2248 if (target_read_prot_action(cmd)) { 2249 ret = transport_send_check_condition_and_sense(cmd, 2250 cmd->pi_err, 0); 2251 if (ret) 2252 goto queue_full; 2253 2254 transport_lun_remove_cmd(cmd); 2255 transport_cmd_check_stop_to_fabric(cmd); 2256 return; 2257 } 2258 2259 trace_target_cmd_complete(cmd); 2260 ret = cmd->se_tfo->queue_data_in(cmd); 2261 if (ret) 2262 goto queue_full; 2263 break; 2264 case DMA_TO_DEVICE: 2265 atomic_long_add(cmd->data_length, 2266 &cmd->se_lun->lun_stats.rx_data_octets); 2267 /* 2268 * Check if we need to send READ payload for BIDI-COMMAND 2269 */ 2270 if (cmd->se_cmd_flags & SCF_BIDI) { 2271 atomic_long_add(cmd->data_length, 2272 &cmd->se_lun->lun_stats.tx_data_octets); 2273 ret = cmd->se_tfo->queue_data_in(cmd); 2274 if (ret) 2275 goto queue_full; 2276 break; 2277 } 2278 /* fall through */ 2279 case DMA_NONE: 2280 queue_status: 2281 trace_target_cmd_complete(cmd); 2282 ret = cmd->se_tfo->queue_status(cmd); 2283 if (ret) 2284 goto queue_full; 2285 break; 2286 default: 2287 break; 2288 } 2289 2290 transport_lun_remove_cmd(cmd); 2291 transport_cmd_check_stop_to_fabric(cmd); 2292 return; 2293 2294 queue_full: 2295 pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p," 2296 " data_direction: %d\n", cmd, cmd->data_direction); 2297 2298 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 2299 } 2300 2301 void target_free_sgl(struct scatterlist *sgl, int nents) 2302 { 2303 sgl_free_n_order(sgl, nents, 0); 2304 } 2305 EXPORT_SYMBOL(target_free_sgl); 2306 2307 static inline void transport_reset_sgl_orig(struct se_cmd *cmd) 2308 { 2309 /* 2310 * Check for saved t_data_sg that may be used for COMPARE_AND_WRITE 2311 * emulation, and free + reset pointers if necessary.. 2312 */ 2313 if (!cmd->t_data_sg_orig) 2314 return; 2315 2316 kfree(cmd->t_data_sg); 2317 cmd->t_data_sg = cmd->t_data_sg_orig; 2318 cmd->t_data_sg_orig = NULL; 2319 cmd->t_data_nents = cmd->t_data_nents_orig; 2320 cmd->t_data_nents_orig = 0; 2321 } 2322 2323 static inline void transport_free_pages(struct se_cmd *cmd) 2324 { 2325 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) { 2326 target_free_sgl(cmd->t_prot_sg, cmd->t_prot_nents); 2327 cmd->t_prot_sg = NULL; 2328 cmd->t_prot_nents = 0; 2329 } 2330 2331 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 2332 /* 2333 * Release special case READ buffer payload required for 2334 * SG_TO_MEM_NOALLOC to function with COMPARE_AND_WRITE 2335 */ 2336 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) { 2337 target_free_sgl(cmd->t_bidi_data_sg, 2338 cmd->t_bidi_data_nents); 2339 cmd->t_bidi_data_sg = NULL; 2340 cmd->t_bidi_data_nents = 0; 2341 } 2342 transport_reset_sgl_orig(cmd); 2343 return; 2344 } 2345 transport_reset_sgl_orig(cmd); 2346 2347 target_free_sgl(cmd->t_data_sg, cmd->t_data_nents); 2348 cmd->t_data_sg = NULL; 2349 cmd->t_data_nents = 0; 2350 2351 target_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents); 2352 cmd->t_bidi_data_sg = NULL; 2353 cmd->t_bidi_data_nents = 0; 2354 } 2355 2356 void *transport_kmap_data_sg(struct se_cmd *cmd) 2357 { 2358 struct scatterlist *sg = cmd->t_data_sg; 2359 struct page **pages; 2360 int i; 2361 2362 /* 2363 * We need to take into account a possible offset here for fabrics like 2364 * tcm_loop who may be using a contig buffer from the SCSI midlayer for 2365 * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd() 2366 */ 2367 if (!cmd->t_data_nents) 2368 return NULL; 2369 2370 BUG_ON(!sg); 2371 if (cmd->t_data_nents == 1) 2372 return kmap(sg_page(sg)) + sg->offset; 2373 2374 /* >1 page. use vmap */ 2375 pages = kmalloc_array(cmd->t_data_nents, sizeof(*pages), GFP_KERNEL); 2376 if (!pages) 2377 return NULL; 2378 2379 /* convert sg[] to pages[] */ 2380 for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) { 2381 pages[i] = sg_page(sg); 2382 } 2383 2384 cmd->t_data_vmap = vmap(pages, cmd->t_data_nents, VM_MAP, PAGE_KERNEL); 2385 kfree(pages); 2386 if (!cmd->t_data_vmap) 2387 return NULL; 2388 2389 return cmd->t_data_vmap + cmd->t_data_sg[0].offset; 2390 } 2391 EXPORT_SYMBOL(transport_kmap_data_sg); 2392 2393 void transport_kunmap_data_sg(struct se_cmd *cmd) 2394 { 2395 if (!cmd->t_data_nents) { 2396 return; 2397 } else if (cmd->t_data_nents == 1) { 2398 kunmap(sg_page(cmd->t_data_sg)); 2399 return; 2400 } 2401 2402 vunmap(cmd->t_data_vmap); 2403 cmd->t_data_vmap = NULL; 2404 } 2405 EXPORT_SYMBOL(transport_kunmap_data_sg); 2406 2407 int 2408 target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length, 2409 bool zero_page, bool chainable) 2410 { 2411 gfp_t gfp = GFP_KERNEL | (zero_page ? __GFP_ZERO : 0); 2412 2413 *sgl = sgl_alloc_order(length, 0, chainable, gfp, nents); 2414 return *sgl ? 0 : -ENOMEM; 2415 } 2416 EXPORT_SYMBOL(target_alloc_sgl); 2417 2418 /* 2419 * Allocate any required resources to execute the command. For writes we 2420 * might not have the payload yet, so notify the fabric via a call to 2421 * ->write_pending instead. Otherwise place it on the execution queue. 2422 */ 2423 sense_reason_t 2424 transport_generic_new_cmd(struct se_cmd *cmd) 2425 { 2426 unsigned long flags; 2427 int ret = 0; 2428 bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB); 2429 2430 if (cmd->prot_op != TARGET_PROT_NORMAL && 2431 !(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) { 2432 ret = target_alloc_sgl(&cmd->t_prot_sg, &cmd->t_prot_nents, 2433 cmd->prot_length, true, false); 2434 if (ret < 0) 2435 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2436 } 2437 2438 /* 2439 * Determine is the TCM fabric module has already allocated physical 2440 * memory, and is directly calling transport_generic_map_mem_to_cmd() 2441 * beforehand. 2442 */ 2443 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) && 2444 cmd->data_length) { 2445 2446 if ((cmd->se_cmd_flags & SCF_BIDI) || 2447 (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)) { 2448 u32 bidi_length; 2449 2450 if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) 2451 bidi_length = cmd->t_task_nolb * 2452 cmd->se_dev->dev_attrib.block_size; 2453 else 2454 bidi_length = cmd->data_length; 2455 2456 ret = target_alloc_sgl(&cmd->t_bidi_data_sg, 2457 &cmd->t_bidi_data_nents, 2458 bidi_length, zero_flag, false); 2459 if (ret < 0) 2460 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2461 } 2462 2463 ret = target_alloc_sgl(&cmd->t_data_sg, &cmd->t_data_nents, 2464 cmd->data_length, zero_flag, false); 2465 if (ret < 0) 2466 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2467 } else if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) && 2468 cmd->data_length) { 2469 /* 2470 * Special case for COMPARE_AND_WRITE with fabrics 2471 * using SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC. 2472 */ 2473 u32 caw_length = cmd->t_task_nolb * 2474 cmd->se_dev->dev_attrib.block_size; 2475 2476 ret = target_alloc_sgl(&cmd->t_bidi_data_sg, 2477 &cmd->t_bidi_data_nents, 2478 caw_length, zero_flag, false); 2479 if (ret < 0) 2480 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 2481 } 2482 /* 2483 * If this command is not a write we can execute it right here, 2484 * for write buffers we need to notify the fabric driver first 2485 * and let it call back once the write buffers are ready. 2486 */ 2487 target_add_to_state_list(cmd); 2488 if (cmd->data_direction != DMA_TO_DEVICE || cmd->data_length == 0) { 2489 target_execute_cmd(cmd); 2490 return 0; 2491 } 2492 2493 spin_lock_irqsave(&cmd->t_state_lock, flags); 2494 cmd->t_state = TRANSPORT_WRITE_PENDING; 2495 /* 2496 * Determine if frontend context caller is requesting the stopping of 2497 * this command for frontend exceptions. 2498 */ 2499 if (cmd->transport_state & CMD_T_STOP) { 2500 pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", 2501 __func__, __LINE__, cmd->tag); 2502 2503 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2504 2505 complete_all(&cmd->t_transport_stop_comp); 2506 return 0; 2507 } 2508 cmd->transport_state &= ~CMD_T_ACTIVE; 2509 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2510 2511 ret = cmd->se_tfo->write_pending(cmd); 2512 if (ret) 2513 goto queue_full; 2514 2515 return 0; 2516 2517 queue_full: 2518 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd); 2519 transport_handle_queue_full(cmd, cmd->se_dev, ret, true); 2520 return 0; 2521 } 2522 EXPORT_SYMBOL(transport_generic_new_cmd); 2523 2524 static void transport_write_pending_qf(struct se_cmd *cmd) 2525 { 2526 unsigned long flags; 2527 int ret; 2528 bool stop; 2529 2530 spin_lock_irqsave(&cmd->t_state_lock, flags); 2531 stop = (cmd->transport_state & (CMD_T_STOP | CMD_T_ABORTED)); 2532 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2533 2534 if (stop) { 2535 pr_debug("%s:%d CMD_T_STOP|CMD_T_ABORTED for ITT: 0x%08llx\n", 2536 __func__, __LINE__, cmd->tag); 2537 complete_all(&cmd->t_transport_stop_comp); 2538 return; 2539 } 2540 2541 ret = cmd->se_tfo->write_pending(cmd); 2542 if (ret) { 2543 pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", 2544 cmd); 2545 transport_handle_queue_full(cmd, cmd->se_dev, ret, true); 2546 } 2547 } 2548 2549 static bool 2550 __transport_wait_for_tasks(struct se_cmd *, bool, bool *, bool *, 2551 unsigned long *flags); 2552 2553 static void target_wait_free_cmd(struct se_cmd *cmd, bool *aborted, bool *tas) 2554 { 2555 unsigned long flags; 2556 2557 spin_lock_irqsave(&cmd->t_state_lock, flags); 2558 __transport_wait_for_tasks(cmd, true, aborted, tas, &flags); 2559 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2560 } 2561 2562 int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks) 2563 { 2564 int ret = 0; 2565 bool aborted = false, tas = false; 2566 2567 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) { 2568 if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2569 target_wait_free_cmd(cmd, &aborted, &tas); 2570 2571 if (!aborted || tas) 2572 ret = target_put_sess_cmd(cmd); 2573 } else { 2574 if (wait_for_tasks) 2575 target_wait_free_cmd(cmd, &aborted, &tas); 2576 /* 2577 * Handle WRITE failure case where transport_generic_new_cmd() 2578 * has already added se_cmd to state_list, but fabric has 2579 * failed command before I/O submission. 2580 */ 2581 if (cmd->state_active) 2582 target_remove_from_state_list(cmd); 2583 2584 if (cmd->se_lun) 2585 transport_lun_remove_cmd(cmd); 2586 2587 if (!aborted || tas) 2588 ret = target_put_sess_cmd(cmd); 2589 } 2590 /* 2591 * If the task has been internally aborted due to TMR ABORT_TASK 2592 * or LUN_RESET, target_core_tmr.c is responsible for performing 2593 * the remaining calls to target_put_sess_cmd(), and not the 2594 * callers of this function. 2595 */ 2596 if (aborted) { 2597 pr_debug("Detected CMD_T_ABORTED for ITT: %llu\n", cmd->tag); 2598 wait_for_completion(&cmd->cmd_wait_comp); 2599 cmd->se_tfo->release_cmd(cmd); 2600 ret = 1; 2601 } 2602 return ret; 2603 } 2604 EXPORT_SYMBOL(transport_generic_free_cmd); 2605 2606 /* target_get_sess_cmd - Add command to active ->sess_cmd_list 2607 * @se_cmd: command descriptor to add 2608 * @ack_kref: Signal that fabric will perform an ack target_put_sess_cmd() 2609 */ 2610 int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref) 2611 { 2612 struct se_session *se_sess = se_cmd->se_sess; 2613 unsigned long flags; 2614 int ret = 0; 2615 2616 /* 2617 * Add a second kref if the fabric caller is expecting to handle 2618 * fabric acknowledgement that requires two target_put_sess_cmd() 2619 * invocations before se_cmd descriptor release. 2620 */ 2621 if (ack_kref) { 2622 if (!kref_get_unless_zero(&se_cmd->cmd_kref)) 2623 return -EINVAL; 2624 2625 se_cmd->se_cmd_flags |= SCF_ACK_KREF; 2626 } 2627 2628 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2629 if (se_sess->sess_tearing_down) { 2630 ret = -ESHUTDOWN; 2631 goto out; 2632 } 2633 se_cmd->transport_state |= CMD_T_PRE_EXECUTE; 2634 list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list); 2635 out: 2636 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2637 2638 if (ret && ack_kref) 2639 target_put_sess_cmd(se_cmd); 2640 2641 return ret; 2642 } 2643 EXPORT_SYMBOL(target_get_sess_cmd); 2644 2645 static void target_free_cmd_mem(struct se_cmd *cmd) 2646 { 2647 transport_free_pages(cmd); 2648 2649 if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) 2650 core_tmr_release_req(cmd->se_tmr_req); 2651 if (cmd->t_task_cdb != cmd->__t_task_cdb) 2652 kfree(cmd->t_task_cdb); 2653 } 2654 2655 static void target_release_cmd_kref(struct kref *kref) 2656 { 2657 struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref); 2658 struct se_session *se_sess = se_cmd->se_sess; 2659 unsigned long flags; 2660 bool fabric_stop; 2661 2662 if (se_sess) { 2663 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2664 2665 spin_lock(&se_cmd->t_state_lock); 2666 fabric_stop = (se_cmd->transport_state & CMD_T_FABRIC_STOP) && 2667 (se_cmd->transport_state & CMD_T_ABORTED); 2668 spin_unlock(&se_cmd->t_state_lock); 2669 2670 if (se_cmd->cmd_wait_set || fabric_stop) { 2671 list_del_init(&se_cmd->se_cmd_list); 2672 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2673 target_free_cmd_mem(se_cmd); 2674 complete(&se_cmd->cmd_wait_comp); 2675 return; 2676 } 2677 list_del_init(&se_cmd->se_cmd_list); 2678 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2679 } 2680 2681 target_free_cmd_mem(se_cmd); 2682 se_cmd->se_tfo->release_cmd(se_cmd); 2683 } 2684 2685 /** 2686 * target_put_sess_cmd - decrease the command reference count 2687 * @se_cmd: command to drop a reference from 2688 * 2689 * Returns 1 if and only if this target_put_sess_cmd() call caused the 2690 * refcount to drop to zero. Returns zero otherwise. 2691 */ 2692 int target_put_sess_cmd(struct se_cmd *se_cmd) 2693 { 2694 return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref); 2695 } 2696 EXPORT_SYMBOL(target_put_sess_cmd); 2697 2698 static const char *data_dir_name(enum dma_data_direction d) 2699 { 2700 switch (d) { 2701 case DMA_BIDIRECTIONAL: return "BIDI"; 2702 case DMA_TO_DEVICE: return "WRITE"; 2703 case DMA_FROM_DEVICE: return "READ"; 2704 case DMA_NONE: return "NONE"; 2705 } 2706 2707 return "(?)"; 2708 } 2709 2710 static const char *cmd_state_name(enum transport_state_table t) 2711 { 2712 switch (t) { 2713 case TRANSPORT_NO_STATE: return "NO_STATE"; 2714 case TRANSPORT_NEW_CMD: return "NEW_CMD"; 2715 case TRANSPORT_WRITE_PENDING: return "WRITE_PENDING"; 2716 case TRANSPORT_PROCESSING: return "PROCESSING"; 2717 case TRANSPORT_COMPLETE: return "COMPLETE"; 2718 case TRANSPORT_ISTATE_PROCESSING: 2719 return "ISTATE_PROCESSING"; 2720 case TRANSPORT_COMPLETE_QF_WP: return "COMPLETE_QF_WP"; 2721 case TRANSPORT_COMPLETE_QF_OK: return "COMPLETE_QF_OK"; 2722 case TRANSPORT_COMPLETE_QF_ERR: return "COMPLETE_QF_ERR"; 2723 } 2724 2725 return "(?)"; 2726 } 2727 2728 static void target_append_str(char **str, const char *txt) 2729 { 2730 char *prev = *str; 2731 2732 *str = *str ? kasprintf(GFP_ATOMIC, "%s,%s", *str, txt) : 2733 kstrdup(txt, GFP_ATOMIC); 2734 kfree(prev); 2735 } 2736 2737 /* 2738 * Convert a transport state bitmask into a string. The caller is 2739 * responsible for freeing the returned pointer. 2740 */ 2741 static char *target_ts_to_str(u32 ts) 2742 { 2743 char *str = NULL; 2744 2745 if (ts & CMD_T_ABORTED) 2746 target_append_str(&str, "aborted"); 2747 if (ts & CMD_T_ACTIVE) 2748 target_append_str(&str, "active"); 2749 if (ts & CMD_T_COMPLETE) 2750 target_append_str(&str, "complete"); 2751 if (ts & CMD_T_SENT) 2752 target_append_str(&str, "sent"); 2753 if (ts & CMD_T_STOP) 2754 target_append_str(&str, "stop"); 2755 if (ts & CMD_T_FABRIC_STOP) 2756 target_append_str(&str, "fabric_stop"); 2757 2758 return str; 2759 } 2760 2761 static const char *target_tmf_name(enum tcm_tmreq_table tmf) 2762 { 2763 switch (tmf) { 2764 case TMR_ABORT_TASK: return "ABORT_TASK"; 2765 case TMR_ABORT_TASK_SET: return "ABORT_TASK_SET"; 2766 case TMR_CLEAR_ACA: return "CLEAR_ACA"; 2767 case TMR_CLEAR_TASK_SET: return "CLEAR_TASK_SET"; 2768 case TMR_LUN_RESET: return "LUN_RESET"; 2769 case TMR_TARGET_WARM_RESET: return "TARGET_WARM_RESET"; 2770 case TMR_TARGET_COLD_RESET: return "TARGET_COLD_RESET"; 2771 case TMR_UNKNOWN: break; 2772 } 2773 return "(?)"; 2774 } 2775 2776 void target_show_cmd(const char *pfx, struct se_cmd *cmd) 2777 { 2778 char *ts_str = target_ts_to_str(cmd->transport_state); 2779 const u8 *cdb = cmd->t_task_cdb; 2780 struct se_tmr_req *tmf = cmd->se_tmr_req; 2781 2782 if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) { 2783 pr_debug("%scmd %#02x:%#02x with tag %#llx dir %s i_state %d t_state %s len %d refcnt %d transport_state %s\n", 2784 pfx, cdb[0], cdb[1], cmd->tag, 2785 data_dir_name(cmd->data_direction), 2786 cmd->se_tfo->get_cmd_state(cmd), 2787 cmd_state_name(cmd->t_state), cmd->data_length, 2788 kref_read(&cmd->cmd_kref), ts_str); 2789 } else { 2790 pr_debug("%stmf %s with tag %#llx ref_task_tag %#llx i_state %d t_state %s refcnt %d transport_state %s\n", 2791 pfx, target_tmf_name(tmf->function), cmd->tag, 2792 tmf->ref_task_tag, cmd->se_tfo->get_cmd_state(cmd), 2793 cmd_state_name(cmd->t_state), 2794 kref_read(&cmd->cmd_kref), ts_str); 2795 } 2796 kfree(ts_str); 2797 } 2798 EXPORT_SYMBOL(target_show_cmd); 2799 2800 /* target_sess_cmd_list_set_waiting - Flag all commands in 2801 * sess_cmd_list to complete cmd_wait_comp. Set 2802 * sess_tearing_down so no more commands are queued. 2803 * @se_sess: session to flag 2804 */ 2805 void target_sess_cmd_list_set_waiting(struct se_session *se_sess) 2806 { 2807 struct se_cmd *se_cmd, *tmp_cmd; 2808 unsigned long flags; 2809 int rc; 2810 2811 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2812 if (se_sess->sess_tearing_down) { 2813 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2814 return; 2815 } 2816 se_sess->sess_tearing_down = 1; 2817 list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list); 2818 2819 list_for_each_entry_safe(se_cmd, tmp_cmd, 2820 &se_sess->sess_wait_list, se_cmd_list) { 2821 rc = kref_get_unless_zero(&se_cmd->cmd_kref); 2822 if (rc) { 2823 se_cmd->cmd_wait_set = 1; 2824 spin_lock(&se_cmd->t_state_lock); 2825 se_cmd->transport_state |= CMD_T_FABRIC_STOP; 2826 spin_unlock(&se_cmd->t_state_lock); 2827 } else 2828 list_del_init(&se_cmd->se_cmd_list); 2829 } 2830 2831 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2832 } 2833 EXPORT_SYMBOL(target_sess_cmd_list_set_waiting); 2834 2835 /* target_wait_for_sess_cmds - Wait for outstanding descriptors 2836 * @se_sess: session to wait for active I/O 2837 */ 2838 void target_wait_for_sess_cmds(struct se_session *se_sess) 2839 { 2840 struct se_cmd *se_cmd, *tmp_cmd; 2841 unsigned long flags; 2842 bool tas; 2843 2844 list_for_each_entry_safe(se_cmd, tmp_cmd, 2845 &se_sess->sess_wait_list, se_cmd_list) { 2846 pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:" 2847 " %d\n", se_cmd, se_cmd->t_state, 2848 se_cmd->se_tfo->get_cmd_state(se_cmd)); 2849 2850 spin_lock_irqsave(&se_cmd->t_state_lock, flags); 2851 tas = (se_cmd->transport_state & CMD_T_TAS); 2852 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); 2853 2854 if (!target_put_sess_cmd(se_cmd)) { 2855 if (tas) 2856 target_put_sess_cmd(se_cmd); 2857 } 2858 2859 wait_for_completion(&se_cmd->cmd_wait_comp); 2860 pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d" 2861 " fabric state: %d\n", se_cmd, se_cmd->t_state, 2862 se_cmd->se_tfo->get_cmd_state(se_cmd)); 2863 2864 se_cmd->se_tfo->release_cmd(se_cmd); 2865 } 2866 2867 spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); 2868 WARN_ON(!list_empty(&se_sess->sess_cmd_list)); 2869 spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); 2870 2871 } 2872 EXPORT_SYMBOL(target_wait_for_sess_cmds); 2873 2874 static void target_lun_confirm(struct percpu_ref *ref) 2875 { 2876 struct se_lun *lun = container_of(ref, struct se_lun, lun_ref); 2877 2878 complete(&lun->lun_ref_comp); 2879 } 2880 2881 void transport_clear_lun_ref(struct se_lun *lun) 2882 { 2883 /* 2884 * Mark the percpu-ref as DEAD, switch to atomic_t mode, drop 2885 * the initial reference and schedule confirm kill to be 2886 * executed after one full RCU grace period has completed. 2887 */ 2888 percpu_ref_kill_and_confirm(&lun->lun_ref, target_lun_confirm); 2889 /* 2890 * The first completion waits for percpu_ref_switch_to_atomic_rcu() 2891 * to call target_lun_confirm after lun->lun_ref has been marked 2892 * as __PERCPU_REF_DEAD on all CPUs, and switches to atomic_t 2893 * mode so that percpu_ref_tryget_live() lookup of lun->lun_ref 2894 * fails for all new incoming I/O. 2895 */ 2896 wait_for_completion(&lun->lun_ref_comp); 2897 /* 2898 * The second completion waits for percpu_ref_put_many() to 2899 * invoke ->release() after lun->lun_ref has switched to 2900 * atomic_t mode, and lun->lun_ref.count has reached zero. 2901 * 2902 * At this point all target-core lun->lun_ref references have 2903 * been dropped via transport_lun_remove_cmd(), and it's safe 2904 * to proceed with the remaining LUN shutdown. 2905 */ 2906 wait_for_completion(&lun->lun_shutdown_comp); 2907 } 2908 2909 static bool 2910 __transport_wait_for_tasks(struct se_cmd *cmd, bool fabric_stop, 2911 bool *aborted, bool *tas, unsigned long *flags) 2912 __releases(&cmd->t_state_lock) 2913 __acquires(&cmd->t_state_lock) 2914 { 2915 2916 assert_spin_locked(&cmd->t_state_lock); 2917 WARN_ON_ONCE(!irqs_disabled()); 2918 2919 if (fabric_stop) 2920 cmd->transport_state |= CMD_T_FABRIC_STOP; 2921 2922 if (cmd->transport_state & CMD_T_ABORTED) 2923 *aborted = true; 2924 2925 if (cmd->transport_state & CMD_T_TAS) 2926 *tas = true; 2927 2928 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) && 2929 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2930 return false; 2931 2932 if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) && 2933 !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) 2934 return false; 2935 2936 if (!(cmd->transport_state & CMD_T_ACTIVE)) 2937 return false; 2938 2939 if (fabric_stop && *aborted) 2940 return false; 2941 2942 cmd->transport_state |= CMD_T_STOP; 2943 2944 target_show_cmd("wait_for_tasks: Stopping ", cmd); 2945 2946 spin_unlock_irqrestore(&cmd->t_state_lock, *flags); 2947 2948 while (!wait_for_completion_timeout(&cmd->t_transport_stop_comp, 2949 180 * HZ)) 2950 target_show_cmd("wait for tasks: ", cmd); 2951 2952 spin_lock_irqsave(&cmd->t_state_lock, *flags); 2953 cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP); 2954 2955 pr_debug("wait_for_tasks: Stopped wait_for_completion(&cmd->" 2956 "t_transport_stop_comp) for ITT: 0x%08llx\n", cmd->tag); 2957 2958 return true; 2959 } 2960 2961 /** 2962 * transport_wait_for_tasks - set CMD_T_STOP and wait for t_transport_stop_comp 2963 * @cmd: command to wait on 2964 */ 2965 bool transport_wait_for_tasks(struct se_cmd *cmd) 2966 { 2967 unsigned long flags; 2968 bool ret, aborted = false, tas = false; 2969 2970 spin_lock_irqsave(&cmd->t_state_lock, flags); 2971 ret = __transport_wait_for_tasks(cmd, false, &aborted, &tas, &flags); 2972 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 2973 2974 return ret; 2975 } 2976 EXPORT_SYMBOL(transport_wait_for_tasks); 2977 2978 struct sense_info { 2979 u8 key; 2980 u8 asc; 2981 u8 ascq; 2982 bool add_sector_info; 2983 }; 2984 2985 static const struct sense_info sense_info_table[] = { 2986 [TCM_NO_SENSE] = { 2987 .key = NOT_READY 2988 }, 2989 [TCM_NON_EXISTENT_LUN] = { 2990 .key = ILLEGAL_REQUEST, 2991 .asc = 0x25 /* LOGICAL UNIT NOT SUPPORTED */ 2992 }, 2993 [TCM_UNSUPPORTED_SCSI_OPCODE] = { 2994 .key = ILLEGAL_REQUEST, 2995 .asc = 0x20, /* INVALID COMMAND OPERATION CODE */ 2996 }, 2997 [TCM_SECTOR_COUNT_TOO_MANY] = { 2998 .key = ILLEGAL_REQUEST, 2999 .asc = 0x20, /* INVALID COMMAND OPERATION CODE */ 3000 }, 3001 [TCM_UNKNOWN_MODE_PAGE] = { 3002 .key = ILLEGAL_REQUEST, 3003 .asc = 0x24, /* INVALID FIELD IN CDB */ 3004 }, 3005 [TCM_CHECK_CONDITION_ABORT_CMD] = { 3006 .key = ABORTED_COMMAND, 3007 .asc = 0x29, /* BUS DEVICE RESET FUNCTION OCCURRED */ 3008 .ascq = 0x03, 3009 }, 3010 [TCM_INCORRECT_AMOUNT_OF_DATA] = { 3011 .key = ABORTED_COMMAND, 3012 .asc = 0x0c, /* WRITE ERROR */ 3013 .ascq = 0x0d, /* NOT ENOUGH UNSOLICITED DATA */ 3014 }, 3015 [TCM_INVALID_CDB_FIELD] = { 3016 .key = ILLEGAL_REQUEST, 3017 .asc = 0x24, /* INVALID FIELD IN CDB */ 3018 }, 3019 [TCM_INVALID_PARAMETER_LIST] = { 3020 .key = ILLEGAL_REQUEST, 3021 .asc = 0x26, /* INVALID FIELD IN PARAMETER LIST */ 3022 }, 3023 [TCM_TOO_MANY_TARGET_DESCS] = { 3024 .key = ILLEGAL_REQUEST, 3025 .asc = 0x26, 3026 .ascq = 0x06, /* TOO MANY TARGET DESCRIPTORS */ 3027 }, 3028 [TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE] = { 3029 .key = ILLEGAL_REQUEST, 3030 .asc = 0x26, 3031 .ascq = 0x07, /* UNSUPPORTED TARGET DESCRIPTOR TYPE CODE */ 3032 }, 3033 [TCM_TOO_MANY_SEGMENT_DESCS] = { 3034 .key = ILLEGAL_REQUEST, 3035 .asc = 0x26, 3036 .ascq = 0x08, /* TOO MANY SEGMENT DESCRIPTORS */ 3037 }, 3038 [TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE] = { 3039 .key = ILLEGAL_REQUEST, 3040 .asc = 0x26, 3041 .ascq = 0x09, /* UNSUPPORTED SEGMENT DESCRIPTOR TYPE CODE */ 3042 }, 3043 [TCM_PARAMETER_LIST_LENGTH_ERROR] = { 3044 .key = ILLEGAL_REQUEST, 3045 .asc = 0x1a, /* PARAMETER LIST LENGTH ERROR */ 3046 }, 3047 [TCM_UNEXPECTED_UNSOLICITED_DATA] = { 3048 .key = ILLEGAL_REQUEST, 3049 .asc = 0x0c, /* WRITE ERROR */ 3050 .ascq = 0x0c, /* UNEXPECTED_UNSOLICITED_DATA */ 3051 }, 3052 [TCM_SERVICE_CRC_ERROR] = { 3053 .key = ABORTED_COMMAND, 3054 .asc = 0x47, /* PROTOCOL SERVICE CRC ERROR */ 3055 .ascq = 0x05, /* N/A */ 3056 }, 3057 [TCM_SNACK_REJECTED] = { 3058 .key = ABORTED_COMMAND, 3059 .asc = 0x11, /* READ ERROR */ 3060 .ascq = 0x13, /* FAILED RETRANSMISSION REQUEST */ 3061 }, 3062 [TCM_WRITE_PROTECTED] = { 3063 .key = DATA_PROTECT, 3064 .asc = 0x27, /* WRITE PROTECTED */ 3065 }, 3066 [TCM_ADDRESS_OUT_OF_RANGE] = { 3067 .key = ILLEGAL_REQUEST, 3068 .asc = 0x21, /* LOGICAL BLOCK ADDRESS OUT OF RANGE */ 3069 }, 3070 [TCM_CHECK_CONDITION_UNIT_ATTENTION] = { 3071 .key = UNIT_ATTENTION, 3072 }, 3073 [TCM_CHECK_CONDITION_NOT_READY] = { 3074 .key = NOT_READY, 3075 }, 3076 [TCM_MISCOMPARE_VERIFY] = { 3077 .key = MISCOMPARE, 3078 .asc = 0x1d, /* MISCOMPARE DURING VERIFY OPERATION */ 3079 .ascq = 0x00, 3080 }, 3081 [TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED] = { 3082 .key = ABORTED_COMMAND, 3083 .asc = 0x10, 3084 .ascq = 0x01, /* LOGICAL BLOCK GUARD CHECK FAILED */ 3085 .add_sector_info = true, 3086 }, 3087 [TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED] = { 3088 .key = ABORTED_COMMAND, 3089 .asc = 0x10, 3090 .ascq = 0x02, /* LOGICAL BLOCK APPLICATION TAG CHECK FAILED */ 3091 .add_sector_info = true, 3092 }, 3093 [TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED] = { 3094 .key = ABORTED_COMMAND, 3095 .asc = 0x10, 3096 .ascq = 0x03, /* LOGICAL BLOCK REFERENCE TAG CHECK FAILED */ 3097 .add_sector_info = true, 3098 }, 3099 [TCM_COPY_TARGET_DEVICE_NOT_REACHABLE] = { 3100 .key = COPY_ABORTED, 3101 .asc = 0x0d, 3102 .ascq = 0x02, /* COPY TARGET DEVICE NOT REACHABLE */ 3103 3104 }, 3105 [TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE] = { 3106 /* 3107 * Returning ILLEGAL REQUEST would cause immediate IO errors on 3108 * Solaris initiators. Returning NOT READY instead means the 3109 * operations will be retried a finite number of times and we 3110 * can survive intermittent errors. 3111 */ 3112 .key = NOT_READY, 3113 .asc = 0x08, /* LOGICAL UNIT COMMUNICATION FAILURE */ 3114 }, 3115 [TCM_INSUFFICIENT_REGISTRATION_RESOURCES] = { 3116 /* 3117 * From spc4r22 section5.7.7,5.7.8 3118 * If a PERSISTENT RESERVE OUT command with a REGISTER service action 3119 * or a REGISTER AND IGNORE EXISTING KEY service action or 3120 * REGISTER AND MOVE service actionis attempted, 3121 * but there are insufficient device server resources to complete the 3122 * operation, then the command shall be terminated with CHECK CONDITION 3123 * status, with the sense key set to ILLEGAL REQUEST,and the additonal 3124 * sense code set to INSUFFICIENT REGISTRATION RESOURCES. 3125 */ 3126 .key = ILLEGAL_REQUEST, 3127 .asc = 0x55, 3128 .ascq = 0x04, /* INSUFFICIENT REGISTRATION RESOURCES */ 3129 }, 3130 }; 3131 3132 static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason) 3133 { 3134 const struct sense_info *si; 3135 u8 *buffer = cmd->sense_buffer; 3136 int r = (__force int)reason; 3137 u8 asc, ascq; 3138 bool desc_format = target_sense_desc_format(cmd->se_dev); 3139 3140 if (r < ARRAY_SIZE(sense_info_table) && sense_info_table[r].key) 3141 si = &sense_info_table[r]; 3142 else 3143 si = &sense_info_table[(__force int) 3144 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE]; 3145 3146 if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) { 3147 core_scsi3_ua_for_check_condition(cmd, &asc, &ascq); 3148 WARN_ON_ONCE(asc == 0); 3149 } else if (si->asc == 0) { 3150 WARN_ON_ONCE(cmd->scsi_asc == 0); 3151 asc = cmd->scsi_asc; 3152 ascq = cmd->scsi_ascq; 3153 } else { 3154 asc = si->asc; 3155 ascq = si->ascq; 3156 } 3157 3158 scsi_build_sense_buffer(desc_format, buffer, si->key, asc, ascq); 3159 if (si->add_sector_info) 3160 return scsi_set_sense_information(buffer, 3161 cmd->scsi_sense_length, 3162 cmd->bad_sector); 3163 3164 return 0; 3165 } 3166 3167 int 3168 transport_send_check_condition_and_sense(struct se_cmd *cmd, 3169 sense_reason_t reason, int from_transport) 3170 { 3171 unsigned long flags; 3172 3173 spin_lock_irqsave(&cmd->t_state_lock, flags); 3174 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) { 3175 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3176 return 0; 3177 } 3178 cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION; 3179 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3180 3181 if (!from_transport) { 3182 int rc; 3183 3184 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; 3185 cmd->scsi_status = SAM_STAT_CHECK_CONDITION; 3186 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; 3187 rc = translate_sense_reason(cmd, reason); 3188 if (rc) 3189 return rc; 3190 } 3191 3192 trace_target_cmd_complete(cmd); 3193 return cmd->se_tfo->queue_status(cmd); 3194 } 3195 EXPORT_SYMBOL(transport_send_check_condition_and_sense); 3196 3197 static int __transport_check_aborted_status(struct se_cmd *cmd, int send_status) 3198 __releases(&cmd->t_state_lock) 3199 __acquires(&cmd->t_state_lock) 3200 { 3201 int ret; 3202 3203 assert_spin_locked(&cmd->t_state_lock); 3204 WARN_ON_ONCE(!irqs_disabled()); 3205 3206 if (!(cmd->transport_state & CMD_T_ABORTED)) 3207 return 0; 3208 /* 3209 * If cmd has been aborted but either no status is to be sent or it has 3210 * already been sent, just return 3211 */ 3212 if (!send_status || !(cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS)) { 3213 if (send_status) 3214 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS; 3215 return 1; 3216 } 3217 3218 pr_debug("Sending delayed SAM_STAT_TASK_ABORTED status for CDB:" 3219 " 0x%02x ITT: 0x%08llx\n", cmd->t_task_cdb[0], cmd->tag); 3220 3221 cmd->se_cmd_flags &= ~SCF_SEND_DELAYED_TAS; 3222 cmd->scsi_status = SAM_STAT_TASK_ABORTED; 3223 trace_target_cmd_complete(cmd); 3224 3225 spin_unlock_irq(&cmd->t_state_lock); 3226 ret = cmd->se_tfo->queue_status(cmd); 3227 if (ret) 3228 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 3229 spin_lock_irq(&cmd->t_state_lock); 3230 3231 return 1; 3232 } 3233 3234 int transport_check_aborted_status(struct se_cmd *cmd, int send_status) 3235 { 3236 int ret; 3237 3238 spin_lock_irq(&cmd->t_state_lock); 3239 ret = __transport_check_aborted_status(cmd, send_status); 3240 spin_unlock_irq(&cmd->t_state_lock); 3241 3242 return ret; 3243 } 3244 EXPORT_SYMBOL(transport_check_aborted_status); 3245 3246 void transport_send_task_abort(struct se_cmd *cmd) 3247 { 3248 unsigned long flags; 3249 int ret; 3250 3251 spin_lock_irqsave(&cmd->t_state_lock, flags); 3252 if (cmd->se_cmd_flags & (SCF_SENT_CHECK_CONDITION)) { 3253 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3254 return; 3255 } 3256 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3257 3258 /* 3259 * If there are still expected incoming fabric WRITEs, we wait 3260 * until until they have completed before sending a TASK_ABORTED 3261 * response. This response with TASK_ABORTED status will be 3262 * queued back to fabric module by transport_check_aborted_status(). 3263 */ 3264 if (cmd->data_direction == DMA_TO_DEVICE) { 3265 if (cmd->se_tfo->write_pending_status(cmd) != 0) { 3266 spin_lock_irqsave(&cmd->t_state_lock, flags); 3267 if (cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS) { 3268 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3269 goto send_abort; 3270 } 3271 cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS; 3272 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3273 return; 3274 } 3275 } 3276 send_abort: 3277 cmd->scsi_status = SAM_STAT_TASK_ABORTED; 3278 3279 transport_lun_remove_cmd(cmd); 3280 3281 pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n", 3282 cmd->t_task_cdb[0], cmd->tag); 3283 3284 trace_target_cmd_complete(cmd); 3285 ret = cmd->se_tfo->queue_status(cmd); 3286 if (ret) 3287 transport_handle_queue_full(cmd, cmd->se_dev, ret, false); 3288 } 3289 3290 static void target_tmr_work(struct work_struct *work) 3291 { 3292 struct se_cmd *cmd = container_of(work, struct se_cmd, work); 3293 struct se_device *dev = cmd->se_dev; 3294 struct se_tmr_req *tmr = cmd->se_tmr_req; 3295 unsigned long flags; 3296 int ret; 3297 3298 spin_lock_irqsave(&cmd->t_state_lock, flags); 3299 if (cmd->transport_state & CMD_T_ABORTED) { 3300 tmr->response = TMR_FUNCTION_REJECTED; 3301 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3302 goto check_stop; 3303 } 3304 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3305 3306 switch (tmr->function) { 3307 case TMR_ABORT_TASK: 3308 core_tmr_abort_task(dev, tmr, cmd->se_sess); 3309 break; 3310 case TMR_ABORT_TASK_SET: 3311 case TMR_CLEAR_ACA: 3312 case TMR_CLEAR_TASK_SET: 3313 tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED; 3314 break; 3315 case TMR_LUN_RESET: 3316 ret = core_tmr_lun_reset(dev, tmr, NULL, NULL); 3317 tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE : 3318 TMR_FUNCTION_REJECTED; 3319 if (tmr->response == TMR_FUNCTION_COMPLETE) { 3320 target_ua_allocate_lun(cmd->se_sess->se_node_acl, 3321 cmd->orig_fe_lun, 0x29, 3322 ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED); 3323 } 3324 break; 3325 case TMR_TARGET_WARM_RESET: 3326 tmr->response = TMR_FUNCTION_REJECTED; 3327 break; 3328 case TMR_TARGET_COLD_RESET: 3329 tmr->response = TMR_FUNCTION_REJECTED; 3330 break; 3331 default: 3332 pr_err("Uknown TMR function: 0x%02x.\n", 3333 tmr->function); 3334 tmr->response = TMR_FUNCTION_REJECTED; 3335 break; 3336 } 3337 3338 spin_lock_irqsave(&cmd->t_state_lock, flags); 3339 if (cmd->transport_state & CMD_T_ABORTED) { 3340 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3341 goto check_stop; 3342 } 3343 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3344 3345 cmd->se_tfo->queue_tm_rsp(cmd); 3346 3347 check_stop: 3348 transport_lun_remove_cmd(cmd); 3349 transport_cmd_check_stop_to_fabric(cmd); 3350 } 3351 3352 int transport_generic_handle_tmr( 3353 struct se_cmd *cmd) 3354 { 3355 unsigned long flags; 3356 bool aborted = false; 3357 3358 spin_lock_irqsave(&cmd->t_state_lock, flags); 3359 if (cmd->transport_state & CMD_T_ABORTED) { 3360 aborted = true; 3361 } else { 3362 cmd->t_state = TRANSPORT_ISTATE_PROCESSING; 3363 cmd->transport_state |= CMD_T_ACTIVE; 3364 } 3365 spin_unlock_irqrestore(&cmd->t_state_lock, flags); 3366 3367 if (aborted) { 3368 pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d" 3369 "ref_tag: %llu tag: %llu\n", cmd->se_tmr_req->function, 3370 cmd->se_tmr_req->ref_task_tag, cmd->tag); 3371 transport_lun_remove_cmd(cmd); 3372 transport_cmd_check_stop_to_fabric(cmd); 3373 return 0; 3374 } 3375 3376 INIT_WORK(&cmd->work, target_tmr_work); 3377 queue_work(cmd->se_dev->tmr_wq, &cmd->work); 3378 return 0; 3379 } 3380 EXPORT_SYMBOL(transport_generic_handle_tmr); 3381 3382 bool 3383 target_check_wce(struct se_device *dev) 3384 { 3385 bool wce = false; 3386 3387 if (dev->transport->get_write_cache) 3388 wce = dev->transport->get_write_cache(dev); 3389 else if (dev->dev_attrib.emulate_write_cache > 0) 3390 wce = true; 3391 3392 return wce; 3393 } 3394 3395 bool 3396 target_check_fua(struct se_device *dev) 3397 { 3398 return target_check_wce(dev) && dev->dev_attrib.emulate_fua_write > 0; 3399 } 3400