1 /* 2 * QLogic iSCSI Offload Driver 3 * Copyright (c) 2016 Cavium Inc. 4 * 5 * This software is available under the terms of the GNU General Public License 6 * (GPL) Version 2, available from the file COPYING in the main directory of 7 * this source tree. 8 */ 9 10 #include <linux/blkdev.h> 11 #include <scsi/scsi_tcq.h> 12 #include <linux/delay.h> 13 14 #include "qedi.h" 15 #include "qedi_iscsi.h" 16 #include "qedi_gbl.h" 17 18 static int qedi_send_iscsi_tmf(struct qedi_conn *qedi_conn, 19 struct iscsi_task *mtask); 20 21 void qedi_iscsi_unmap_sg_list(struct qedi_cmd *cmd) 22 { 23 struct scsi_cmnd *sc = cmd->scsi_cmd; 24 25 if (cmd->io_tbl.sge_valid && sc) { 26 cmd->io_tbl.sge_valid = 0; 27 scsi_dma_unmap(sc); 28 } 29 } 30 31 static void qedi_process_logout_resp(struct qedi_ctx *qedi, 32 union iscsi_cqe *cqe, 33 struct iscsi_task *task, 34 struct qedi_conn *qedi_conn) 35 { 36 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 37 struct iscsi_logout_rsp *resp_hdr; 38 struct iscsi_session *session = conn->session; 39 struct iscsi_logout_response_hdr *cqe_logout_response; 40 struct qedi_cmd *cmd; 41 42 cmd = (struct qedi_cmd *)task->dd_data; 43 cqe_logout_response = &cqe->cqe_common.iscsi_hdr.logout_response; 44 spin_lock(&session->back_lock); 45 resp_hdr = (struct iscsi_logout_rsp *)&qedi_conn->gen_pdu.resp_hdr; 46 memset(resp_hdr, 0, sizeof(struct iscsi_hdr)); 47 resp_hdr->opcode = cqe_logout_response->opcode; 48 resp_hdr->flags = cqe_logout_response->flags; 49 resp_hdr->hlength = 0; 50 51 resp_hdr->itt = build_itt(cqe->cqe_solicited.itid, conn->session->age); 52 resp_hdr->statsn = cpu_to_be32(cqe_logout_response->stat_sn); 53 resp_hdr->exp_cmdsn = cpu_to_be32(cqe_logout_response->exp_cmd_sn); 54 resp_hdr->max_cmdsn = cpu_to_be32(cqe_logout_response->max_cmd_sn); 55 56 resp_hdr->t2wait = cpu_to_be32(cqe_logout_response->time2wait); 57 resp_hdr->t2retain = cpu_to_be32(cqe_logout_response->time2retain); 58 59 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID, 60 "Freeing tid=0x%x for cid=0x%x\n", 61 cmd->task_id, qedi_conn->iscsi_conn_id); 62 63 if (likely(cmd->io_cmd_in_list)) { 64 cmd->io_cmd_in_list = false; 65 list_del_init(&cmd->io_cmd); 66 qedi_conn->active_cmd_count--; 67 } else { 68 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, 69 "Active cmd list node already deleted, tid=0x%x, cid=0x%x, io_cmd_node=%p\n", 70 cmd->task_id, qedi_conn->iscsi_conn_id, 71 &cmd->io_cmd); 72 } 73 74 cmd->state = RESPONSE_RECEIVED; 75 qedi_clear_task_idx(qedi, cmd->task_id); 76 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr, NULL, 0); 77 78 spin_unlock(&session->back_lock); 79 } 80 81 static void qedi_process_text_resp(struct qedi_ctx *qedi, 82 union iscsi_cqe *cqe, 83 struct iscsi_task *task, 84 struct qedi_conn *qedi_conn) 85 { 86 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 87 struct iscsi_session *session = conn->session; 88 struct iscsi_task_context *task_ctx; 89 struct iscsi_text_rsp *resp_hdr_ptr; 90 struct iscsi_text_response_hdr *cqe_text_response; 91 struct qedi_cmd *cmd; 92 int pld_len; 93 u32 *tmp; 94 95 cmd = (struct qedi_cmd *)task->dd_data; 96 task_ctx = qedi_get_task_mem(&qedi->tasks, cmd->task_id); 97 98 cqe_text_response = &cqe->cqe_common.iscsi_hdr.text_response; 99 spin_lock(&session->back_lock); 100 resp_hdr_ptr = (struct iscsi_text_rsp *)&qedi_conn->gen_pdu.resp_hdr; 101 memset(resp_hdr_ptr, 0, sizeof(struct iscsi_hdr)); 102 resp_hdr_ptr->opcode = cqe_text_response->opcode; 103 resp_hdr_ptr->flags = cqe_text_response->flags; 104 resp_hdr_ptr->hlength = 0; 105 106 hton24(resp_hdr_ptr->dlength, 107 (cqe_text_response->hdr_second_dword & 108 ISCSI_TEXT_RESPONSE_HDR_DATA_SEG_LEN_MASK)); 109 tmp = (u32 *)resp_hdr_ptr->dlength; 110 111 resp_hdr_ptr->itt = build_itt(cqe->cqe_solicited.itid, 112 conn->session->age); 113 resp_hdr_ptr->ttt = cqe_text_response->ttt; 114 resp_hdr_ptr->statsn = cpu_to_be32(cqe_text_response->stat_sn); 115 resp_hdr_ptr->exp_cmdsn = cpu_to_be32(cqe_text_response->exp_cmd_sn); 116 resp_hdr_ptr->max_cmdsn = cpu_to_be32(cqe_text_response->max_cmd_sn); 117 118 pld_len = cqe_text_response->hdr_second_dword & 119 ISCSI_TEXT_RESPONSE_HDR_DATA_SEG_LEN_MASK; 120 qedi_conn->gen_pdu.resp_wr_ptr = qedi_conn->gen_pdu.resp_buf + pld_len; 121 122 memset(task_ctx, '\0', sizeof(*task_ctx)); 123 124 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID, 125 "Freeing tid=0x%x for cid=0x%x\n", 126 cmd->task_id, qedi_conn->iscsi_conn_id); 127 128 if (likely(cmd->io_cmd_in_list)) { 129 cmd->io_cmd_in_list = false; 130 list_del_init(&cmd->io_cmd); 131 qedi_conn->active_cmd_count--; 132 } else { 133 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, 134 "Active cmd list node already deleted, tid=0x%x, cid=0x%x, io_cmd_node=%p\n", 135 cmd->task_id, qedi_conn->iscsi_conn_id, 136 &cmd->io_cmd); 137 } 138 139 cmd->state = RESPONSE_RECEIVED; 140 qedi_clear_task_idx(qedi, cmd->task_id); 141 142 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr, 143 qedi_conn->gen_pdu.resp_buf, 144 (qedi_conn->gen_pdu.resp_wr_ptr - 145 qedi_conn->gen_pdu.resp_buf)); 146 spin_unlock(&session->back_lock); 147 } 148 149 static void qedi_tmf_resp_work(struct work_struct *work) 150 { 151 struct qedi_cmd *qedi_cmd = 152 container_of(work, struct qedi_cmd, tmf_work); 153 struct qedi_conn *qedi_conn = qedi_cmd->conn; 154 struct qedi_ctx *qedi = qedi_conn->qedi; 155 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 156 struct iscsi_session *session = conn->session; 157 struct iscsi_tm_rsp *resp_hdr_ptr; 158 struct iscsi_cls_session *cls_sess; 159 int rval = 0; 160 161 set_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags); 162 resp_hdr_ptr = (struct iscsi_tm_rsp *)qedi_cmd->tmf_resp_buf; 163 cls_sess = iscsi_conn_to_session(qedi_conn->cls_conn); 164 165 iscsi_block_session(session->cls_session); 166 rval = qedi_cleanup_all_io(qedi, qedi_conn, qedi_cmd->task, true); 167 if (rval) { 168 qedi_clear_task_idx(qedi, qedi_cmd->task_id); 169 iscsi_unblock_session(session->cls_session); 170 goto exit_tmf_resp; 171 } 172 173 iscsi_unblock_session(session->cls_session); 174 qedi_clear_task_idx(qedi, qedi_cmd->task_id); 175 176 spin_lock(&session->back_lock); 177 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr, NULL, 0); 178 spin_unlock(&session->back_lock); 179 180 exit_tmf_resp: 181 kfree(resp_hdr_ptr); 182 clear_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags); 183 } 184 185 static void qedi_process_tmf_resp(struct qedi_ctx *qedi, 186 union iscsi_cqe *cqe, 187 struct iscsi_task *task, 188 struct qedi_conn *qedi_conn) 189 190 { 191 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 192 struct iscsi_session *session = conn->session; 193 struct iscsi_tmf_response_hdr *cqe_tmp_response; 194 struct iscsi_tm_rsp *resp_hdr_ptr; 195 struct iscsi_tm *tmf_hdr; 196 struct qedi_cmd *qedi_cmd = NULL; 197 u32 *tmp; 198 199 cqe_tmp_response = &cqe->cqe_common.iscsi_hdr.tmf_response; 200 201 qedi_cmd = task->dd_data; 202 qedi_cmd->tmf_resp_buf = kzalloc(sizeof(*resp_hdr_ptr), GFP_KERNEL); 203 if (!qedi_cmd->tmf_resp_buf) { 204 QEDI_ERR(&qedi->dbg_ctx, 205 "Failed to allocate resp buf, cid=0x%x\n", 206 qedi_conn->iscsi_conn_id); 207 return; 208 } 209 210 spin_lock(&session->back_lock); 211 resp_hdr_ptr = (struct iscsi_tm_rsp *)qedi_cmd->tmf_resp_buf; 212 memset(resp_hdr_ptr, 0, sizeof(struct iscsi_tm_rsp)); 213 214 /* Fill up the header */ 215 resp_hdr_ptr->opcode = cqe_tmp_response->opcode; 216 resp_hdr_ptr->flags = cqe_tmp_response->hdr_flags; 217 resp_hdr_ptr->response = cqe_tmp_response->hdr_response; 218 resp_hdr_ptr->hlength = 0; 219 220 hton24(resp_hdr_ptr->dlength, 221 (cqe_tmp_response->hdr_second_dword & 222 ISCSI_TMF_RESPONSE_HDR_DATA_SEG_LEN_MASK)); 223 tmp = (u32 *)resp_hdr_ptr->dlength; 224 resp_hdr_ptr->itt = build_itt(cqe->cqe_solicited.itid, 225 conn->session->age); 226 resp_hdr_ptr->statsn = cpu_to_be32(cqe_tmp_response->stat_sn); 227 resp_hdr_ptr->exp_cmdsn = cpu_to_be32(cqe_tmp_response->exp_cmd_sn); 228 resp_hdr_ptr->max_cmdsn = cpu_to_be32(cqe_tmp_response->max_cmd_sn); 229 230 tmf_hdr = (struct iscsi_tm *)qedi_cmd->task->hdr; 231 232 if (likely(qedi_cmd->io_cmd_in_list)) { 233 qedi_cmd->io_cmd_in_list = false; 234 list_del_init(&qedi_cmd->io_cmd); 235 qedi_conn->active_cmd_count--; 236 } 237 238 if (((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) == 239 ISCSI_TM_FUNC_LOGICAL_UNIT_RESET) || 240 ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) == 241 ISCSI_TM_FUNC_TARGET_WARM_RESET) || 242 ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) == 243 ISCSI_TM_FUNC_TARGET_COLD_RESET)) { 244 INIT_WORK(&qedi_cmd->tmf_work, qedi_tmf_resp_work); 245 queue_work(qedi->tmf_thread, &qedi_cmd->tmf_work); 246 goto unblock_sess; 247 } 248 249 qedi_clear_task_idx(qedi, qedi_cmd->task_id); 250 251 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr, NULL, 0); 252 kfree(resp_hdr_ptr); 253 254 unblock_sess: 255 spin_unlock(&session->back_lock); 256 } 257 258 static void qedi_process_login_resp(struct qedi_ctx *qedi, 259 union iscsi_cqe *cqe, 260 struct iscsi_task *task, 261 struct qedi_conn *qedi_conn) 262 { 263 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 264 struct iscsi_session *session = conn->session; 265 struct iscsi_task_context *task_ctx; 266 struct iscsi_login_rsp *resp_hdr_ptr; 267 struct iscsi_login_response_hdr *cqe_login_response; 268 struct qedi_cmd *cmd; 269 int pld_len; 270 u32 *tmp; 271 272 cmd = (struct qedi_cmd *)task->dd_data; 273 274 cqe_login_response = &cqe->cqe_common.iscsi_hdr.login_response; 275 task_ctx = qedi_get_task_mem(&qedi->tasks, cmd->task_id); 276 277 spin_lock(&session->back_lock); 278 resp_hdr_ptr = (struct iscsi_login_rsp *)&qedi_conn->gen_pdu.resp_hdr; 279 memset(resp_hdr_ptr, 0, sizeof(struct iscsi_login_rsp)); 280 resp_hdr_ptr->opcode = cqe_login_response->opcode; 281 resp_hdr_ptr->flags = cqe_login_response->flags_attr; 282 resp_hdr_ptr->hlength = 0; 283 284 hton24(resp_hdr_ptr->dlength, 285 (cqe_login_response->hdr_second_dword & 286 ISCSI_LOGIN_RESPONSE_HDR_DATA_SEG_LEN_MASK)); 287 tmp = (u32 *)resp_hdr_ptr->dlength; 288 resp_hdr_ptr->itt = build_itt(cqe->cqe_solicited.itid, 289 conn->session->age); 290 resp_hdr_ptr->tsih = cqe_login_response->tsih; 291 resp_hdr_ptr->statsn = cpu_to_be32(cqe_login_response->stat_sn); 292 resp_hdr_ptr->exp_cmdsn = cpu_to_be32(cqe_login_response->exp_cmd_sn); 293 resp_hdr_ptr->max_cmdsn = cpu_to_be32(cqe_login_response->max_cmd_sn); 294 resp_hdr_ptr->status_class = cqe_login_response->status_class; 295 resp_hdr_ptr->status_detail = cqe_login_response->status_detail; 296 pld_len = cqe_login_response->hdr_second_dword & 297 ISCSI_LOGIN_RESPONSE_HDR_DATA_SEG_LEN_MASK; 298 qedi_conn->gen_pdu.resp_wr_ptr = qedi_conn->gen_pdu.resp_buf + pld_len; 299 300 if (likely(cmd->io_cmd_in_list)) { 301 cmd->io_cmd_in_list = false; 302 list_del_init(&cmd->io_cmd); 303 qedi_conn->active_cmd_count--; 304 } 305 306 memset(task_ctx, '\0', sizeof(*task_ctx)); 307 308 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr_ptr, 309 qedi_conn->gen_pdu.resp_buf, 310 (qedi_conn->gen_pdu.resp_wr_ptr - 311 qedi_conn->gen_pdu.resp_buf)); 312 313 spin_unlock(&session->back_lock); 314 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID, 315 "Freeing tid=0x%x for cid=0x%x\n", 316 cmd->task_id, qedi_conn->iscsi_conn_id); 317 cmd->state = RESPONSE_RECEIVED; 318 qedi_clear_task_idx(qedi, cmd->task_id); 319 } 320 321 static void qedi_get_rq_bdq_buf(struct qedi_ctx *qedi, 322 struct iscsi_cqe_unsolicited *cqe, 323 char *ptr, int len) 324 { 325 u16 idx = 0; 326 327 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, 328 "pld_len [%d], bdq_prod_idx [%d], idx [%d]\n", 329 len, qedi->bdq_prod_idx, 330 (qedi->bdq_prod_idx % qedi->rq_num_entries)); 331 332 /* Obtain buffer address from rqe_opaque */ 333 idx = cqe->rqe_opaque.lo; 334 if ((idx < 0) || (idx > (QEDI_BDQ_NUM - 1))) { 335 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, 336 "wrong idx %d returned by FW, dropping the unsolicited pkt\n", 337 idx); 338 return; 339 } 340 341 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, 342 "rqe_opaque.lo [0x%p], rqe_opaque.hi [0x%p], idx [%d]\n", 343 cqe->rqe_opaque.lo, cqe->rqe_opaque.hi, idx); 344 345 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, 346 "unsol_cqe_type = %d\n", cqe->unsol_cqe_type); 347 switch (cqe->unsol_cqe_type) { 348 case ISCSI_CQE_UNSOLICITED_SINGLE: 349 case ISCSI_CQE_UNSOLICITED_FIRST: 350 if (len) 351 memcpy(ptr, (void *)qedi->bdq[idx].buf_addr, len); 352 break; 353 case ISCSI_CQE_UNSOLICITED_MIDDLE: 354 case ISCSI_CQE_UNSOLICITED_LAST: 355 break; 356 default: 357 break; 358 } 359 } 360 361 static void qedi_put_rq_bdq_buf(struct qedi_ctx *qedi, 362 struct iscsi_cqe_unsolicited *cqe, 363 int count) 364 { 365 u16 tmp; 366 u16 idx = 0; 367 struct scsi_bd *pbl; 368 369 /* Obtain buffer address from rqe_opaque */ 370 idx = cqe->rqe_opaque.lo; 371 if ((idx < 0) || (idx > (QEDI_BDQ_NUM - 1))) { 372 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, 373 "wrong idx %d returned by FW, dropping the unsolicited pkt\n", 374 idx); 375 return; 376 } 377 378 pbl = (struct scsi_bd *)qedi->bdq_pbl; 379 pbl += (qedi->bdq_prod_idx % qedi->rq_num_entries); 380 pbl->address.hi = cpu_to_le32(QEDI_U64_HI(qedi->bdq[idx].buf_dma)); 381 pbl->address.lo = cpu_to_le32(QEDI_U64_LO(qedi->bdq[idx].buf_dma)); 382 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, 383 "pbl [0x%p] pbl->address hi [0x%llx] lo [0x%llx] idx [%d]\n", 384 pbl, pbl->address.hi, pbl->address.lo, idx); 385 pbl->opaque.hi = 0; 386 pbl->opaque.lo = cpu_to_le32(QEDI_U64_LO(idx)); 387 388 /* Increment producer to let f/w know we've handled the frame */ 389 qedi->bdq_prod_idx += count; 390 391 writew(qedi->bdq_prod_idx, qedi->bdq_primary_prod); 392 tmp = readw(qedi->bdq_primary_prod); 393 394 writew(qedi->bdq_prod_idx, qedi->bdq_secondary_prod); 395 tmp = readw(qedi->bdq_secondary_prod); 396 } 397 398 static void qedi_unsol_pdu_adjust_bdq(struct qedi_ctx *qedi, 399 struct iscsi_cqe_unsolicited *cqe, 400 u32 pdu_len, u32 num_bdqs, 401 char *bdq_data) 402 { 403 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, 404 "num_bdqs [%d]\n", num_bdqs); 405 406 qedi_get_rq_bdq_buf(qedi, cqe, bdq_data, pdu_len); 407 qedi_put_rq_bdq_buf(qedi, cqe, (num_bdqs + 1)); 408 } 409 410 static int qedi_process_nopin_mesg(struct qedi_ctx *qedi, 411 union iscsi_cqe *cqe, 412 struct iscsi_task *task, 413 struct qedi_conn *qedi_conn, u16 que_idx) 414 { 415 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 416 struct iscsi_session *session = conn->session; 417 struct iscsi_nop_in_hdr *cqe_nop_in; 418 struct iscsi_nopin *hdr; 419 struct qedi_cmd *cmd; 420 int tgt_async_nop = 0; 421 u32 lun[2]; 422 u32 pdu_len, num_bdqs; 423 char bdq_data[QEDI_BDQ_BUF_SIZE]; 424 unsigned long flags; 425 426 spin_lock_bh(&session->back_lock); 427 cqe_nop_in = &cqe->cqe_common.iscsi_hdr.nop_in; 428 429 pdu_len = cqe_nop_in->hdr_second_dword & 430 ISCSI_NOP_IN_HDR_DATA_SEG_LEN_MASK; 431 num_bdqs = pdu_len / QEDI_BDQ_BUF_SIZE; 432 433 hdr = (struct iscsi_nopin *)&qedi_conn->gen_pdu.resp_hdr; 434 memset(hdr, 0, sizeof(struct iscsi_hdr)); 435 hdr->opcode = cqe_nop_in->opcode; 436 hdr->max_cmdsn = cpu_to_be32(cqe_nop_in->max_cmd_sn); 437 hdr->exp_cmdsn = cpu_to_be32(cqe_nop_in->exp_cmd_sn); 438 hdr->statsn = cpu_to_be32(cqe_nop_in->stat_sn); 439 hdr->ttt = cpu_to_be32(cqe_nop_in->ttt); 440 441 if (cqe->cqe_common.cqe_type == ISCSI_CQE_TYPE_UNSOLICITED) { 442 spin_lock_irqsave(&qedi->hba_lock, flags); 443 qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited, 444 pdu_len, num_bdqs, bdq_data); 445 hdr->itt = RESERVED_ITT; 446 tgt_async_nop = 1; 447 spin_unlock_irqrestore(&qedi->hba_lock, flags); 448 goto done; 449 } 450 451 /* Response to one of our nop-outs */ 452 if (task) { 453 cmd = task->dd_data; 454 hdr->flags = ISCSI_FLAG_CMD_FINAL; 455 hdr->itt = build_itt(cqe->cqe_solicited.itid, 456 conn->session->age); 457 lun[0] = 0xffffffff; 458 lun[1] = 0xffffffff; 459 memcpy(&hdr->lun, lun, sizeof(struct scsi_lun)); 460 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID, 461 "Freeing tid=0x%x for cid=0x%x\n", 462 cmd->task_id, qedi_conn->iscsi_conn_id); 463 cmd->state = RESPONSE_RECEIVED; 464 spin_lock(&qedi_conn->list_lock); 465 if (likely(cmd->io_cmd_in_list)) { 466 cmd->io_cmd_in_list = false; 467 list_del_init(&cmd->io_cmd); 468 qedi_conn->active_cmd_count--; 469 } 470 471 spin_unlock(&qedi_conn->list_lock); 472 qedi_clear_task_idx(qedi, cmd->task_id); 473 } 474 475 done: 476 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, bdq_data, pdu_len); 477 478 spin_unlock_bh(&session->back_lock); 479 return tgt_async_nop; 480 } 481 482 static void qedi_process_async_mesg(struct qedi_ctx *qedi, 483 union iscsi_cqe *cqe, 484 struct iscsi_task *task, 485 struct qedi_conn *qedi_conn, 486 u16 que_idx) 487 { 488 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 489 struct iscsi_session *session = conn->session; 490 struct iscsi_async_msg_hdr *cqe_async_msg; 491 struct iscsi_async *resp_hdr; 492 u32 lun[2]; 493 u32 pdu_len, num_bdqs; 494 char bdq_data[QEDI_BDQ_BUF_SIZE]; 495 unsigned long flags; 496 497 spin_lock_bh(&session->back_lock); 498 499 cqe_async_msg = &cqe->cqe_common.iscsi_hdr.async_msg; 500 pdu_len = cqe_async_msg->hdr_second_dword & 501 ISCSI_ASYNC_MSG_HDR_DATA_SEG_LEN_MASK; 502 num_bdqs = pdu_len / QEDI_BDQ_BUF_SIZE; 503 504 if (cqe->cqe_common.cqe_type == ISCSI_CQE_TYPE_UNSOLICITED) { 505 spin_lock_irqsave(&qedi->hba_lock, flags); 506 qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited, 507 pdu_len, num_bdqs, bdq_data); 508 spin_unlock_irqrestore(&qedi->hba_lock, flags); 509 } 510 511 resp_hdr = (struct iscsi_async *)&qedi_conn->gen_pdu.resp_hdr; 512 memset(resp_hdr, 0, sizeof(struct iscsi_hdr)); 513 resp_hdr->opcode = cqe_async_msg->opcode; 514 resp_hdr->flags = 0x80; 515 516 lun[0] = cpu_to_be32(cqe_async_msg->lun.lo); 517 lun[1] = cpu_to_be32(cqe_async_msg->lun.hi); 518 memcpy(&resp_hdr->lun, lun, sizeof(struct scsi_lun)); 519 resp_hdr->exp_cmdsn = cpu_to_be32(cqe_async_msg->exp_cmd_sn); 520 resp_hdr->max_cmdsn = cpu_to_be32(cqe_async_msg->max_cmd_sn); 521 resp_hdr->statsn = cpu_to_be32(cqe_async_msg->stat_sn); 522 523 resp_hdr->async_event = cqe_async_msg->async_event; 524 resp_hdr->async_vcode = cqe_async_msg->async_vcode; 525 526 resp_hdr->param1 = cpu_to_be16(cqe_async_msg->param1_rsrv); 527 resp_hdr->param2 = cpu_to_be16(cqe_async_msg->param2_rsrv); 528 resp_hdr->param3 = cpu_to_be16(cqe_async_msg->param3_rsrv); 529 530 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)resp_hdr, bdq_data, 531 pdu_len); 532 533 spin_unlock_bh(&session->back_lock); 534 } 535 536 static void qedi_process_reject_mesg(struct qedi_ctx *qedi, 537 union iscsi_cqe *cqe, 538 struct iscsi_task *task, 539 struct qedi_conn *qedi_conn, 540 uint16_t que_idx) 541 { 542 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 543 struct iscsi_session *session = conn->session; 544 struct iscsi_reject_hdr *cqe_reject; 545 struct iscsi_reject *hdr; 546 u32 pld_len, num_bdqs; 547 unsigned long flags; 548 549 spin_lock_bh(&session->back_lock); 550 cqe_reject = &cqe->cqe_common.iscsi_hdr.reject; 551 pld_len = cqe_reject->hdr_second_dword & 552 ISCSI_REJECT_HDR_DATA_SEG_LEN_MASK; 553 num_bdqs = pld_len / QEDI_BDQ_BUF_SIZE; 554 555 if (cqe->cqe_common.cqe_type == ISCSI_CQE_TYPE_UNSOLICITED) { 556 spin_lock_irqsave(&qedi->hba_lock, flags); 557 qedi_unsol_pdu_adjust_bdq(qedi, &cqe->cqe_unsolicited, 558 pld_len, num_bdqs, conn->data); 559 spin_unlock_irqrestore(&qedi->hba_lock, flags); 560 } 561 hdr = (struct iscsi_reject *)&qedi_conn->gen_pdu.resp_hdr; 562 memset(hdr, 0, sizeof(struct iscsi_hdr)); 563 hdr->opcode = cqe_reject->opcode; 564 hdr->reason = cqe_reject->hdr_reason; 565 hdr->flags = cqe_reject->hdr_flags; 566 hton24(hdr->dlength, (cqe_reject->hdr_second_dword & 567 ISCSI_REJECT_HDR_DATA_SEG_LEN_MASK)); 568 hdr->max_cmdsn = cpu_to_be32(cqe_reject->max_cmd_sn); 569 hdr->exp_cmdsn = cpu_to_be32(cqe_reject->exp_cmd_sn); 570 hdr->statsn = cpu_to_be32(cqe_reject->stat_sn); 571 hdr->ffffffff = cpu_to_be32(0xffffffff); 572 573 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, 574 conn->data, pld_len); 575 spin_unlock_bh(&session->back_lock); 576 } 577 578 static void qedi_scsi_completion(struct qedi_ctx *qedi, 579 union iscsi_cqe *cqe, 580 struct iscsi_task *task, 581 struct iscsi_conn *conn) 582 { 583 struct scsi_cmnd *sc_cmd; 584 struct qedi_cmd *cmd = task->dd_data; 585 struct iscsi_session *session = conn->session; 586 struct iscsi_scsi_rsp *hdr; 587 struct iscsi_data_in_hdr *cqe_data_in; 588 int datalen = 0; 589 struct qedi_conn *qedi_conn; 590 u32 iscsi_cid; 591 bool mark_cmd_node_deleted = false; 592 u8 cqe_err_bits = 0; 593 594 iscsi_cid = cqe->cqe_common.conn_id; 595 qedi_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid]; 596 597 cqe_data_in = &cqe->cqe_common.iscsi_hdr.data_in; 598 cqe_err_bits = 599 cqe->cqe_common.error_bitmap.error_bits.cqe_error_status_bits; 600 601 spin_lock_bh(&session->back_lock); 602 /* get the scsi command */ 603 sc_cmd = cmd->scsi_cmd; 604 605 if (!sc_cmd) { 606 QEDI_WARN(&qedi->dbg_ctx, "sc_cmd is NULL!\n"); 607 goto error; 608 } 609 610 if (!sc_cmd->SCp.ptr) { 611 QEDI_WARN(&qedi->dbg_ctx, 612 "SCp.ptr is NULL, returned in another context.\n"); 613 goto error; 614 } 615 616 if (!sc_cmd->request) { 617 QEDI_WARN(&qedi->dbg_ctx, 618 "sc_cmd->request is NULL, sc_cmd=%p.\n", 619 sc_cmd); 620 goto error; 621 } 622 623 if (!sc_cmd->request->special) { 624 QEDI_WARN(&qedi->dbg_ctx, 625 "request->special is NULL so request not valid, sc_cmd=%p.\n", 626 sc_cmd); 627 goto error; 628 } 629 630 if (!sc_cmd->request->q) { 631 QEDI_WARN(&qedi->dbg_ctx, 632 "request->q is NULL so request is not valid, sc_cmd=%p.\n", 633 sc_cmd); 634 goto error; 635 } 636 637 qedi_iscsi_unmap_sg_list(cmd); 638 639 hdr = (struct iscsi_scsi_rsp *)task->hdr; 640 hdr->opcode = cqe_data_in->opcode; 641 hdr->max_cmdsn = cpu_to_be32(cqe_data_in->max_cmd_sn); 642 hdr->exp_cmdsn = cpu_to_be32(cqe_data_in->exp_cmd_sn); 643 hdr->itt = build_itt(cqe->cqe_solicited.itid, conn->session->age); 644 hdr->response = cqe_data_in->reserved1; 645 hdr->cmd_status = cqe_data_in->status_rsvd; 646 hdr->flags = cqe_data_in->flags; 647 hdr->residual_count = cpu_to_be32(cqe_data_in->residual_count); 648 649 if (hdr->cmd_status == SAM_STAT_CHECK_CONDITION) { 650 datalen = cqe_data_in->reserved2 & 651 ISCSI_COMMON_HDR_DATA_SEG_LEN_MASK; 652 memcpy((char *)conn->data, (char *)cmd->sense_buffer, datalen); 653 } 654 655 /* If f/w reports data underrun err then set residual to IO transfer 656 * length, set Underrun flag and clear Overrun flag explicitly 657 */ 658 if (unlikely(cqe_err_bits && 659 GET_FIELD(cqe_err_bits, CQE_ERROR_BITMAP_UNDER_RUN_ERR))) { 660 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, 661 "Under flow itt=0x%x proto flags=0x%x tid=0x%x cid 0x%x fw resid 0x%x sc dlen 0x%x\n", 662 hdr->itt, cqe_data_in->flags, cmd->task_id, 663 qedi_conn->iscsi_conn_id, hdr->residual_count, 664 scsi_bufflen(sc_cmd)); 665 hdr->residual_count = cpu_to_be32(scsi_bufflen(sc_cmd)); 666 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW; 667 hdr->flags &= (~ISCSI_FLAG_CMD_OVERFLOW); 668 } 669 670 spin_lock(&qedi_conn->list_lock); 671 if (likely(cmd->io_cmd_in_list)) { 672 cmd->io_cmd_in_list = false; 673 list_del_init(&cmd->io_cmd); 674 qedi_conn->active_cmd_count--; 675 mark_cmd_node_deleted = true; 676 } 677 spin_unlock(&qedi_conn->list_lock); 678 679 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID, 680 "Freeing tid=0x%x for cid=0x%x\n", 681 cmd->task_id, qedi_conn->iscsi_conn_id); 682 cmd->state = RESPONSE_RECEIVED; 683 if (qedi_io_tracing) 684 qedi_trace_io(qedi, task, cmd->task_id, QEDI_IO_TRACE_RSP); 685 686 qedi_clear_task_idx(qedi, cmd->task_id); 687 __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, 688 conn->data, datalen); 689 error: 690 spin_unlock_bh(&session->back_lock); 691 } 692 693 static void qedi_mtask_completion(struct qedi_ctx *qedi, 694 union iscsi_cqe *cqe, 695 struct iscsi_task *task, 696 struct qedi_conn *conn, uint16_t que_idx) 697 { 698 struct iscsi_conn *iscsi_conn; 699 u32 hdr_opcode; 700 701 hdr_opcode = cqe->cqe_common.iscsi_hdr.common.hdr_first_byte; 702 iscsi_conn = conn->cls_conn->dd_data; 703 704 switch (hdr_opcode) { 705 case ISCSI_OPCODE_SCSI_RESPONSE: 706 case ISCSI_OPCODE_DATA_IN: 707 qedi_scsi_completion(qedi, cqe, task, iscsi_conn); 708 break; 709 case ISCSI_OPCODE_LOGIN_RESPONSE: 710 qedi_process_login_resp(qedi, cqe, task, conn); 711 break; 712 case ISCSI_OPCODE_TMF_RESPONSE: 713 qedi_process_tmf_resp(qedi, cqe, task, conn); 714 break; 715 case ISCSI_OPCODE_TEXT_RESPONSE: 716 qedi_process_text_resp(qedi, cqe, task, conn); 717 break; 718 case ISCSI_OPCODE_LOGOUT_RESPONSE: 719 qedi_process_logout_resp(qedi, cqe, task, conn); 720 break; 721 case ISCSI_OPCODE_NOP_IN: 722 qedi_process_nopin_mesg(qedi, cqe, task, conn, que_idx); 723 break; 724 default: 725 QEDI_ERR(&qedi->dbg_ctx, "unknown opcode\n"); 726 } 727 } 728 729 static void qedi_process_nopin_local_cmpl(struct qedi_ctx *qedi, 730 struct iscsi_cqe_solicited *cqe, 731 struct iscsi_task *task, 732 struct qedi_conn *qedi_conn) 733 { 734 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 735 struct iscsi_session *session = conn->session; 736 struct qedi_cmd *cmd = task->dd_data; 737 738 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_UNSOL, 739 "itid=0x%x, cmd task id=0x%x\n", 740 cqe->itid, cmd->task_id); 741 742 cmd->state = RESPONSE_RECEIVED; 743 qedi_clear_task_idx(qedi, cmd->task_id); 744 745 spin_lock_bh(&session->back_lock); 746 __iscsi_put_task(task); 747 spin_unlock_bh(&session->back_lock); 748 } 749 750 static void qedi_process_cmd_cleanup_resp(struct qedi_ctx *qedi, 751 struct iscsi_cqe_solicited *cqe, 752 struct iscsi_task *task, 753 struct iscsi_conn *conn) 754 { 755 struct qedi_work_map *work, *work_tmp; 756 u32 proto_itt = cqe->itid; 757 u32 ptmp_itt = 0; 758 itt_t protoitt = 0; 759 int found = 0; 760 struct qedi_cmd *qedi_cmd = NULL; 761 u32 rtid = 0; 762 u32 iscsi_cid; 763 struct qedi_conn *qedi_conn; 764 struct qedi_cmd *cmd_new, *dbg_cmd; 765 struct iscsi_task *mtask; 766 struct iscsi_tm *tmf_hdr = NULL; 767 768 iscsi_cid = cqe->conn_id; 769 qedi_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid]; 770 771 /* Based on this itt get the corresponding qedi_cmd */ 772 spin_lock_bh(&qedi_conn->tmf_work_lock); 773 list_for_each_entry_safe(work, work_tmp, &qedi_conn->tmf_work_list, 774 list) { 775 if (work->rtid == proto_itt) { 776 /* We found the command */ 777 qedi_cmd = work->qedi_cmd; 778 if (!qedi_cmd->list_tmf_work) { 779 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 780 "TMF work not found, cqe->tid=0x%x, cid=0x%x\n", 781 proto_itt, qedi_conn->iscsi_conn_id); 782 WARN_ON(1); 783 } 784 found = 1; 785 mtask = qedi_cmd->task; 786 tmf_hdr = (struct iscsi_tm *)mtask->hdr; 787 rtid = work->rtid; 788 789 list_del_init(&work->list); 790 kfree(work); 791 qedi_cmd->list_tmf_work = NULL; 792 } 793 } 794 spin_unlock_bh(&qedi_conn->tmf_work_lock); 795 796 if (found) { 797 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 798 "TMF work, cqe->tid=0x%x, tmf flags=0x%x, cid=0x%x\n", 799 proto_itt, tmf_hdr->flags, qedi_conn->iscsi_conn_id); 800 801 if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) == 802 ISCSI_TM_FUNC_ABORT_TASK) { 803 spin_lock_bh(&conn->session->back_lock); 804 805 protoitt = build_itt(get_itt(tmf_hdr->rtt), 806 conn->session->age); 807 task = iscsi_itt_to_task(conn, protoitt); 808 809 spin_unlock_bh(&conn->session->back_lock); 810 811 if (!task) { 812 QEDI_NOTICE(&qedi->dbg_ctx, 813 "IO task completed, tmf rtt=0x%x, cid=0x%x\n", 814 get_itt(tmf_hdr->rtt), 815 qedi_conn->iscsi_conn_id); 816 return; 817 } 818 819 dbg_cmd = task->dd_data; 820 821 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 822 "Abort tmf rtt=0x%x, i/o itt=0x%x, i/o tid=0x%x, cid=0x%x\n", 823 get_itt(tmf_hdr->rtt), get_itt(task->itt), 824 dbg_cmd->task_id, qedi_conn->iscsi_conn_id); 825 826 if (qedi_cmd->state == CLEANUP_WAIT_FAILED) 827 qedi_cmd->state = CLEANUP_RECV; 828 829 qedi_clear_task_idx(qedi_conn->qedi, rtid); 830 831 spin_lock(&qedi_conn->list_lock); 832 list_del_init(&dbg_cmd->io_cmd); 833 qedi_conn->active_cmd_count--; 834 spin_unlock(&qedi_conn->list_lock); 835 qedi_cmd->state = CLEANUP_RECV; 836 wake_up_interruptible(&qedi_conn->wait_queue); 837 } 838 } else if (qedi_conn->cmd_cleanup_req > 0) { 839 spin_lock_bh(&conn->session->back_lock); 840 qedi_get_proto_itt(qedi, cqe->itid, &ptmp_itt); 841 protoitt = build_itt(ptmp_itt, conn->session->age); 842 task = iscsi_itt_to_task(conn, protoitt); 843 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 844 "cleanup io itid=0x%x, protoitt=0x%x, cmd_cleanup_cmpl=%d, cid=0x%x\n", 845 cqe->itid, protoitt, qedi_conn->cmd_cleanup_cmpl, 846 qedi_conn->iscsi_conn_id); 847 848 spin_unlock_bh(&conn->session->back_lock); 849 if (!task) { 850 QEDI_NOTICE(&qedi->dbg_ctx, 851 "task is null, itid=0x%x, cid=0x%x\n", 852 cqe->itid, qedi_conn->iscsi_conn_id); 853 return; 854 } 855 qedi_conn->cmd_cleanup_cmpl++; 856 wake_up(&qedi_conn->wait_queue); 857 cmd_new = task->dd_data; 858 859 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_TID, 860 "Freeing tid=0x%x for cid=0x%x\n", 861 cqe->itid, qedi_conn->iscsi_conn_id); 862 qedi_clear_task_idx(qedi_conn->qedi, cqe->itid); 863 864 } else { 865 qedi_get_proto_itt(qedi, cqe->itid, &ptmp_itt); 866 protoitt = build_itt(ptmp_itt, conn->session->age); 867 task = iscsi_itt_to_task(conn, protoitt); 868 QEDI_ERR(&qedi->dbg_ctx, 869 "Delayed or untracked cleanup response, itt=0x%x, tid=0x%x, cid=0x%x, task=%p\n", 870 protoitt, cqe->itid, qedi_conn->iscsi_conn_id, task); 871 WARN_ON(1); 872 } 873 } 874 875 void qedi_fp_process_cqes(struct qedi_work *work) 876 { 877 struct qedi_ctx *qedi = work->qedi; 878 union iscsi_cqe *cqe = &work->cqe; 879 struct iscsi_task *task = NULL; 880 struct iscsi_nopout *nopout_hdr; 881 struct qedi_conn *q_conn; 882 struct iscsi_conn *conn; 883 struct qedi_cmd *qedi_cmd; 884 u32 comp_type; 885 u32 iscsi_cid; 886 u32 hdr_opcode; 887 u16 que_idx = work->que_idx; 888 u8 cqe_err_bits = 0; 889 890 comp_type = cqe->cqe_common.cqe_type; 891 hdr_opcode = cqe->cqe_common.iscsi_hdr.common.hdr_first_byte; 892 cqe_err_bits = 893 cqe->cqe_common.error_bitmap.error_bits.cqe_error_status_bits; 894 895 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, 896 "fw_cid=0x%x, cqe type=0x%x, opcode=0x%x\n", 897 cqe->cqe_common.conn_id, comp_type, hdr_opcode); 898 899 if (comp_type >= MAX_ISCSI_CQES_TYPE) { 900 QEDI_WARN(&qedi->dbg_ctx, "Invalid CqE type\n"); 901 return; 902 } 903 904 iscsi_cid = cqe->cqe_common.conn_id; 905 q_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid]; 906 if (!q_conn) { 907 QEDI_WARN(&qedi->dbg_ctx, 908 "Session no longer exists for cid=0x%x!!\n", 909 iscsi_cid); 910 return; 911 } 912 913 conn = q_conn->cls_conn->dd_data; 914 915 if (unlikely(cqe_err_bits && 916 GET_FIELD(cqe_err_bits, 917 CQE_ERROR_BITMAP_DATA_DIGEST_ERR))) { 918 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST); 919 return; 920 } 921 922 switch (comp_type) { 923 case ISCSI_CQE_TYPE_SOLICITED: 924 case ISCSI_CQE_TYPE_SOLICITED_WITH_SENSE: 925 qedi_cmd = container_of(work, struct qedi_cmd, cqe_work); 926 task = qedi_cmd->task; 927 if (!task) { 928 QEDI_WARN(&qedi->dbg_ctx, "task is NULL\n"); 929 return; 930 } 931 932 /* Process NOPIN local completion */ 933 nopout_hdr = (struct iscsi_nopout *)task->hdr; 934 if ((nopout_hdr->itt == RESERVED_ITT) && 935 (cqe->cqe_solicited.itid != (u16)RESERVED_ITT)) { 936 qedi_process_nopin_local_cmpl(qedi, &cqe->cqe_solicited, 937 task, q_conn); 938 } else { 939 cqe->cqe_solicited.itid = 940 qedi_get_itt(cqe->cqe_solicited); 941 /* Process other solicited responses */ 942 qedi_mtask_completion(qedi, cqe, task, q_conn, que_idx); 943 } 944 break; 945 case ISCSI_CQE_TYPE_UNSOLICITED: 946 switch (hdr_opcode) { 947 case ISCSI_OPCODE_NOP_IN: 948 qedi_process_nopin_mesg(qedi, cqe, task, q_conn, 949 que_idx); 950 break; 951 case ISCSI_OPCODE_ASYNC_MSG: 952 qedi_process_async_mesg(qedi, cqe, task, q_conn, 953 que_idx); 954 break; 955 case ISCSI_OPCODE_REJECT: 956 qedi_process_reject_mesg(qedi, cqe, task, q_conn, 957 que_idx); 958 break; 959 } 960 goto exit_fp_process; 961 case ISCSI_CQE_TYPE_DUMMY: 962 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, "Dummy CqE\n"); 963 goto exit_fp_process; 964 case ISCSI_CQE_TYPE_TASK_CLEANUP: 965 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, "CleanUp CqE\n"); 966 qedi_process_cmd_cleanup_resp(qedi, &cqe->cqe_solicited, task, 967 conn); 968 goto exit_fp_process; 969 default: 970 QEDI_ERR(&qedi->dbg_ctx, "Error cqe.\n"); 971 break; 972 } 973 974 exit_fp_process: 975 return; 976 } 977 978 static void qedi_add_to_sq(struct qedi_conn *qedi_conn, struct iscsi_task *task, 979 u16 tid, uint16_t ptu_invalidate, int is_cleanup) 980 { 981 struct iscsi_wqe *wqe; 982 struct iscsi_wqe_field *cont_field; 983 struct qedi_endpoint *ep; 984 struct scsi_cmnd *sc = task->sc; 985 struct iscsi_login_req *login_hdr; 986 struct qedi_cmd *cmd = task->dd_data; 987 988 login_hdr = (struct iscsi_login_req *)task->hdr; 989 ep = qedi_conn->ep; 990 wqe = &ep->sq[ep->sq_prod_idx]; 991 992 memset(wqe, 0, sizeof(*wqe)); 993 994 ep->sq_prod_idx++; 995 ep->fw_sq_prod_idx++; 996 if (ep->sq_prod_idx == QEDI_SQ_SIZE) 997 ep->sq_prod_idx = 0; 998 999 if (is_cleanup) { 1000 SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE, 1001 ISCSI_WQE_TYPE_TASK_CLEANUP); 1002 wqe->task_id = tid; 1003 return; 1004 } 1005 1006 if (ptu_invalidate) { 1007 SET_FIELD(wqe->flags, ISCSI_WQE_PTU_INVALIDATE, 1008 ISCSI_WQE_SET_PTU_INVALIDATE); 1009 } 1010 1011 cont_field = &wqe->cont_prevtid_union.cont_field; 1012 1013 switch (task->hdr->opcode & ISCSI_OPCODE_MASK) { 1014 case ISCSI_OP_LOGIN: 1015 case ISCSI_OP_TEXT: 1016 SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE, 1017 ISCSI_WQE_TYPE_MIDDLE_PATH); 1018 SET_FIELD(wqe->flags, ISCSI_WQE_NUM_FAST_SGES, 1019 1); 1020 cont_field->contlen_cdbsize_field = ntoh24(login_hdr->dlength); 1021 break; 1022 case ISCSI_OP_LOGOUT: 1023 case ISCSI_OP_NOOP_OUT: 1024 case ISCSI_OP_SCSI_TMFUNC: 1025 SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE, 1026 ISCSI_WQE_TYPE_NORMAL); 1027 break; 1028 default: 1029 if (!sc) 1030 break; 1031 1032 SET_FIELD(wqe->flags, ISCSI_WQE_WQE_TYPE, 1033 ISCSI_WQE_TYPE_NORMAL); 1034 cont_field->contlen_cdbsize_field = 1035 (sc->sc_data_direction == DMA_TO_DEVICE) ? 1036 scsi_bufflen(sc) : 0; 1037 if (cmd->use_slowpath) 1038 SET_FIELD(wqe->flags, ISCSI_WQE_NUM_FAST_SGES, 0); 1039 else 1040 SET_FIELD(wqe->flags, ISCSI_WQE_NUM_FAST_SGES, 1041 (sc->sc_data_direction == 1042 DMA_TO_DEVICE) ? 1043 min((u16)QEDI_FAST_SGE_COUNT, 1044 (u16)cmd->io_tbl.sge_valid) : 0); 1045 break; 1046 } 1047 1048 wqe->task_id = tid; 1049 /* Make sure SQ data is coherent */ 1050 wmb(); 1051 } 1052 1053 static void qedi_ring_doorbell(struct qedi_conn *qedi_conn) 1054 { 1055 struct iscsi_db_data dbell = { 0 }; 1056 1057 dbell.agg_flags = 0; 1058 1059 dbell.params |= DB_DEST_XCM << ISCSI_DB_DATA_DEST_SHIFT; 1060 dbell.params |= DB_AGG_CMD_SET << ISCSI_DB_DATA_AGG_CMD_SHIFT; 1061 dbell.params |= 1062 DQ_XCM_ISCSI_SQ_PROD_CMD << ISCSI_DB_DATA_AGG_VAL_SEL_SHIFT; 1063 1064 dbell.sq_prod = qedi_conn->ep->fw_sq_prod_idx; 1065 writel(*(u32 *)&dbell, qedi_conn->ep->p_doorbell); 1066 1067 /* Make sure fw write idx is coherent, and include both memory barriers 1068 * as a failsafe as for some architectures the call is the same but on 1069 * others they are two different assembly operations. 1070 */ 1071 wmb(); 1072 mmiowb(); 1073 QEDI_INFO(&qedi_conn->qedi->dbg_ctx, QEDI_LOG_MP_REQ, 1074 "prod_idx=0x%x, fw_prod_idx=0x%x, cid=0x%x\n", 1075 qedi_conn->ep->sq_prod_idx, qedi_conn->ep->fw_sq_prod_idx, 1076 qedi_conn->iscsi_conn_id); 1077 } 1078 1079 int qedi_send_iscsi_login(struct qedi_conn *qedi_conn, 1080 struct iscsi_task *task) 1081 { 1082 struct qedi_ctx *qedi = qedi_conn->qedi; 1083 struct iscsi_task_context *fw_task_ctx; 1084 struct iscsi_login_req *login_hdr; 1085 struct iscsi_login_req_hdr *fw_login_req = NULL; 1086 struct iscsi_cached_sge_ctx *cached_sge = NULL; 1087 struct iscsi_sge *single_sge = NULL; 1088 struct iscsi_sge *req_sge = NULL; 1089 struct iscsi_sge *resp_sge = NULL; 1090 struct qedi_cmd *qedi_cmd; 1091 s16 ptu_invalidate = 0; 1092 s16 tid = 0; 1093 1094 req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl; 1095 resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl; 1096 qedi_cmd = (struct qedi_cmd *)task->dd_data; 1097 login_hdr = (struct iscsi_login_req *)task->hdr; 1098 1099 tid = qedi_get_task_idx(qedi); 1100 if (tid == -1) 1101 return -ENOMEM; 1102 1103 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid); 1104 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context)); 1105 1106 qedi_cmd->task_id = tid; 1107 1108 /* Ystorm context */ 1109 fw_login_req = &fw_task_ctx->ystorm_st_context.pdu_hdr.login_req; 1110 fw_login_req->opcode = login_hdr->opcode; 1111 fw_login_req->version_min = login_hdr->min_version; 1112 fw_login_req->version_max = login_hdr->max_version; 1113 fw_login_req->flags_attr = login_hdr->flags; 1114 fw_login_req->isid_tabc = *((u16 *)login_hdr->isid + 2); 1115 fw_login_req->isid_d = *((u32 *)login_hdr->isid); 1116 fw_login_req->tsih = login_hdr->tsih; 1117 qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd); 1118 fw_login_req->itt = qedi_set_itt(tid, get_itt(task->itt)); 1119 fw_login_req->cid = qedi_conn->iscsi_conn_id; 1120 fw_login_req->cmd_sn = be32_to_cpu(login_hdr->cmdsn); 1121 fw_login_req->exp_stat_sn = be32_to_cpu(login_hdr->exp_statsn); 1122 fw_login_req->exp_stat_sn = 0; 1123 1124 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) { 1125 ptu_invalidate = 1; 1126 qedi->tid_reuse_count[tid] = 0; 1127 } 1128 1129 fw_task_ctx->ystorm_st_context.state.reuse_count = 1130 qedi->tid_reuse_count[tid]; 1131 fw_task_ctx->mstorm_st_context.reuse_count = 1132 qedi->tid_reuse_count[tid]++; 1133 cached_sge = 1134 &fw_task_ctx->ystorm_st_context.state.sgl_ctx_union.cached_sge; 1135 cached_sge->sge.sge_len = req_sge->sge_len; 1136 cached_sge->sge.sge_addr.lo = (u32)(qedi_conn->gen_pdu.req_dma_addr); 1137 cached_sge->sge.sge_addr.hi = 1138 (u32)((u64)qedi_conn->gen_pdu.req_dma_addr >> 32); 1139 1140 /* Mstorm context */ 1141 single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge; 1142 fw_task_ctx->mstorm_st_context.task_type = 0x2; 1143 fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id; 1144 single_sge->sge_addr.lo = resp_sge->sge_addr.lo; 1145 single_sge->sge_addr.hi = resp_sge->sge_addr.hi; 1146 single_sge->sge_len = resp_sge->sge_len; 1147 1148 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 1149 ISCSI_MFLAGS_SINGLE_SGE, 1); 1150 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 1151 ISCSI_MFLAGS_SLOW_IO, 0); 1152 fw_task_ctx->mstorm_st_context.sgl_size = 1; 1153 fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len; 1154 1155 /* Ustorm context */ 1156 fw_task_ctx->ustorm_st_context.rem_rcv_len = resp_sge->sge_len; 1157 fw_task_ctx->ustorm_st_context.exp_data_transfer_len = 1158 ntoh24(login_hdr->dlength); 1159 fw_task_ctx->ustorm_st_context.exp_data_sn = 0; 1160 fw_task_ctx->ustorm_st_context.cq_rss_number = 0; 1161 fw_task_ctx->ustorm_st_context.task_type = 0x2; 1162 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id; 1163 fw_task_ctx->ustorm_ag_context.exp_data_acked = 1164 ntoh24(login_hdr->dlength); 1165 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1, 1166 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1); 1167 SET_FIELD(fw_task_ctx->ustorm_st_context.flags, 1168 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0); 1169 1170 spin_lock(&qedi_conn->list_lock); 1171 list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list); 1172 qedi_cmd->io_cmd_in_list = true; 1173 qedi_conn->active_cmd_count++; 1174 spin_unlock(&qedi_conn->list_lock); 1175 1176 qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false); 1177 qedi_ring_doorbell(qedi_conn); 1178 return 0; 1179 } 1180 1181 int qedi_send_iscsi_logout(struct qedi_conn *qedi_conn, 1182 struct iscsi_task *task) 1183 { 1184 struct qedi_ctx *qedi = qedi_conn->qedi; 1185 struct iscsi_logout_req_hdr *fw_logout_req = NULL; 1186 struct iscsi_task_context *fw_task_ctx = NULL; 1187 struct iscsi_logout *logout_hdr = NULL; 1188 struct qedi_cmd *qedi_cmd = NULL; 1189 s16 tid = 0; 1190 s16 ptu_invalidate = 0; 1191 1192 qedi_cmd = (struct qedi_cmd *)task->dd_data; 1193 logout_hdr = (struct iscsi_logout *)task->hdr; 1194 1195 tid = qedi_get_task_idx(qedi); 1196 if (tid == -1) 1197 return -ENOMEM; 1198 1199 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid); 1200 1201 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context)); 1202 qedi_cmd->task_id = tid; 1203 1204 /* Ystorm context */ 1205 fw_logout_req = &fw_task_ctx->ystorm_st_context.pdu_hdr.logout_req; 1206 fw_logout_req->opcode = ISCSI_OPCODE_LOGOUT_REQUEST; 1207 fw_logout_req->reason_code = 0x80 | logout_hdr->flags; 1208 qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd); 1209 fw_logout_req->itt = qedi_set_itt(tid, get_itt(task->itt)); 1210 fw_logout_req->exp_stat_sn = be32_to_cpu(logout_hdr->exp_statsn); 1211 fw_logout_req->cmd_sn = be32_to_cpu(logout_hdr->cmdsn); 1212 1213 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) { 1214 ptu_invalidate = 1; 1215 qedi->tid_reuse_count[tid] = 0; 1216 } 1217 fw_task_ctx->ystorm_st_context.state.reuse_count = 1218 qedi->tid_reuse_count[tid]; 1219 fw_task_ctx->mstorm_st_context.reuse_count = 1220 qedi->tid_reuse_count[tid]++; 1221 fw_logout_req->cid = qedi_conn->iscsi_conn_id; 1222 fw_task_ctx->ystorm_st_context.state.buffer_offset[0] = 0; 1223 1224 /* Mstorm context */ 1225 fw_task_ctx->mstorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH; 1226 fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id; 1227 1228 /* Ustorm context */ 1229 fw_task_ctx->ustorm_st_context.rem_rcv_len = 0; 1230 fw_task_ctx->ustorm_st_context.exp_data_transfer_len = 0; 1231 fw_task_ctx->ustorm_st_context.exp_data_sn = 0; 1232 fw_task_ctx->ustorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH; 1233 fw_task_ctx->ustorm_st_context.cq_rss_number = 0; 1234 1235 SET_FIELD(fw_task_ctx->ustorm_st_context.flags, 1236 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0); 1237 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map, 1238 ISCSI_REG1_NUM_FAST_SGES, 0); 1239 1240 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id; 1241 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1, 1242 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1); 1243 1244 spin_lock(&qedi_conn->list_lock); 1245 list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list); 1246 qedi_cmd->io_cmd_in_list = true; 1247 qedi_conn->active_cmd_count++; 1248 spin_unlock(&qedi_conn->list_lock); 1249 1250 qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false); 1251 qedi_ring_doorbell(qedi_conn); 1252 1253 return 0; 1254 } 1255 1256 int qedi_cleanup_all_io(struct qedi_ctx *qedi, struct qedi_conn *qedi_conn, 1257 struct iscsi_task *task, bool in_recovery) 1258 { 1259 int rval; 1260 struct iscsi_task *ctask; 1261 struct qedi_cmd *cmd, *cmd_tmp; 1262 struct iscsi_tm *tmf_hdr; 1263 unsigned int lun = 0; 1264 bool lun_reset = false; 1265 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 1266 struct iscsi_session *session = conn->session; 1267 1268 /* From recovery, task is NULL or from tmf resp valid task */ 1269 if (task) { 1270 tmf_hdr = (struct iscsi_tm *)task->hdr; 1271 1272 if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) == 1273 ISCSI_TM_FUNC_LOGICAL_UNIT_RESET) { 1274 lun_reset = true; 1275 lun = scsilun_to_int(&tmf_hdr->lun); 1276 } 1277 } 1278 1279 qedi_conn->cmd_cleanup_req = 0; 1280 qedi_conn->cmd_cleanup_cmpl = 0; 1281 1282 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 1283 "active_cmd_count=%d, cid=0x%x, in_recovery=%d, lun_reset=%d\n", 1284 qedi_conn->active_cmd_count, qedi_conn->iscsi_conn_id, 1285 in_recovery, lun_reset); 1286 1287 if (lun_reset) 1288 spin_lock_bh(&session->back_lock); 1289 1290 spin_lock(&qedi_conn->list_lock); 1291 1292 list_for_each_entry_safe(cmd, cmd_tmp, &qedi_conn->active_cmd_list, 1293 io_cmd) { 1294 ctask = cmd->task; 1295 if (ctask == task) 1296 continue; 1297 1298 if (lun_reset) { 1299 if (cmd->scsi_cmd && cmd->scsi_cmd->device) { 1300 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 1301 "tid=0x%x itt=0x%x scsi_cmd_ptr=%p device=%p task_state=%d cmd_state=0%x cid=0x%x\n", 1302 cmd->task_id, get_itt(ctask->itt), 1303 cmd->scsi_cmd, cmd->scsi_cmd->device, 1304 ctask->state, cmd->state, 1305 qedi_conn->iscsi_conn_id); 1306 if (cmd->scsi_cmd->device->lun != lun) 1307 continue; 1308 } 1309 } 1310 qedi_conn->cmd_cleanup_req++; 1311 qedi_iscsi_cleanup_task(ctask, true); 1312 1313 list_del_init(&cmd->io_cmd); 1314 qedi_conn->active_cmd_count--; 1315 QEDI_WARN(&qedi->dbg_ctx, 1316 "Deleted active cmd list node io_cmd=%p, cid=0x%x\n", 1317 &cmd->io_cmd, qedi_conn->iscsi_conn_id); 1318 } 1319 1320 spin_unlock(&qedi_conn->list_lock); 1321 1322 if (lun_reset) 1323 spin_unlock_bh(&session->back_lock); 1324 1325 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 1326 "cmd_cleanup_req=%d, cid=0x%x\n", 1327 qedi_conn->cmd_cleanup_req, 1328 qedi_conn->iscsi_conn_id); 1329 1330 rval = wait_event_interruptible_timeout(qedi_conn->wait_queue, 1331 ((qedi_conn->cmd_cleanup_req == 1332 qedi_conn->cmd_cleanup_cmpl) || 1333 qedi_conn->ep), 1334 5 * HZ); 1335 if (rval) { 1336 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 1337 "i/o cmd_cleanup_req=%d, equal to cmd_cleanup_cmpl=%d, cid=0x%x\n", 1338 qedi_conn->cmd_cleanup_req, 1339 qedi_conn->cmd_cleanup_cmpl, 1340 qedi_conn->iscsi_conn_id); 1341 1342 return 0; 1343 } 1344 1345 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 1346 "i/o cmd_cleanup_req=%d, not equal to cmd_cleanup_cmpl=%d, cid=0x%x\n", 1347 qedi_conn->cmd_cleanup_req, 1348 qedi_conn->cmd_cleanup_cmpl, 1349 qedi_conn->iscsi_conn_id); 1350 1351 iscsi_host_for_each_session(qedi->shost, 1352 qedi_mark_device_missing); 1353 qedi_ops->common->drain(qedi->cdev); 1354 1355 /* Enable IOs for all other sessions except current.*/ 1356 if (!wait_event_interruptible_timeout(qedi_conn->wait_queue, 1357 (qedi_conn->cmd_cleanup_req == 1358 qedi_conn->cmd_cleanup_cmpl), 1359 5 * HZ)) { 1360 iscsi_host_for_each_session(qedi->shost, 1361 qedi_mark_device_available); 1362 return -1; 1363 } 1364 1365 iscsi_host_for_each_session(qedi->shost, 1366 qedi_mark_device_available); 1367 1368 return 0; 1369 } 1370 1371 void qedi_clearsq(struct qedi_ctx *qedi, struct qedi_conn *qedi_conn, 1372 struct iscsi_task *task) 1373 { 1374 struct qedi_endpoint *qedi_ep; 1375 int rval; 1376 1377 qedi_ep = qedi_conn->ep; 1378 qedi_conn->cmd_cleanup_req = 0; 1379 qedi_conn->cmd_cleanup_cmpl = 0; 1380 1381 if (!qedi_ep) { 1382 QEDI_WARN(&qedi->dbg_ctx, 1383 "Cannot proceed, ep already disconnected, cid=0x%x\n", 1384 qedi_conn->iscsi_conn_id); 1385 return; 1386 } 1387 1388 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, 1389 "Clearing SQ for cid=0x%x, conn=%p, ep=%p\n", 1390 qedi_conn->iscsi_conn_id, qedi_conn, qedi_ep); 1391 1392 qedi_ops->clear_sq(qedi->cdev, qedi_ep->handle); 1393 1394 rval = qedi_cleanup_all_io(qedi, qedi_conn, task, true); 1395 if (rval) { 1396 QEDI_ERR(&qedi->dbg_ctx, 1397 "fatal error, need hard reset, cid=0x%x\n", 1398 qedi_conn->iscsi_conn_id); 1399 WARN_ON(1); 1400 } 1401 } 1402 1403 static int qedi_wait_for_cleanup_request(struct qedi_ctx *qedi, 1404 struct qedi_conn *qedi_conn, 1405 struct iscsi_task *task, 1406 struct qedi_cmd *qedi_cmd, 1407 struct qedi_work_map *list_work) 1408 { 1409 struct qedi_cmd *cmd = (struct qedi_cmd *)task->dd_data; 1410 int wait; 1411 1412 wait = wait_event_interruptible_timeout(qedi_conn->wait_queue, 1413 ((qedi_cmd->state == 1414 CLEANUP_RECV) || 1415 ((qedi_cmd->type == TYPEIO) && 1416 (cmd->state == 1417 RESPONSE_RECEIVED))), 1418 5 * HZ); 1419 if (!wait) { 1420 qedi_cmd->state = CLEANUP_WAIT_FAILED; 1421 1422 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 1423 "Cleanup timedout tid=0x%x, issue connection recovery, cid=0x%x\n", 1424 cmd->task_id, qedi_conn->iscsi_conn_id); 1425 1426 return -1; 1427 } 1428 return 0; 1429 } 1430 1431 static void qedi_tmf_work(struct work_struct *work) 1432 { 1433 struct qedi_cmd *qedi_cmd = 1434 container_of(work, struct qedi_cmd, tmf_work); 1435 struct qedi_conn *qedi_conn = qedi_cmd->conn; 1436 struct qedi_ctx *qedi = qedi_conn->qedi; 1437 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 1438 struct iscsi_cls_session *cls_sess; 1439 struct qedi_work_map *list_work = NULL; 1440 struct iscsi_task *mtask; 1441 struct qedi_cmd *cmd; 1442 struct iscsi_task *ctask; 1443 struct iscsi_tm *tmf_hdr; 1444 s16 rval = 0; 1445 s16 tid = 0; 1446 1447 mtask = qedi_cmd->task; 1448 tmf_hdr = (struct iscsi_tm *)mtask->hdr; 1449 cls_sess = iscsi_conn_to_session(qedi_conn->cls_conn); 1450 set_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags); 1451 1452 ctask = iscsi_itt_to_task(conn, tmf_hdr->rtt); 1453 if (!ctask || !ctask->sc) { 1454 QEDI_ERR(&qedi->dbg_ctx, "Task already completed\n"); 1455 goto abort_ret; 1456 } 1457 1458 cmd = (struct qedi_cmd *)ctask->dd_data; 1459 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, 1460 "Abort tmf rtt=0x%x, cmd itt=0x%x, cmd tid=0x%x, cid=0x%x\n", 1461 get_itt(tmf_hdr->rtt), get_itt(ctask->itt), cmd->task_id, 1462 qedi_conn->iscsi_conn_id); 1463 1464 if (qedi_do_not_recover) { 1465 QEDI_ERR(&qedi->dbg_ctx, "DONT SEND CLEANUP/ABORT %d\n", 1466 qedi_do_not_recover); 1467 goto abort_ret; 1468 } 1469 1470 list_work = kzalloc(sizeof(*list_work), GFP_ATOMIC); 1471 if (!list_work) { 1472 QEDI_ERR(&qedi->dbg_ctx, "Memory alloction failed\n"); 1473 goto abort_ret; 1474 } 1475 1476 qedi_cmd->type = TYPEIO; 1477 list_work->qedi_cmd = qedi_cmd; 1478 list_work->rtid = cmd->task_id; 1479 list_work->state = QEDI_WORK_SCHEDULED; 1480 qedi_cmd->list_tmf_work = list_work; 1481 1482 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 1483 "Queue tmf work=%p, list node=%p, cid=0x%x, tmf flags=0x%x\n", 1484 list_work->ptr_tmf_work, list_work, qedi_conn->iscsi_conn_id, 1485 tmf_hdr->flags); 1486 1487 spin_lock_bh(&qedi_conn->tmf_work_lock); 1488 list_add_tail(&list_work->list, &qedi_conn->tmf_work_list); 1489 spin_unlock_bh(&qedi_conn->tmf_work_lock); 1490 1491 qedi_iscsi_cleanup_task(ctask, false); 1492 1493 rval = qedi_wait_for_cleanup_request(qedi, qedi_conn, ctask, qedi_cmd, 1494 list_work); 1495 if (rval == -1) { 1496 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, 1497 "FW cleanup got escalated, cid=0x%x\n", 1498 qedi_conn->iscsi_conn_id); 1499 goto ldel_exit; 1500 } 1501 1502 tid = qedi_get_task_idx(qedi); 1503 if (tid == -1) { 1504 QEDI_ERR(&qedi->dbg_ctx, "Invalid tid, cid=0x%x\n", 1505 qedi_conn->iscsi_conn_id); 1506 goto ldel_exit; 1507 } 1508 1509 qedi_cmd->task_id = tid; 1510 qedi_send_iscsi_tmf(qedi_conn, qedi_cmd->task); 1511 1512 abort_ret: 1513 clear_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags); 1514 return; 1515 1516 ldel_exit: 1517 spin_lock_bh(&qedi_conn->tmf_work_lock); 1518 if (!qedi_cmd->list_tmf_work) { 1519 list_del_init(&list_work->list); 1520 qedi_cmd->list_tmf_work = NULL; 1521 kfree(list_work); 1522 } 1523 spin_unlock_bh(&qedi_conn->tmf_work_lock); 1524 1525 spin_lock(&qedi_conn->list_lock); 1526 list_del_init(&cmd->io_cmd); 1527 qedi_conn->active_cmd_count--; 1528 spin_unlock(&qedi_conn->list_lock); 1529 1530 clear_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags); 1531 } 1532 1533 static int qedi_send_iscsi_tmf(struct qedi_conn *qedi_conn, 1534 struct iscsi_task *mtask) 1535 { 1536 struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data; 1537 struct qedi_ctx *qedi = qedi_conn->qedi; 1538 struct iscsi_task_context *fw_task_ctx; 1539 struct iscsi_tmf_request_hdr *fw_tmf_request; 1540 struct iscsi_sge *single_sge; 1541 struct qedi_cmd *qedi_cmd; 1542 struct qedi_cmd *cmd; 1543 struct iscsi_task *ctask; 1544 struct iscsi_tm *tmf_hdr; 1545 struct iscsi_sge *req_sge; 1546 struct iscsi_sge *resp_sge; 1547 u32 lun[2]; 1548 s16 tid = 0, ptu_invalidate = 0; 1549 1550 req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl; 1551 resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl; 1552 qedi_cmd = (struct qedi_cmd *)mtask->dd_data; 1553 tmf_hdr = (struct iscsi_tm *)mtask->hdr; 1554 1555 tid = qedi_cmd->task_id; 1556 qedi_update_itt_map(qedi, tid, mtask->itt, qedi_cmd); 1557 1558 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid); 1559 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context)); 1560 1561 fw_tmf_request = &fw_task_ctx->ystorm_st_context.pdu_hdr.tmf_request; 1562 fw_tmf_request->itt = qedi_set_itt(tid, get_itt(mtask->itt)); 1563 fw_tmf_request->cmd_sn = be32_to_cpu(tmf_hdr->cmdsn); 1564 1565 memcpy(lun, &tmf_hdr->lun, sizeof(struct scsi_lun)); 1566 fw_tmf_request->lun.lo = be32_to_cpu(lun[0]); 1567 fw_tmf_request->lun.hi = be32_to_cpu(lun[1]); 1568 1569 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) { 1570 ptu_invalidate = 1; 1571 qedi->tid_reuse_count[tid] = 0; 1572 } 1573 fw_task_ctx->ystorm_st_context.state.reuse_count = 1574 qedi->tid_reuse_count[tid]; 1575 fw_task_ctx->mstorm_st_context.reuse_count = 1576 qedi->tid_reuse_count[tid]++; 1577 1578 if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) == 1579 ISCSI_TM_FUNC_ABORT_TASK) { 1580 ctask = iscsi_itt_to_task(conn, tmf_hdr->rtt); 1581 if (!ctask || !ctask->sc) { 1582 QEDI_ERR(&qedi->dbg_ctx, 1583 "Could not get reference task\n"); 1584 return 0; 1585 } 1586 cmd = (struct qedi_cmd *)ctask->dd_data; 1587 fw_tmf_request->rtt = 1588 qedi_set_itt(cmd->task_id, 1589 get_itt(tmf_hdr->rtt)); 1590 } else { 1591 fw_tmf_request->rtt = ISCSI_RESERVED_TAG; 1592 } 1593 1594 fw_tmf_request->opcode = tmf_hdr->opcode; 1595 fw_tmf_request->function = tmf_hdr->flags; 1596 fw_tmf_request->hdr_second_dword = ntoh24(tmf_hdr->dlength); 1597 fw_tmf_request->ref_cmd_sn = be32_to_cpu(tmf_hdr->refcmdsn); 1598 1599 single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge; 1600 fw_task_ctx->mstorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH; 1601 fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id; 1602 single_sge->sge_addr.lo = resp_sge->sge_addr.lo; 1603 single_sge->sge_addr.hi = resp_sge->sge_addr.hi; 1604 single_sge->sge_len = resp_sge->sge_len; 1605 1606 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 1607 ISCSI_MFLAGS_SINGLE_SGE, 1); 1608 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 1609 ISCSI_MFLAGS_SLOW_IO, 0); 1610 fw_task_ctx->mstorm_st_context.sgl_size = 1; 1611 fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len; 1612 1613 /* Ustorm context */ 1614 fw_task_ctx->ustorm_st_context.rem_rcv_len = 0; 1615 fw_task_ctx->ustorm_st_context.exp_data_transfer_len = 0; 1616 fw_task_ctx->ustorm_st_context.exp_data_sn = 0; 1617 fw_task_ctx->ustorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH; 1618 fw_task_ctx->ustorm_st_context.cq_rss_number = 0; 1619 1620 SET_FIELD(fw_task_ctx->ustorm_st_context.flags, 1621 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0); 1622 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map, 1623 ISCSI_REG1_NUM_FAST_SGES, 0); 1624 1625 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id; 1626 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1, 1627 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1); 1628 fw_task_ctx->ustorm_st_context.lun.lo = be32_to_cpu(lun[0]); 1629 fw_task_ctx->ustorm_st_context.lun.hi = be32_to_cpu(lun[1]); 1630 1631 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 1632 "Add TMF to SQ, tmf tid=0x%x, itt=0x%x, cid=0x%x\n", 1633 tid, mtask->itt, qedi_conn->iscsi_conn_id); 1634 1635 spin_lock(&qedi_conn->list_lock); 1636 list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list); 1637 qedi_cmd->io_cmd_in_list = true; 1638 qedi_conn->active_cmd_count++; 1639 spin_unlock(&qedi_conn->list_lock); 1640 1641 qedi_add_to_sq(qedi_conn, mtask, tid, ptu_invalidate, false); 1642 qedi_ring_doorbell(qedi_conn); 1643 return 0; 1644 } 1645 1646 int qedi_iscsi_abort_work(struct qedi_conn *qedi_conn, 1647 struct iscsi_task *mtask) 1648 { 1649 struct qedi_ctx *qedi = qedi_conn->qedi; 1650 struct iscsi_tm *tmf_hdr; 1651 struct qedi_cmd *qedi_cmd = (struct qedi_cmd *)mtask->dd_data; 1652 s16 tid = 0; 1653 1654 tmf_hdr = (struct iscsi_tm *)mtask->hdr; 1655 qedi_cmd->task = mtask; 1656 1657 /* If abort task then schedule the work and return */ 1658 if ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) == 1659 ISCSI_TM_FUNC_ABORT_TASK) { 1660 qedi_cmd->state = CLEANUP_WAIT; 1661 INIT_WORK(&qedi_cmd->tmf_work, qedi_tmf_work); 1662 queue_work(qedi->tmf_thread, &qedi_cmd->tmf_work); 1663 1664 } else if (((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) == 1665 ISCSI_TM_FUNC_LOGICAL_UNIT_RESET) || 1666 ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) == 1667 ISCSI_TM_FUNC_TARGET_WARM_RESET) || 1668 ((tmf_hdr->flags & ISCSI_FLAG_TM_FUNC_MASK) == 1669 ISCSI_TM_FUNC_TARGET_COLD_RESET)) { 1670 tid = qedi_get_task_idx(qedi); 1671 if (tid == -1) { 1672 QEDI_ERR(&qedi->dbg_ctx, "Invalid tid, cid=0x%x\n", 1673 qedi_conn->iscsi_conn_id); 1674 return -1; 1675 } 1676 qedi_cmd->task_id = tid; 1677 1678 qedi_send_iscsi_tmf(qedi_conn, qedi_cmd->task); 1679 1680 } else { 1681 QEDI_ERR(&qedi->dbg_ctx, "Invalid tmf, cid=0x%x\n", 1682 qedi_conn->iscsi_conn_id); 1683 return -1; 1684 } 1685 1686 return 0; 1687 } 1688 1689 int qedi_send_iscsi_text(struct qedi_conn *qedi_conn, 1690 struct iscsi_task *task) 1691 { 1692 struct qedi_ctx *qedi = qedi_conn->qedi; 1693 struct iscsi_task_context *fw_task_ctx; 1694 struct iscsi_text_request_hdr *fw_text_request; 1695 struct iscsi_cached_sge_ctx *cached_sge; 1696 struct iscsi_sge *single_sge; 1697 struct qedi_cmd *qedi_cmd; 1698 /* For 6.5 hdr iscsi_hdr */ 1699 struct iscsi_text *text_hdr; 1700 struct iscsi_sge *req_sge; 1701 struct iscsi_sge *resp_sge; 1702 s16 ptu_invalidate = 0; 1703 s16 tid = 0; 1704 1705 req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl; 1706 resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl; 1707 qedi_cmd = (struct qedi_cmd *)task->dd_data; 1708 text_hdr = (struct iscsi_text *)task->hdr; 1709 1710 tid = qedi_get_task_idx(qedi); 1711 if (tid == -1) 1712 return -ENOMEM; 1713 1714 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid); 1715 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context)); 1716 1717 qedi_cmd->task_id = tid; 1718 1719 /* Ystorm context */ 1720 fw_text_request = 1721 &fw_task_ctx->ystorm_st_context.pdu_hdr.text_request; 1722 fw_text_request->opcode = text_hdr->opcode; 1723 fw_text_request->flags_attr = text_hdr->flags; 1724 1725 qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd); 1726 fw_text_request->itt = qedi_set_itt(tid, get_itt(task->itt)); 1727 fw_text_request->ttt = text_hdr->ttt; 1728 fw_text_request->cmd_sn = be32_to_cpu(text_hdr->cmdsn); 1729 fw_text_request->exp_stat_sn = be32_to_cpu(text_hdr->exp_statsn); 1730 fw_text_request->hdr_second_dword = ntoh24(text_hdr->dlength); 1731 1732 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) { 1733 ptu_invalidate = 1; 1734 qedi->tid_reuse_count[tid] = 0; 1735 } 1736 fw_task_ctx->ystorm_st_context.state.reuse_count = 1737 qedi->tid_reuse_count[tid]; 1738 fw_task_ctx->mstorm_st_context.reuse_count = 1739 qedi->tid_reuse_count[tid]++; 1740 1741 cached_sge = 1742 &fw_task_ctx->ystorm_st_context.state.sgl_ctx_union.cached_sge; 1743 cached_sge->sge.sge_len = req_sge->sge_len; 1744 cached_sge->sge.sge_addr.lo = (u32)(qedi_conn->gen_pdu.req_dma_addr); 1745 cached_sge->sge.sge_addr.hi = 1746 (u32)((u64)qedi_conn->gen_pdu.req_dma_addr >> 32); 1747 1748 /* Mstorm context */ 1749 single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge; 1750 fw_task_ctx->mstorm_st_context.task_type = 0x2; 1751 fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id; 1752 single_sge->sge_addr.lo = resp_sge->sge_addr.lo; 1753 single_sge->sge_addr.hi = resp_sge->sge_addr.hi; 1754 single_sge->sge_len = resp_sge->sge_len; 1755 1756 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 1757 ISCSI_MFLAGS_SINGLE_SGE, 1); 1758 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 1759 ISCSI_MFLAGS_SLOW_IO, 0); 1760 fw_task_ctx->mstorm_st_context.sgl_size = 1; 1761 fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len; 1762 1763 /* Ustorm context */ 1764 fw_task_ctx->ustorm_ag_context.exp_data_acked = 1765 ntoh24(text_hdr->dlength); 1766 fw_task_ctx->ustorm_st_context.rem_rcv_len = resp_sge->sge_len; 1767 fw_task_ctx->ustorm_st_context.exp_data_transfer_len = 1768 ntoh24(text_hdr->dlength); 1769 fw_task_ctx->ustorm_st_context.exp_data_sn = 1770 be32_to_cpu(text_hdr->exp_statsn); 1771 fw_task_ctx->ustorm_st_context.cq_rss_number = 0; 1772 fw_task_ctx->ustorm_st_context.task_type = 0x2; 1773 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id; 1774 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1, 1775 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1); 1776 1777 /* Add command in active command list */ 1778 spin_lock(&qedi_conn->list_lock); 1779 list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list); 1780 qedi_cmd->io_cmd_in_list = true; 1781 qedi_conn->active_cmd_count++; 1782 spin_unlock(&qedi_conn->list_lock); 1783 1784 qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false); 1785 qedi_ring_doorbell(qedi_conn); 1786 1787 return 0; 1788 } 1789 1790 int qedi_send_iscsi_nopout(struct qedi_conn *qedi_conn, 1791 struct iscsi_task *task, 1792 char *datap, int data_len, int unsol) 1793 { 1794 struct qedi_ctx *qedi = qedi_conn->qedi; 1795 struct iscsi_task_context *fw_task_ctx; 1796 struct iscsi_nop_out_hdr *fw_nop_out; 1797 struct qedi_cmd *qedi_cmd; 1798 /* For 6.5 hdr iscsi_hdr */ 1799 struct iscsi_nopout *nopout_hdr; 1800 struct iscsi_cached_sge_ctx *cached_sge; 1801 struct iscsi_sge *single_sge; 1802 struct iscsi_sge *req_sge; 1803 struct iscsi_sge *resp_sge; 1804 u32 lun[2]; 1805 s16 ptu_invalidate = 0; 1806 s16 tid = 0; 1807 1808 req_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.req_bd_tbl; 1809 resp_sge = (struct iscsi_sge *)qedi_conn->gen_pdu.resp_bd_tbl; 1810 qedi_cmd = (struct qedi_cmd *)task->dd_data; 1811 nopout_hdr = (struct iscsi_nopout *)task->hdr; 1812 1813 tid = qedi_get_task_idx(qedi); 1814 if (tid == -1) { 1815 QEDI_WARN(&qedi->dbg_ctx, "Invalid tid\n"); 1816 return -ENOMEM; 1817 } 1818 1819 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid); 1820 1821 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context)); 1822 qedi_cmd->task_id = tid; 1823 1824 /* Ystorm context */ 1825 fw_nop_out = &fw_task_ctx->ystorm_st_context.pdu_hdr.nop_out; 1826 SET_FIELD(fw_nop_out->flags_attr, ISCSI_NOP_OUT_HDR_CONST1, 1); 1827 SET_FIELD(fw_nop_out->flags_attr, ISCSI_NOP_OUT_HDR_RSRV, 0); 1828 1829 memcpy(lun, &nopout_hdr->lun, sizeof(struct scsi_lun)); 1830 fw_nop_out->lun.lo = be32_to_cpu(lun[0]); 1831 fw_nop_out->lun.hi = be32_to_cpu(lun[1]); 1832 1833 qedi_update_itt_map(qedi, tid, task->itt, qedi_cmd); 1834 1835 if (nopout_hdr->ttt != ISCSI_TTT_ALL_ONES) { 1836 fw_nop_out->itt = be32_to_cpu(nopout_hdr->itt); 1837 fw_nop_out->ttt = be32_to_cpu(nopout_hdr->ttt); 1838 fw_task_ctx->ystorm_st_context.state.buffer_offset[0] = 0; 1839 fw_task_ctx->ystorm_st_context.state.local_comp = 1; 1840 SET_FIELD(fw_task_ctx->ustorm_st_context.flags, 1841 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 1); 1842 } else { 1843 fw_nop_out->itt = qedi_set_itt(tid, get_itt(task->itt)); 1844 fw_nop_out->ttt = ISCSI_TTT_ALL_ONES; 1845 fw_task_ctx->ystorm_st_context.state.buffer_offset[0] = 0; 1846 1847 spin_lock(&qedi_conn->list_lock); 1848 list_add_tail(&qedi_cmd->io_cmd, &qedi_conn->active_cmd_list); 1849 qedi_cmd->io_cmd_in_list = true; 1850 qedi_conn->active_cmd_count++; 1851 spin_unlock(&qedi_conn->list_lock); 1852 } 1853 1854 fw_nop_out->opcode = ISCSI_OPCODE_NOP_OUT; 1855 fw_nop_out->cmd_sn = be32_to_cpu(nopout_hdr->cmdsn); 1856 fw_nop_out->exp_stat_sn = be32_to_cpu(nopout_hdr->exp_statsn); 1857 1858 cached_sge = 1859 &fw_task_ctx->ystorm_st_context.state.sgl_ctx_union.cached_sge; 1860 cached_sge->sge.sge_len = req_sge->sge_len; 1861 cached_sge->sge.sge_addr.lo = (u32)(qedi_conn->gen_pdu.req_dma_addr); 1862 cached_sge->sge.sge_addr.hi = 1863 (u32)((u64)qedi_conn->gen_pdu.req_dma_addr >> 32); 1864 1865 /* Mstorm context */ 1866 fw_task_ctx->mstorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH; 1867 fw_task_ctx->mstorm_ag_context.task_cid = (u16)qedi_conn->iscsi_conn_id; 1868 1869 single_sge = &fw_task_ctx->mstorm_st_context.sgl_union.single_sge; 1870 single_sge->sge_addr.lo = resp_sge->sge_addr.lo; 1871 single_sge->sge_addr.hi = resp_sge->sge_addr.hi; 1872 single_sge->sge_len = resp_sge->sge_len; 1873 fw_task_ctx->mstorm_st_context.rem_task_size = resp_sge->sge_len; 1874 1875 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) { 1876 ptu_invalidate = 1; 1877 qedi->tid_reuse_count[tid] = 0; 1878 } 1879 fw_task_ctx->ystorm_st_context.state.reuse_count = 1880 qedi->tid_reuse_count[tid]; 1881 fw_task_ctx->mstorm_st_context.reuse_count = 1882 qedi->tid_reuse_count[tid]++; 1883 /* Ustorm context */ 1884 fw_task_ctx->ustorm_st_context.rem_rcv_len = resp_sge->sge_len; 1885 fw_task_ctx->ustorm_st_context.exp_data_transfer_len = data_len; 1886 fw_task_ctx->ustorm_st_context.exp_data_sn = 0; 1887 fw_task_ctx->ustorm_st_context.task_type = ISCSI_TASK_TYPE_MIDPATH; 1888 fw_task_ctx->ustorm_st_context.cq_rss_number = 0; 1889 1890 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map, 1891 ISCSI_REG1_NUM_FAST_SGES, 0); 1892 1893 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id; 1894 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1, 1895 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1); 1896 1897 fw_task_ctx->ustorm_st_context.lun.lo = be32_to_cpu(lun[0]); 1898 fw_task_ctx->ustorm_st_context.lun.hi = be32_to_cpu(lun[1]); 1899 1900 qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false); 1901 qedi_ring_doorbell(qedi_conn); 1902 return 0; 1903 } 1904 1905 static int qedi_split_bd(struct qedi_cmd *cmd, u64 addr, int sg_len, 1906 int bd_index) 1907 { 1908 struct iscsi_sge *bd = cmd->io_tbl.sge_tbl; 1909 int frag_size, sg_frags; 1910 1911 sg_frags = 0; 1912 1913 while (sg_len) { 1914 if (addr % QEDI_PAGE_SIZE) 1915 frag_size = 1916 (QEDI_PAGE_SIZE - (addr % QEDI_PAGE_SIZE)); 1917 else 1918 frag_size = (sg_len > QEDI_BD_SPLIT_SZ) ? 0 : 1919 (sg_len % QEDI_BD_SPLIT_SZ); 1920 1921 if (frag_size == 0) 1922 frag_size = QEDI_BD_SPLIT_SZ; 1923 1924 bd[bd_index + sg_frags].sge_addr.lo = (addr & 0xffffffff); 1925 bd[bd_index + sg_frags].sge_addr.hi = (addr >> 32); 1926 bd[bd_index + sg_frags].sge_len = (u16)frag_size; 1927 QEDI_INFO(&cmd->conn->qedi->dbg_ctx, QEDI_LOG_IO, 1928 "split sge %d: addr=%llx, len=%x", 1929 (bd_index + sg_frags), addr, frag_size); 1930 1931 addr += (u64)frag_size; 1932 sg_frags++; 1933 sg_len -= frag_size; 1934 } 1935 return sg_frags; 1936 } 1937 1938 static int qedi_map_scsi_sg(struct qedi_ctx *qedi, struct qedi_cmd *cmd) 1939 { 1940 struct scsi_cmnd *sc = cmd->scsi_cmd; 1941 struct iscsi_sge *bd = cmd->io_tbl.sge_tbl; 1942 struct scatterlist *sg; 1943 int byte_count = 0; 1944 int bd_count = 0; 1945 int sg_count; 1946 int sg_len; 1947 int sg_frags; 1948 u64 addr, end_addr; 1949 int i; 1950 1951 WARN_ON(scsi_sg_count(sc) > QEDI_ISCSI_MAX_BDS_PER_CMD); 1952 1953 sg_count = dma_map_sg(&qedi->pdev->dev, scsi_sglist(sc), 1954 scsi_sg_count(sc), sc->sc_data_direction); 1955 1956 /* 1957 * New condition to send single SGE as cached-SGL. 1958 * Single SGE with length less than 64K. 1959 */ 1960 sg = scsi_sglist(sc); 1961 if ((sg_count == 1) && (sg_dma_len(sg) <= MAX_SGLEN_FOR_CACHESGL)) { 1962 sg_len = sg_dma_len(sg); 1963 addr = (u64)sg_dma_address(sg); 1964 1965 bd[bd_count].sge_addr.lo = (addr & 0xffffffff); 1966 bd[bd_count].sge_addr.hi = (addr >> 32); 1967 bd[bd_count].sge_len = (u16)sg_len; 1968 1969 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, 1970 "single-cashed-sgl: bd_count:%d addr=%llx, len=%x", 1971 sg_count, addr, sg_len); 1972 1973 return ++bd_count; 1974 } 1975 1976 scsi_for_each_sg(sc, sg, sg_count, i) { 1977 sg_len = sg_dma_len(sg); 1978 addr = (u64)sg_dma_address(sg); 1979 end_addr = (addr + sg_len); 1980 1981 /* 1982 * first sg elem in the 'list', 1983 * check if end addr is page-aligned. 1984 */ 1985 if ((i == 0) && (sg_count > 1) && (end_addr % QEDI_PAGE_SIZE)) 1986 cmd->use_slowpath = true; 1987 1988 /* 1989 * last sg elem in the 'list', 1990 * check if start addr is page-aligned. 1991 */ 1992 else if ((i == (sg_count - 1)) && 1993 (sg_count > 1) && (addr % QEDI_PAGE_SIZE)) 1994 cmd->use_slowpath = true; 1995 1996 /* 1997 * middle sg elements in list, 1998 * check if start and end addr is page-aligned 1999 */ 2000 else if ((i != 0) && (i != (sg_count - 1)) && 2001 ((addr % QEDI_PAGE_SIZE) || 2002 (end_addr % QEDI_PAGE_SIZE))) 2003 cmd->use_slowpath = true; 2004 2005 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, "sg[%d] size=0x%x", 2006 i, sg_len); 2007 2008 if (sg_len > QEDI_BD_SPLIT_SZ) { 2009 sg_frags = qedi_split_bd(cmd, addr, sg_len, bd_count); 2010 } else { 2011 sg_frags = 1; 2012 bd[bd_count].sge_addr.lo = addr & 0xffffffff; 2013 bd[bd_count].sge_addr.hi = addr >> 32; 2014 bd[bd_count].sge_len = sg_len; 2015 } 2016 byte_count += sg_len; 2017 bd_count += sg_frags; 2018 } 2019 2020 if (byte_count != scsi_bufflen(sc)) 2021 QEDI_ERR(&qedi->dbg_ctx, 2022 "byte_count = %d != scsi_bufflen = %d\n", byte_count, 2023 scsi_bufflen(sc)); 2024 else 2025 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, "byte_count = %d\n", 2026 byte_count); 2027 2028 WARN_ON(byte_count != scsi_bufflen(sc)); 2029 2030 return bd_count; 2031 } 2032 2033 static void qedi_iscsi_map_sg_list(struct qedi_cmd *cmd) 2034 { 2035 int bd_count; 2036 struct scsi_cmnd *sc = cmd->scsi_cmd; 2037 2038 if (scsi_sg_count(sc)) { 2039 bd_count = qedi_map_scsi_sg(cmd->conn->qedi, cmd); 2040 if (bd_count == 0) 2041 return; 2042 } else { 2043 struct iscsi_sge *bd = cmd->io_tbl.sge_tbl; 2044 2045 bd[0].sge_addr.lo = 0; 2046 bd[0].sge_addr.hi = 0; 2047 bd[0].sge_len = 0; 2048 bd_count = 0; 2049 } 2050 cmd->io_tbl.sge_valid = bd_count; 2051 } 2052 2053 static void qedi_cpy_scsi_cdb(struct scsi_cmnd *sc, u32 *dstp) 2054 { 2055 u32 dword; 2056 int lpcnt; 2057 u8 *srcp; 2058 2059 lpcnt = sc->cmd_len / sizeof(dword); 2060 srcp = (u8 *)sc->cmnd; 2061 while (lpcnt--) { 2062 memcpy(&dword, (const void *)srcp, 4); 2063 *dstp = cpu_to_be32(dword); 2064 srcp += 4; 2065 dstp++; 2066 } 2067 if (sc->cmd_len & 0x3) { 2068 dword = (u32)srcp[0] | ((u32)srcp[1] << 8); 2069 *dstp = cpu_to_be32(dword); 2070 } 2071 } 2072 2073 void qedi_trace_io(struct qedi_ctx *qedi, struct iscsi_task *task, 2074 u16 tid, int8_t direction) 2075 { 2076 struct qedi_io_log *io_log; 2077 struct iscsi_conn *conn = task->conn; 2078 struct qedi_conn *qedi_conn = conn->dd_data; 2079 struct scsi_cmnd *sc_cmd = task->sc; 2080 unsigned long flags; 2081 u8 op; 2082 2083 spin_lock_irqsave(&qedi->io_trace_lock, flags); 2084 2085 io_log = &qedi->io_trace_buf[qedi->io_trace_idx]; 2086 io_log->direction = direction; 2087 io_log->task_id = tid; 2088 io_log->cid = qedi_conn->iscsi_conn_id; 2089 io_log->lun = sc_cmd->device->lun; 2090 io_log->op = sc_cmd->cmnd[0]; 2091 op = sc_cmd->cmnd[0]; 2092 io_log->lba[0] = sc_cmd->cmnd[2]; 2093 io_log->lba[1] = sc_cmd->cmnd[3]; 2094 io_log->lba[2] = sc_cmd->cmnd[4]; 2095 io_log->lba[3] = sc_cmd->cmnd[5]; 2096 io_log->bufflen = scsi_bufflen(sc_cmd); 2097 io_log->sg_count = scsi_sg_count(sc_cmd); 2098 io_log->fast_sgs = qedi->fast_sgls; 2099 io_log->cached_sgs = qedi->cached_sgls; 2100 io_log->slow_sgs = qedi->slow_sgls; 2101 io_log->cached_sge = qedi->use_cached_sge; 2102 io_log->slow_sge = qedi->use_slow_sge; 2103 io_log->fast_sge = qedi->use_fast_sge; 2104 io_log->result = sc_cmd->result; 2105 io_log->jiffies = jiffies; 2106 io_log->blk_req_cpu = smp_processor_id(); 2107 2108 if (direction == QEDI_IO_TRACE_REQ) { 2109 /* For requests we only care about the submission CPU */ 2110 io_log->req_cpu = smp_processor_id() % qedi->num_queues; 2111 io_log->intr_cpu = 0; 2112 io_log->blk_rsp_cpu = 0; 2113 } else if (direction == QEDI_IO_TRACE_RSP) { 2114 io_log->req_cpu = smp_processor_id() % qedi->num_queues; 2115 io_log->intr_cpu = qedi->intr_cpu; 2116 io_log->blk_rsp_cpu = smp_processor_id(); 2117 } 2118 2119 qedi->io_trace_idx++; 2120 if (qedi->io_trace_idx == QEDI_IO_TRACE_SIZE) 2121 qedi->io_trace_idx = 0; 2122 2123 qedi->use_cached_sge = false; 2124 qedi->use_slow_sge = false; 2125 qedi->use_fast_sge = false; 2126 2127 spin_unlock_irqrestore(&qedi->io_trace_lock, flags); 2128 } 2129 2130 int qedi_iscsi_send_ioreq(struct iscsi_task *task) 2131 { 2132 struct iscsi_conn *conn = task->conn; 2133 struct iscsi_session *session = conn->session; 2134 struct Scsi_Host *shost = iscsi_session_to_shost(session->cls_session); 2135 struct qedi_ctx *qedi = iscsi_host_priv(shost); 2136 struct qedi_conn *qedi_conn = conn->dd_data; 2137 struct qedi_cmd *cmd = task->dd_data; 2138 struct scsi_cmnd *sc = task->sc; 2139 struct iscsi_task_context *fw_task_ctx; 2140 struct iscsi_cached_sge_ctx *cached_sge; 2141 struct iscsi_phys_sgl_ctx *phys_sgl; 2142 struct iscsi_virt_sgl_ctx *virt_sgl; 2143 struct ystorm_iscsi_task_st_ctx *yst_cxt; 2144 struct mstorm_iscsi_task_st_ctx *mst_cxt; 2145 struct iscsi_sgl *sgl_struct; 2146 struct iscsi_sge *single_sge; 2147 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr; 2148 struct iscsi_sge *bd = cmd->io_tbl.sge_tbl; 2149 enum iscsi_task_type task_type; 2150 struct iscsi_cmd_hdr *fw_cmd; 2151 u32 lun[2]; 2152 u32 exp_data; 2153 u16 cq_idx = smp_processor_id() % qedi->num_queues; 2154 s16 ptu_invalidate = 0; 2155 s16 tid = 0; 2156 u8 num_fast_sgs; 2157 2158 tid = qedi_get_task_idx(qedi); 2159 if (tid == -1) 2160 return -ENOMEM; 2161 2162 qedi_iscsi_map_sg_list(cmd); 2163 2164 int_to_scsilun(sc->device->lun, (struct scsi_lun *)lun); 2165 fw_task_ctx = qedi_get_task_mem(&qedi->tasks, tid); 2166 2167 memset(fw_task_ctx, 0, sizeof(struct iscsi_task_context)); 2168 cmd->task_id = tid; 2169 2170 /* Ystorm context */ 2171 fw_cmd = &fw_task_ctx->ystorm_st_context.pdu_hdr.cmd; 2172 SET_FIELD(fw_cmd->flags_attr, ISCSI_CMD_HDR_ATTR, ISCSI_ATTR_SIMPLE); 2173 2174 if (sc->sc_data_direction == DMA_TO_DEVICE) { 2175 if (conn->session->initial_r2t_en) { 2176 exp_data = min((conn->session->imm_data_en * 2177 conn->max_xmit_dlength), 2178 conn->session->first_burst); 2179 exp_data = min(exp_data, scsi_bufflen(sc)); 2180 fw_task_ctx->ustorm_ag_context.exp_data_acked = 2181 cpu_to_le32(exp_data); 2182 } else { 2183 fw_task_ctx->ustorm_ag_context.exp_data_acked = 2184 min(conn->session->first_burst, scsi_bufflen(sc)); 2185 } 2186 2187 SET_FIELD(fw_cmd->flags_attr, ISCSI_CMD_HDR_WRITE, 1); 2188 task_type = ISCSI_TASK_TYPE_INITIATOR_WRITE; 2189 } else { 2190 if (scsi_bufflen(sc)) 2191 SET_FIELD(fw_cmd->flags_attr, ISCSI_CMD_HDR_READ, 1); 2192 task_type = ISCSI_TASK_TYPE_INITIATOR_READ; 2193 } 2194 2195 fw_cmd->lun.lo = be32_to_cpu(lun[0]); 2196 fw_cmd->lun.hi = be32_to_cpu(lun[1]); 2197 2198 qedi_update_itt_map(qedi, tid, task->itt, cmd); 2199 fw_cmd->itt = qedi_set_itt(tid, get_itt(task->itt)); 2200 fw_cmd->expected_transfer_length = scsi_bufflen(sc); 2201 fw_cmd->cmd_sn = be32_to_cpu(hdr->cmdsn); 2202 fw_cmd->opcode = hdr->opcode; 2203 qedi_cpy_scsi_cdb(sc, (u32 *)fw_cmd->cdb); 2204 2205 /* Mstorm context */ 2206 fw_task_ctx->mstorm_st_context.sense_db.lo = (u32)cmd->sense_buffer_dma; 2207 fw_task_ctx->mstorm_st_context.sense_db.hi = 2208 (u32)((u64)cmd->sense_buffer_dma >> 32); 2209 fw_task_ctx->mstorm_ag_context.task_cid = qedi_conn->iscsi_conn_id; 2210 fw_task_ctx->mstorm_st_context.task_type = task_type; 2211 2212 if (qedi->tid_reuse_count[tid] == QEDI_MAX_TASK_NUM) { 2213 ptu_invalidate = 1; 2214 qedi->tid_reuse_count[tid] = 0; 2215 } 2216 fw_task_ctx->ystorm_st_context.state.reuse_count = 2217 qedi->tid_reuse_count[tid]; 2218 fw_task_ctx->mstorm_st_context.reuse_count = 2219 qedi->tid_reuse_count[tid]++; 2220 2221 /* Ustorm context */ 2222 fw_task_ctx->ustorm_st_context.rem_rcv_len = scsi_bufflen(sc); 2223 fw_task_ctx->ustorm_st_context.exp_data_transfer_len = scsi_bufflen(sc); 2224 fw_task_ctx->ustorm_st_context.exp_data_sn = 2225 be32_to_cpu(hdr->exp_statsn); 2226 fw_task_ctx->ustorm_st_context.task_type = task_type; 2227 fw_task_ctx->ustorm_st_context.cq_rss_number = cq_idx; 2228 fw_task_ctx->ustorm_ag_context.icid = (u16)qedi_conn->iscsi_conn_id; 2229 2230 SET_FIELD(fw_task_ctx->ustorm_ag_context.flags1, 2231 USTORM_ISCSI_TASK_AG_CTX_R2T2RECV, 1); 2232 SET_FIELD(fw_task_ctx->ustorm_st_context.flags, 2233 USTORM_ISCSI_TASK_ST_CTX_LOCAL_COMP, 0); 2234 2235 num_fast_sgs = (cmd->io_tbl.sge_valid ? 2236 min((u16)QEDI_FAST_SGE_COUNT, 2237 (u16)cmd->io_tbl.sge_valid) : 0); 2238 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map, 2239 ISCSI_REG1_NUM_FAST_SGES, num_fast_sgs); 2240 2241 fw_task_ctx->ustorm_st_context.lun.lo = be32_to_cpu(lun[0]); 2242 fw_task_ctx->ustorm_st_context.lun.hi = be32_to_cpu(lun[1]); 2243 2244 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, "Total sge count [%d]\n", 2245 cmd->io_tbl.sge_valid); 2246 2247 yst_cxt = &fw_task_ctx->ystorm_st_context; 2248 mst_cxt = &fw_task_ctx->mstorm_st_context; 2249 /* Tx path */ 2250 if (task_type == ISCSI_TASK_TYPE_INITIATOR_WRITE) { 2251 /* not considering superIO or FastIO */ 2252 if (cmd->io_tbl.sge_valid == 1) { 2253 cached_sge = &yst_cxt->state.sgl_ctx_union.cached_sge; 2254 cached_sge->sge.sge_addr.lo = bd[0].sge_addr.lo; 2255 cached_sge->sge.sge_addr.hi = bd[0].sge_addr.hi; 2256 cached_sge->sge.sge_len = bd[0].sge_len; 2257 qedi->cached_sgls++; 2258 } else if ((cmd->io_tbl.sge_valid != 1) && cmd->use_slowpath) { 2259 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 2260 ISCSI_MFLAGS_SLOW_IO, 1); 2261 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map, 2262 ISCSI_REG1_NUM_FAST_SGES, 0); 2263 phys_sgl = &yst_cxt->state.sgl_ctx_union.phys_sgl; 2264 phys_sgl->sgl_base.lo = (u32)(cmd->io_tbl.sge_tbl_dma); 2265 phys_sgl->sgl_base.hi = 2266 (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32); 2267 phys_sgl->sgl_size = cmd->io_tbl.sge_valid; 2268 qedi->slow_sgls++; 2269 } else if ((cmd->io_tbl.sge_valid != 1) && !cmd->use_slowpath) { 2270 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 2271 ISCSI_MFLAGS_SLOW_IO, 0); 2272 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map, 2273 ISCSI_REG1_NUM_FAST_SGES, 2274 min((u16)QEDI_FAST_SGE_COUNT, 2275 (u16)cmd->io_tbl.sge_valid)); 2276 virt_sgl = &yst_cxt->state.sgl_ctx_union.virt_sgl; 2277 virt_sgl->sgl_base.lo = (u32)(cmd->io_tbl.sge_tbl_dma); 2278 virt_sgl->sgl_base.hi = 2279 (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32); 2280 virt_sgl->sgl_initial_offset = 2281 (u32)bd[0].sge_addr.lo & (QEDI_PAGE_SIZE - 1); 2282 qedi->fast_sgls++; 2283 } 2284 fw_task_ctx->mstorm_st_context.sgl_size = cmd->io_tbl.sge_valid; 2285 fw_task_ctx->mstorm_st_context.rem_task_size = scsi_bufflen(sc); 2286 } else { 2287 /* Rx path */ 2288 if (cmd->io_tbl.sge_valid == 1) { 2289 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 2290 ISCSI_MFLAGS_SLOW_IO, 0); 2291 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 2292 ISCSI_MFLAGS_SINGLE_SGE, 1); 2293 single_sge = &mst_cxt->sgl_union.single_sge; 2294 single_sge->sge_addr.lo = bd[0].sge_addr.lo; 2295 single_sge->sge_addr.hi = bd[0].sge_addr.hi; 2296 single_sge->sge_len = bd[0].sge_len; 2297 qedi->cached_sgls++; 2298 } else if ((cmd->io_tbl.sge_valid != 1) && cmd->use_slowpath) { 2299 sgl_struct = &mst_cxt->sgl_union.sgl_struct; 2300 sgl_struct->sgl_addr.lo = 2301 (u32)(cmd->io_tbl.sge_tbl_dma); 2302 sgl_struct->sgl_addr.hi = 2303 (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32); 2304 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 2305 ISCSI_MFLAGS_SLOW_IO, 1); 2306 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map, 2307 ISCSI_REG1_NUM_FAST_SGES, 0); 2308 sgl_struct->updated_sge_size = 0; 2309 sgl_struct->updated_sge_offset = 0; 2310 qedi->slow_sgls++; 2311 } else if ((cmd->io_tbl.sge_valid != 1) && !cmd->use_slowpath) { 2312 sgl_struct = &mst_cxt->sgl_union.sgl_struct; 2313 sgl_struct->sgl_addr.lo = 2314 (u32)(cmd->io_tbl.sge_tbl_dma); 2315 sgl_struct->sgl_addr.hi = 2316 (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32); 2317 sgl_struct->byte_offset = 2318 (u32)bd[0].sge_addr.lo & (QEDI_PAGE_SIZE - 1); 2319 SET_FIELD(fw_task_ctx->mstorm_st_context.flags.mflags, 2320 ISCSI_MFLAGS_SLOW_IO, 0); 2321 SET_FIELD(fw_task_ctx->ustorm_st_context.reg1.reg1_map, 2322 ISCSI_REG1_NUM_FAST_SGES, 0); 2323 sgl_struct->updated_sge_size = 0; 2324 sgl_struct->updated_sge_offset = 0; 2325 qedi->fast_sgls++; 2326 } 2327 fw_task_ctx->mstorm_st_context.sgl_size = cmd->io_tbl.sge_valid; 2328 fw_task_ctx->mstorm_st_context.rem_task_size = scsi_bufflen(sc); 2329 } 2330 2331 if (cmd->io_tbl.sge_valid == 1) 2332 /* Singel-SGL */ 2333 qedi->use_cached_sge = true; 2334 else { 2335 if (cmd->use_slowpath) 2336 qedi->use_slow_sge = true; 2337 else 2338 qedi->use_fast_sge = true; 2339 } 2340 QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO, 2341 "%s: %s-SGL: num_sges=0x%x first-sge-lo=0x%x first-sge-hi=0x%x", 2342 (task_type == ISCSI_TASK_TYPE_INITIATOR_WRITE) ? 2343 "Write " : "Read ", (cmd->io_tbl.sge_valid == 1) ? 2344 "Single" : (cmd->use_slowpath ? "SLOW" : "FAST"), 2345 (u16)cmd->io_tbl.sge_valid, (u32)(cmd->io_tbl.sge_tbl_dma), 2346 (u32)((u64)cmd->io_tbl.sge_tbl_dma >> 32)); 2347 2348 /* Add command in active command list */ 2349 spin_lock(&qedi_conn->list_lock); 2350 list_add_tail(&cmd->io_cmd, &qedi_conn->active_cmd_list); 2351 cmd->io_cmd_in_list = true; 2352 qedi_conn->active_cmd_count++; 2353 spin_unlock(&qedi_conn->list_lock); 2354 2355 qedi_add_to_sq(qedi_conn, task, tid, ptu_invalidate, false); 2356 qedi_ring_doorbell(qedi_conn); 2357 if (qedi_io_tracing) 2358 qedi_trace_io(qedi, task, tid, QEDI_IO_TRACE_REQ); 2359 2360 return 0; 2361 } 2362 2363 int qedi_iscsi_cleanup_task(struct iscsi_task *task, bool mark_cmd_node_deleted) 2364 { 2365 struct iscsi_conn *conn = task->conn; 2366 struct qedi_conn *qedi_conn = conn->dd_data; 2367 struct qedi_cmd *cmd = task->dd_data; 2368 s16 ptu_invalidate = 0; 2369 2370 QEDI_INFO(&qedi_conn->qedi->dbg_ctx, QEDI_LOG_SCSI_TM, 2371 "issue cleanup tid=0x%x itt=0x%x task_state=%d cmd_state=0%x cid=0x%x\n", 2372 cmd->task_id, get_itt(task->itt), task->state, 2373 cmd->state, qedi_conn->iscsi_conn_id); 2374 2375 qedi_add_to_sq(qedi_conn, task, cmd->task_id, ptu_invalidate, true); 2376 qedi_ring_doorbell(qedi_conn); 2377 2378 return 0; 2379 } 2380