14814a0a4SEdward Tomasz Napierala /* $FreeBSD$ */ 24814a0a4SEdward Tomasz Napierala /*- 34814a0a4SEdward Tomasz Napierala * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved. 44814a0a4SEdward Tomasz Napierala * 54814a0a4SEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without 64814a0a4SEdward Tomasz Napierala * modification, are permitted provided that the following conditions 74814a0a4SEdward Tomasz Napierala * are met: 84814a0a4SEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright 94814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer. 104814a0a4SEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright 114814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the 124814a0a4SEdward Tomasz Napierala * documentation and/or other materials provided with the distribution. 134814a0a4SEdward Tomasz Napierala * 144814a0a4SEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 154814a0a4SEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 164814a0a4SEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 174814a0a4SEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 184814a0a4SEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 194814a0a4SEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 204814a0a4SEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 214814a0a4SEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 224814a0a4SEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 234814a0a4SEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 244814a0a4SEdward Tomasz Napierala * SUCH DAMAGE. 254814a0a4SEdward Tomasz Napierala */ 264814a0a4SEdward Tomasz Napierala 274814a0a4SEdward Tomasz Napierala #include "icl_iser.h" 284814a0a4SEdward Tomasz Napierala 29*7029da5cSPawel Biernacki SYSCTL_NODE(_kern, OID_AUTO, iser, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 30*7029da5cSPawel Biernacki "iSER module"); 314814a0a4SEdward Tomasz Napierala int iser_debug = 0; 324814a0a4SEdward Tomasz Napierala SYSCTL_INT(_kern_iser, OID_AUTO, debug, CTLFLAG_RWTUN, 334814a0a4SEdward Tomasz Napierala &iser_debug, 0, "Enable iser debug messages"); 344814a0a4SEdward Tomasz Napierala 354814a0a4SEdward Tomasz Napierala static MALLOC_DEFINE(M_ICL_ISER, "icl_iser", "iSCSI iser backend"); 364814a0a4SEdward Tomasz Napierala static uma_zone_t icl_pdu_zone; 374814a0a4SEdward Tomasz Napierala 384814a0a4SEdward Tomasz Napierala static volatile u_int icl_iser_ncons; 394814a0a4SEdward Tomasz Napierala struct iser_global ig; 404814a0a4SEdward Tomasz Napierala 4195c4a392SEdward Tomasz Napierala static void iser_conn_release(struct icl_conn *ic); 4295c4a392SEdward Tomasz Napierala 434814a0a4SEdward Tomasz Napierala static icl_conn_new_pdu_t iser_conn_new_pdu; 444814a0a4SEdward Tomasz Napierala static icl_conn_pdu_free_t iser_conn_pdu_free; 454814a0a4SEdward Tomasz Napierala static icl_conn_pdu_data_segment_length_t iser_conn_pdu_data_segment_length; 464814a0a4SEdward Tomasz Napierala static icl_conn_pdu_append_data_t iser_conn_pdu_append_data; 474814a0a4SEdward Tomasz Napierala static icl_conn_pdu_queue_t iser_conn_pdu_queue; 484814a0a4SEdward Tomasz Napierala static icl_conn_handoff_t iser_conn_handoff; 494814a0a4SEdward Tomasz Napierala static icl_conn_free_t iser_conn_free; 504814a0a4SEdward Tomasz Napierala static icl_conn_close_t iser_conn_close; 514814a0a4SEdward Tomasz Napierala static icl_conn_connect_t iser_conn_connect; 524814a0a4SEdward Tomasz Napierala static icl_conn_task_setup_t iser_conn_task_setup; 534814a0a4SEdward Tomasz Napierala static icl_conn_task_done_t iser_conn_task_done; 544814a0a4SEdward Tomasz Napierala static icl_conn_pdu_get_data_t iser_conn_pdu_get_data; 554814a0a4SEdward Tomasz Napierala 564814a0a4SEdward Tomasz Napierala static kobj_method_t icl_iser_methods[] = { 574814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_new_pdu, iser_conn_new_pdu), 584814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_free, iser_conn_pdu_free), 594814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_data_segment_length, iser_conn_pdu_data_segment_length), 604814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_append_data, iser_conn_pdu_append_data), 614814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_queue, iser_conn_pdu_queue), 624814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_handoff, iser_conn_handoff), 634814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_free, iser_conn_free), 644814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_close, iser_conn_close), 654814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_connect, iser_conn_connect), 664814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_task_setup, iser_conn_task_setup), 674814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_task_done, iser_conn_task_done), 684814a0a4SEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_get_data, iser_conn_pdu_get_data), 694814a0a4SEdward Tomasz Napierala { 0, 0 } 704814a0a4SEdward Tomasz Napierala }; 714814a0a4SEdward Tomasz Napierala 724814a0a4SEdward Tomasz Napierala DEFINE_CLASS(icl_iser, icl_iser_methods, sizeof(struct iser_conn)); 734814a0a4SEdward Tomasz Napierala 744814a0a4SEdward Tomasz Napierala /** 754814a0a4SEdward Tomasz Napierala * iser_initialize_headers() - Initialize task headers 764814a0a4SEdward Tomasz Napierala * @pdu: iser pdu 774814a0a4SEdward Tomasz Napierala * @iser_conn: iser connection 784814a0a4SEdward Tomasz Napierala * 794814a0a4SEdward Tomasz Napierala * Notes: 804814a0a4SEdward Tomasz Napierala * This routine may race with iser teardown flow for scsi 814814a0a4SEdward Tomasz Napierala * error handling TMFs. So for TMF we should acquire the 824814a0a4SEdward Tomasz Napierala * state mutex to avoid dereferencing the IB device which 834814a0a4SEdward Tomasz Napierala * may have already been terminated (racing teardown sequence). 844814a0a4SEdward Tomasz Napierala */ 854814a0a4SEdward Tomasz Napierala int 864814a0a4SEdward Tomasz Napierala iser_initialize_headers(struct icl_iser_pdu *pdu, struct iser_conn *iser_conn) 874814a0a4SEdward Tomasz Napierala { 884814a0a4SEdward Tomasz Napierala struct iser_tx_desc *tx_desc = &pdu->desc; 894814a0a4SEdward Tomasz Napierala struct iser_device *device = iser_conn->ib_conn.device; 904814a0a4SEdward Tomasz Napierala u64 dma_addr; 914814a0a4SEdward Tomasz Napierala int ret = 0; 924814a0a4SEdward Tomasz Napierala 934814a0a4SEdward Tomasz Napierala dma_addr = ib_dma_map_single(device->ib_device, (void *)tx_desc, 944814a0a4SEdward Tomasz Napierala ISER_HEADERS_LEN, DMA_TO_DEVICE); 954814a0a4SEdward Tomasz Napierala if (ib_dma_mapping_error(device->ib_device, dma_addr)) { 964814a0a4SEdward Tomasz Napierala ret = -ENOMEM; 974814a0a4SEdward Tomasz Napierala goto out; 984814a0a4SEdward Tomasz Napierala } 994814a0a4SEdward Tomasz Napierala 1004814a0a4SEdward Tomasz Napierala tx_desc->mapped = true; 1014814a0a4SEdward Tomasz Napierala tx_desc->dma_addr = dma_addr; 1024814a0a4SEdward Tomasz Napierala tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 1034814a0a4SEdward Tomasz Napierala tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 1044814a0a4SEdward Tomasz Napierala tx_desc->tx_sg[0].lkey = device->mr->lkey; 1054814a0a4SEdward Tomasz Napierala 1064814a0a4SEdward Tomasz Napierala out: 1074814a0a4SEdward Tomasz Napierala 1084814a0a4SEdward Tomasz Napierala return (ret); 1094814a0a4SEdward Tomasz Napierala } 1104814a0a4SEdward Tomasz Napierala 1114814a0a4SEdward Tomasz Napierala int 1124814a0a4SEdward Tomasz Napierala iser_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request, 1134814a0a4SEdward Tomasz Napierala const void *addr, size_t len, int flags) 1144814a0a4SEdward Tomasz Napierala { 1154814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = icl_to_iser_conn(ic); 1164814a0a4SEdward Tomasz Napierala 1174814a0a4SEdward Tomasz Napierala if (request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_LOGIN_REQUEST || 1184814a0a4SEdward Tomasz Napierala request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_TEXT_REQUEST) { 1194814a0a4SEdward Tomasz Napierala ISER_DBG("copy to login buff"); 1204814a0a4SEdward Tomasz Napierala memcpy(iser_conn->login_req_buf, addr, len); 1214814a0a4SEdward Tomasz Napierala request->ip_data_len = len; 1224814a0a4SEdward Tomasz Napierala } 1234814a0a4SEdward Tomasz Napierala 1244814a0a4SEdward Tomasz Napierala return (0); 1254814a0a4SEdward Tomasz Napierala } 1264814a0a4SEdward Tomasz Napierala 1274814a0a4SEdward Tomasz Napierala void 1284814a0a4SEdward Tomasz Napierala iser_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, 1294814a0a4SEdward Tomasz Napierala size_t off, void *addr, size_t len) 1304814a0a4SEdward Tomasz Napierala { 1314814a0a4SEdward Tomasz Napierala /* If we have a receive data, copy it to upper layer buffer */ 1324814a0a4SEdward Tomasz Napierala if (ip->ip_data_mbuf) 1334814a0a4SEdward Tomasz Napierala memcpy(addr, ip->ip_data_mbuf + off, len); 1344814a0a4SEdward Tomasz Napierala } 1354814a0a4SEdward Tomasz Napierala 1364814a0a4SEdward Tomasz Napierala /* 1374814a0a4SEdward Tomasz Napierala * Allocate icl_pdu with empty BHS to fill up by the caller. 1384814a0a4SEdward Tomasz Napierala */ 1394814a0a4SEdward Tomasz Napierala struct icl_pdu * 1404814a0a4SEdward Tomasz Napierala iser_new_pdu(struct icl_conn *ic, int flags) 1414814a0a4SEdward Tomasz Napierala { 1424814a0a4SEdward Tomasz Napierala struct icl_iser_pdu *iser_pdu; 1434814a0a4SEdward Tomasz Napierala struct icl_pdu *ip; 1444814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = icl_to_iser_conn(ic); 1454814a0a4SEdward Tomasz Napierala 1464814a0a4SEdward Tomasz Napierala iser_pdu = uma_zalloc(icl_pdu_zone, flags | M_ZERO); 1474814a0a4SEdward Tomasz Napierala if (iser_pdu == NULL) { 1484814a0a4SEdward Tomasz Napierala ISER_WARN("failed to allocate %zd bytes", sizeof(*iser_pdu)); 1494814a0a4SEdward Tomasz Napierala return (NULL); 1504814a0a4SEdward Tomasz Napierala } 1514814a0a4SEdward Tomasz Napierala 1524814a0a4SEdward Tomasz Napierala iser_pdu->iser_conn = iser_conn; 1534814a0a4SEdward Tomasz Napierala ip = &iser_pdu->icl_pdu; 1544814a0a4SEdward Tomasz Napierala ip->ip_conn = ic; 1554814a0a4SEdward Tomasz Napierala ip->ip_bhs = &iser_pdu->desc.iscsi_header; 1564814a0a4SEdward Tomasz Napierala 1574814a0a4SEdward Tomasz Napierala return (ip); 1584814a0a4SEdward Tomasz Napierala } 1594814a0a4SEdward Tomasz Napierala 1604814a0a4SEdward Tomasz Napierala struct icl_pdu * 1614814a0a4SEdward Tomasz Napierala iser_conn_new_pdu(struct icl_conn *ic, int flags) 1624814a0a4SEdward Tomasz Napierala { 1634814a0a4SEdward Tomasz Napierala return (iser_new_pdu(ic, flags)); 1644814a0a4SEdward Tomasz Napierala } 1654814a0a4SEdward Tomasz Napierala 1664814a0a4SEdward Tomasz Napierala void 1674814a0a4SEdward Tomasz Napierala iser_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 1684814a0a4SEdward Tomasz Napierala { 1694814a0a4SEdward Tomasz Napierala struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip); 1704814a0a4SEdward Tomasz Napierala 1714814a0a4SEdward Tomasz Napierala uma_zfree(icl_pdu_zone, iser_pdu); 1724814a0a4SEdward Tomasz Napierala } 1734814a0a4SEdward Tomasz Napierala 1744814a0a4SEdward Tomasz Napierala size_t 1754814a0a4SEdward Tomasz Napierala iser_conn_pdu_data_segment_length(struct icl_conn *ic, 1764814a0a4SEdward Tomasz Napierala const struct icl_pdu *request) 1774814a0a4SEdward Tomasz Napierala { 1784814a0a4SEdward Tomasz Napierala uint32_t len = 0; 1794814a0a4SEdward Tomasz Napierala 1804814a0a4SEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[0]; 1814814a0a4SEdward Tomasz Napierala len <<= 8; 1824814a0a4SEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[1]; 1834814a0a4SEdward Tomasz Napierala len <<= 8; 1844814a0a4SEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[2]; 1854814a0a4SEdward Tomasz Napierala 1864814a0a4SEdward Tomasz Napierala return (len); 1874814a0a4SEdward Tomasz Napierala } 1884814a0a4SEdward Tomasz Napierala 1894814a0a4SEdward Tomasz Napierala void 1904814a0a4SEdward Tomasz Napierala iser_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 1914814a0a4SEdward Tomasz Napierala { 1924814a0a4SEdward Tomasz Napierala iser_pdu_free(ic, ip); 1934814a0a4SEdward Tomasz Napierala } 1944814a0a4SEdward Tomasz Napierala 1954814a0a4SEdward Tomasz Napierala static bool 1964814a0a4SEdward Tomasz Napierala is_control_opcode(uint8_t opcode) 1974814a0a4SEdward Tomasz Napierala { 1984814a0a4SEdward Tomasz Napierala bool is_control = false; 1994814a0a4SEdward Tomasz Napierala 2004814a0a4SEdward Tomasz Napierala switch (opcode & ISCSI_OPCODE_MASK) { 2014814a0a4SEdward Tomasz Napierala case ISCSI_BHS_OPCODE_NOP_OUT: 2024814a0a4SEdward Tomasz Napierala case ISCSI_BHS_OPCODE_LOGIN_REQUEST: 2034814a0a4SEdward Tomasz Napierala case ISCSI_BHS_OPCODE_LOGOUT_REQUEST: 2044814a0a4SEdward Tomasz Napierala case ISCSI_BHS_OPCODE_TEXT_REQUEST: 2054814a0a4SEdward Tomasz Napierala is_control = true; 2064814a0a4SEdward Tomasz Napierala break; 2074814a0a4SEdward Tomasz Napierala case ISCSI_BHS_OPCODE_SCSI_COMMAND: 2084814a0a4SEdward Tomasz Napierala is_control = false; 2094814a0a4SEdward Tomasz Napierala break; 2104814a0a4SEdward Tomasz Napierala default: 2114814a0a4SEdward Tomasz Napierala ISER_ERR("unknown opcode %d", opcode); 2124814a0a4SEdward Tomasz Napierala } 2134814a0a4SEdward Tomasz Napierala 2144814a0a4SEdward Tomasz Napierala return (is_control); 2154814a0a4SEdward Tomasz Napierala } 2164814a0a4SEdward Tomasz Napierala 2174814a0a4SEdward Tomasz Napierala void 2184814a0a4SEdward Tomasz Napierala iser_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) 2194814a0a4SEdward Tomasz Napierala { 2204814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = icl_to_iser_conn(ic); 2214814a0a4SEdward Tomasz Napierala struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip); 2224814a0a4SEdward Tomasz Napierala int ret; 2234814a0a4SEdward Tomasz Napierala 2244686f1a6SEdward Tomasz Napierala if (iser_conn->state != ISER_CONN_UP) 2254686f1a6SEdward Tomasz Napierala return; 2264686f1a6SEdward Tomasz Napierala 2274814a0a4SEdward Tomasz Napierala ret = iser_initialize_headers(iser_pdu, iser_conn); 2284814a0a4SEdward Tomasz Napierala if (ret) { 2294814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to map TX descriptor pdu %p", iser_pdu); 2304814a0a4SEdward Tomasz Napierala return; 2314814a0a4SEdward Tomasz Napierala } 2324814a0a4SEdward Tomasz Napierala 2334814a0a4SEdward Tomasz Napierala if (is_control_opcode(ip->ip_bhs->bhs_opcode)) { 2344814a0a4SEdward Tomasz Napierala ret = iser_send_control(iser_conn, iser_pdu); 2354814a0a4SEdward Tomasz Napierala if (unlikely(ret)) 2364814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to send control pdu %p", iser_pdu); 2374814a0a4SEdward Tomasz Napierala } else { 2384814a0a4SEdward Tomasz Napierala ret = iser_send_command(iser_conn, iser_pdu); 2394814a0a4SEdward Tomasz Napierala if (unlikely(ret)) 2404814a0a4SEdward Tomasz Napierala ISER_ERR("Failed to send command pdu %p", iser_pdu); 2414814a0a4SEdward Tomasz Napierala } 2424814a0a4SEdward Tomasz Napierala } 2434814a0a4SEdward Tomasz Napierala 2444814a0a4SEdward Tomasz Napierala static struct icl_conn * 2454814a0a4SEdward Tomasz Napierala iser_new_conn(const char *name, struct mtx *lock) 2464814a0a4SEdward Tomasz Napierala { 2474814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn; 2484814a0a4SEdward Tomasz Napierala struct icl_conn *ic; 2494814a0a4SEdward Tomasz Napierala 2504814a0a4SEdward Tomasz Napierala refcount_acquire(&icl_iser_ncons); 2514814a0a4SEdward Tomasz Napierala 2524814a0a4SEdward Tomasz Napierala iser_conn = (struct iser_conn *)kobj_create(&icl_iser_class, M_ICL_ISER, M_WAITOK | M_ZERO); 2534814a0a4SEdward Tomasz Napierala if (!iser_conn) { 2544814a0a4SEdward Tomasz Napierala ISER_ERR("failed to allocate iser conn"); 2554814a0a4SEdward Tomasz Napierala refcount_release(&icl_iser_ncons); 2564814a0a4SEdward Tomasz Napierala return (NULL); 2574814a0a4SEdward Tomasz Napierala } 2584814a0a4SEdward Tomasz Napierala 2594814a0a4SEdward Tomasz Napierala cv_init(&iser_conn->up_cv, "iser_cv"); 2604814a0a4SEdward Tomasz Napierala sx_init(&iser_conn->state_mutex, "iser_conn_state_mutex"); 2612b378326SHans Petter Selasky mtx_init(&iser_conn->ib_conn.beacon.flush_lock, "iser_flush_lock", NULL, MTX_DEF); 2624814a0a4SEdward Tomasz Napierala cv_init(&iser_conn->ib_conn.beacon.flush_cv, "flush_cv"); 2632b378326SHans Petter Selasky mtx_init(&iser_conn->ib_conn.lock, "iser_lock", NULL, MTX_DEF); 2644814a0a4SEdward Tomasz Napierala 2654814a0a4SEdward Tomasz Napierala ic = &iser_conn->icl_conn; 2664814a0a4SEdward Tomasz Napierala ic->ic_lock = lock; 2674814a0a4SEdward Tomasz Napierala ic->ic_name = name; 26895c4a392SEdward Tomasz Napierala ic->ic_offload = strdup("iser", M_TEMP); 2694814a0a4SEdward Tomasz Napierala ic->ic_iser = true; 27095c4a392SEdward Tomasz Napierala ic->ic_unmapped = true; 2714814a0a4SEdward Tomasz Napierala 2724814a0a4SEdward Tomasz Napierala return (ic); 2734814a0a4SEdward Tomasz Napierala } 2744814a0a4SEdward Tomasz Napierala 2754814a0a4SEdward Tomasz Napierala void 2764814a0a4SEdward Tomasz Napierala iser_conn_free(struct icl_conn *ic) 2774814a0a4SEdward Tomasz Napierala { 2784814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = icl_to_iser_conn(ic); 2794814a0a4SEdward Tomasz Napierala 28095c4a392SEdward Tomasz Napierala iser_conn_release(ic); 2812b378326SHans Petter Selasky mtx_destroy(&iser_conn->ib_conn.lock); 2824814a0a4SEdward Tomasz Napierala cv_destroy(&iser_conn->ib_conn.beacon.flush_cv); 2834814a0a4SEdward Tomasz Napierala mtx_destroy(&iser_conn->ib_conn.beacon.flush_lock); 2844814a0a4SEdward Tomasz Napierala sx_destroy(&iser_conn->state_mutex); 2854814a0a4SEdward Tomasz Napierala cv_destroy(&iser_conn->up_cv); 2864814a0a4SEdward Tomasz Napierala kobj_delete((struct kobj *)iser_conn, M_ICL_ISER); 2874814a0a4SEdward Tomasz Napierala refcount_release(&icl_iser_ncons); 2884814a0a4SEdward Tomasz Napierala } 2894814a0a4SEdward Tomasz Napierala 2904814a0a4SEdward Tomasz Napierala int 29195c4a392SEdward Tomasz Napierala iser_conn_handoff(struct icl_conn *ic, int fd) 2924814a0a4SEdward Tomasz Napierala { 2934814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = icl_to_iser_conn(ic); 2944814a0a4SEdward Tomasz Napierala int error = 0; 2954814a0a4SEdward Tomasz Napierala 2964814a0a4SEdward Tomasz Napierala sx_xlock(&iser_conn->state_mutex); 2974814a0a4SEdward Tomasz Napierala if (iser_conn->state != ISER_CONN_UP) { 2984814a0a4SEdward Tomasz Napierala error = EINVAL; 2994814a0a4SEdward Tomasz Napierala ISER_ERR("iser_conn %p state is %d, teardown started\n", 3004814a0a4SEdward Tomasz Napierala iser_conn, iser_conn->state); 3014814a0a4SEdward Tomasz Napierala goto out; 3024814a0a4SEdward Tomasz Napierala } 3034814a0a4SEdward Tomasz Napierala 30495c4a392SEdward Tomasz Napierala error = iser_alloc_rx_descriptors(iser_conn, ic->ic_maxtags); 3054814a0a4SEdward Tomasz Napierala if (error) 3064814a0a4SEdward Tomasz Napierala goto out; 3074814a0a4SEdward Tomasz Napierala 3084814a0a4SEdward Tomasz Napierala error = iser_post_recvm(iser_conn, iser_conn->min_posted_rx); 3094814a0a4SEdward Tomasz Napierala if (error) 3104814a0a4SEdward Tomasz Napierala goto post_error; 3114814a0a4SEdward Tomasz Napierala 31295c4a392SEdward Tomasz Napierala iser_conn->handoff_done = true; 31395c4a392SEdward Tomasz Napierala 3144814a0a4SEdward Tomasz Napierala sx_xunlock(&iser_conn->state_mutex); 3154814a0a4SEdward Tomasz Napierala return (error); 3164814a0a4SEdward Tomasz Napierala 3174814a0a4SEdward Tomasz Napierala post_error: 3184814a0a4SEdward Tomasz Napierala iser_free_rx_descriptors(iser_conn); 3194814a0a4SEdward Tomasz Napierala out: 3204814a0a4SEdward Tomasz Napierala sx_xunlock(&iser_conn->state_mutex); 3214814a0a4SEdward Tomasz Napierala return (error); 3224814a0a4SEdward Tomasz Napierala 3234814a0a4SEdward Tomasz Napierala } 3244814a0a4SEdward Tomasz Napierala 3254814a0a4SEdward Tomasz Napierala /** 3264814a0a4SEdward Tomasz Napierala * Frees all conn objects 3274814a0a4SEdward Tomasz Napierala */ 32895c4a392SEdward Tomasz Napierala static void 3294814a0a4SEdward Tomasz Napierala iser_conn_release(struct icl_conn *ic) 3304814a0a4SEdward Tomasz Napierala { 3314814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = icl_to_iser_conn(ic); 3324814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 3334814a0a4SEdward Tomasz Napierala struct iser_conn *curr, *tmp; 3344814a0a4SEdward Tomasz Napierala 3354814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex); 3364814a0a4SEdward Tomasz Napierala /* 3374814a0a4SEdward Tomasz Napierala * Search for iser connection in global list. 3384814a0a4SEdward Tomasz Napierala * It may not be there in case of failure in connection establishment 3394814a0a4SEdward Tomasz Napierala * stage. 3404814a0a4SEdward Tomasz Napierala */ 3414814a0a4SEdward Tomasz Napierala list_for_each_entry_safe(curr, tmp, &ig.connlist, conn_list) { 3424814a0a4SEdward Tomasz Napierala if (iser_conn == curr) { 3434814a0a4SEdward Tomasz Napierala ISER_WARN("found iser_conn %p", iser_conn); 3444814a0a4SEdward Tomasz Napierala list_del(&iser_conn->conn_list); 3454814a0a4SEdward Tomasz Napierala } 3464814a0a4SEdward Tomasz Napierala } 3474814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex); 3484814a0a4SEdward Tomasz Napierala 3494814a0a4SEdward Tomasz Napierala /* 3504814a0a4SEdward Tomasz Napierala * In case we reconnecting or removing session, we need to 3514814a0a4SEdward Tomasz Napierala * release IB resources (which is safe to call more than once). 3524814a0a4SEdward Tomasz Napierala */ 3534814a0a4SEdward Tomasz Napierala sx_xlock(&iser_conn->state_mutex); 3544814a0a4SEdward Tomasz Napierala iser_free_ib_conn_res(iser_conn, true); 3554814a0a4SEdward Tomasz Napierala sx_xunlock(&iser_conn->state_mutex); 3564814a0a4SEdward Tomasz Napierala 3574814a0a4SEdward Tomasz Napierala if (ib_conn->cma_id != NULL) { 3584814a0a4SEdward Tomasz Napierala rdma_destroy_id(ib_conn->cma_id); 3594814a0a4SEdward Tomasz Napierala ib_conn->cma_id = NULL; 3604814a0a4SEdward Tomasz Napierala } 3614814a0a4SEdward Tomasz Napierala 3624814a0a4SEdward Tomasz Napierala } 3634814a0a4SEdward Tomasz Napierala 3644814a0a4SEdward Tomasz Napierala void 3654814a0a4SEdward Tomasz Napierala iser_conn_close(struct icl_conn *ic) 3664814a0a4SEdward Tomasz Napierala { 3674814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = icl_to_iser_conn(ic); 3684814a0a4SEdward Tomasz Napierala 3694814a0a4SEdward Tomasz Napierala ISER_INFO("closing conn %p", iser_conn); 3704814a0a4SEdward Tomasz Napierala 3714814a0a4SEdward Tomasz Napierala sx_xlock(&iser_conn->state_mutex); 3724814a0a4SEdward Tomasz Napierala /* 3734814a0a4SEdward Tomasz Napierala * In case iser connection is waiting on conditional variable 3744814a0a4SEdward Tomasz Napierala * (state PENDING) and we try to close it before connection establishment, 3754814a0a4SEdward Tomasz Napierala * we need to signal it to continue releasing connection properly. 3764814a0a4SEdward Tomasz Napierala */ 3774814a0a4SEdward Tomasz Napierala if (!iser_conn_terminate(iser_conn) && iser_conn->state == ISER_CONN_PENDING) 3784814a0a4SEdward Tomasz Napierala cv_signal(&iser_conn->up_cv); 3794814a0a4SEdward Tomasz Napierala sx_xunlock(&iser_conn->state_mutex); 3804814a0a4SEdward Tomasz Napierala 3814814a0a4SEdward Tomasz Napierala } 3824814a0a4SEdward Tomasz Napierala 3834814a0a4SEdward Tomasz Napierala int 3844814a0a4SEdward Tomasz Napierala iser_conn_connect(struct icl_conn *ic, int domain, int socktype, 3854814a0a4SEdward Tomasz Napierala int protocol, struct sockaddr *from_sa, struct sockaddr *to_sa) 3864814a0a4SEdward Tomasz Napierala { 3874814a0a4SEdward Tomasz Napierala struct iser_conn *iser_conn = icl_to_iser_conn(ic); 3884814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_conn->ib_conn; 3894814a0a4SEdward Tomasz Napierala int err = 0; 3904814a0a4SEdward Tomasz Napierala 39195c4a392SEdward Tomasz Napierala iser_conn_release(ic); 39295c4a392SEdward Tomasz Napierala 3934814a0a4SEdward Tomasz Napierala sx_xlock(&iser_conn->state_mutex); 3944814a0a4SEdward Tomasz Napierala /* the device is known only --after-- address resolution */ 3954814a0a4SEdward Tomasz Napierala ib_conn->device = NULL; 39695c4a392SEdward Tomasz Napierala iser_conn->handoff_done = false; 3974814a0a4SEdward Tomasz Napierala 3984814a0a4SEdward Tomasz Napierala iser_conn->state = ISER_CONN_PENDING; 3994814a0a4SEdward Tomasz Napierala 40041dbd9ddSHans Petter Selasky ib_conn->cma_id = rdma_create_id(&init_net, iser_cma_handler, (void *)iser_conn, 4014814a0a4SEdward Tomasz Napierala RDMA_PS_TCP, IB_QPT_RC); 4024814a0a4SEdward Tomasz Napierala if (IS_ERR(ib_conn->cma_id)) { 4034814a0a4SEdward Tomasz Napierala err = -PTR_ERR(ib_conn->cma_id); 4044814a0a4SEdward Tomasz Napierala ISER_ERR("rdma_create_id failed: %d", err); 4054814a0a4SEdward Tomasz Napierala goto id_failure; 4064814a0a4SEdward Tomasz Napierala } 4074814a0a4SEdward Tomasz Napierala 4084814a0a4SEdward Tomasz Napierala err = rdma_resolve_addr(ib_conn->cma_id, from_sa, to_sa, 1000); 4094814a0a4SEdward Tomasz Napierala if (err) { 4104814a0a4SEdward Tomasz Napierala ISER_ERR("rdma_resolve_addr failed: %d", err); 4114814a0a4SEdward Tomasz Napierala if (err < 0) 4124814a0a4SEdward Tomasz Napierala err = -err; 4134814a0a4SEdward Tomasz Napierala goto addr_failure; 4144814a0a4SEdward Tomasz Napierala } 4154814a0a4SEdward Tomasz Napierala 4164814a0a4SEdward Tomasz Napierala ISER_DBG("before cv_wait: %p", iser_conn); 4174814a0a4SEdward Tomasz Napierala cv_wait(&iser_conn->up_cv, &iser_conn->state_mutex); 4184814a0a4SEdward Tomasz Napierala ISER_DBG("after cv_wait: %p", iser_conn); 4194814a0a4SEdward Tomasz Napierala 4204814a0a4SEdward Tomasz Napierala if (iser_conn->state != ISER_CONN_UP) { 4214814a0a4SEdward Tomasz Napierala err = EIO; 4224814a0a4SEdward Tomasz Napierala goto addr_failure; 4234814a0a4SEdward Tomasz Napierala } 4244814a0a4SEdward Tomasz Napierala 4254814a0a4SEdward Tomasz Napierala err = iser_alloc_login_buf(iser_conn); 4264814a0a4SEdward Tomasz Napierala if (err) 4274814a0a4SEdward Tomasz Napierala goto addr_failure; 4284814a0a4SEdward Tomasz Napierala sx_xunlock(&iser_conn->state_mutex); 4294814a0a4SEdward Tomasz Napierala 4304814a0a4SEdward Tomasz Napierala mtx_lock(&ig.connlist_mutex); 4314814a0a4SEdward Tomasz Napierala list_add(&iser_conn->conn_list, &ig.connlist); 4324814a0a4SEdward Tomasz Napierala mtx_unlock(&ig.connlist_mutex); 4334814a0a4SEdward Tomasz Napierala 4344814a0a4SEdward Tomasz Napierala return (0); 4354814a0a4SEdward Tomasz Napierala 4364814a0a4SEdward Tomasz Napierala id_failure: 4374814a0a4SEdward Tomasz Napierala ib_conn->cma_id = NULL; 4384814a0a4SEdward Tomasz Napierala addr_failure: 4394814a0a4SEdward Tomasz Napierala sx_xunlock(&iser_conn->state_mutex); 4404814a0a4SEdward Tomasz Napierala return (err); 4414814a0a4SEdward Tomasz Napierala } 4424814a0a4SEdward Tomasz Napierala 4434814a0a4SEdward Tomasz Napierala int 44495c4a392SEdward Tomasz Napierala iser_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip, 44595c4a392SEdward Tomasz Napierala struct ccb_scsiio *csio, 44695c4a392SEdward Tomasz Napierala uint32_t *task_tagp, void **prvp) 4474814a0a4SEdward Tomasz Napierala { 4484814a0a4SEdward Tomasz Napierala struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip); 4494814a0a4SEdward Tomasz Napierala 4504814a0a4SEdward Tomasz Napierala *prvp = ip; 4514814a0a4SEdward Tomasz Napierala iser_pdu->csio = csio; 4524814a0a4SEdward Tomasz Napierala 4534814a0a4SEdward Tomasz Napierala return (0); 4544814a0a4SEdward Tomasz Napierala } 4554814a0a4SEdward Tomasz Napierala 4564814a0a4SEdward Tomasz Napierala void 4574814a0a4SEdward Tomasz Napierala iser_conn_task_done(struct icl_conn *ic, void *prv) 4584814a0a4SEdward Tomasz Napierala { 4594814a0a4SEdward Tomasz Napierala struct icl_pdu *ip = prv; 4604814a0a4SEdward Tomasz Napierala struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip); 4614814a0a4SEdward Tomasz Napierala struct iser_device *device = iser_pdu->iser_conn->ib_conn.device; 4624814a0a4SEdward Tomasz Napierala struct iser_tx_desc *tx_desc = &iser_pdu->desc; 4634814a0a4SEdward Tomasz Napierala 4644814a0a4SEdward Tomasz Napierala if (iser_pdu->dir[ISER_DIR_IN]) { 4654814a0a4SEdward Tomasz Napierala iser_unreg_rdma_mem(iser_pdu, ISER_DIR_IN); 4664814a0a4SEdward Tomasz Napierala iser_dma_unmap_task_data(iser_pdu, 4674814a0a4SEdward Tomasz Napierala &iser_pdu->data[ISER_DIR_IN], 4684814a0a4SEdward Tomasz Napierala DMA_FROM_DEVICE); 4694814a0a4SEdward Tomasz Napierala } 4704814a0a4SEdward Tomasz Napierala 4714814a0a4SEdward Tomasz Napierala if (iser_pdu->dir[ISER_DIR_OUT]) { 4724814a0a4SEdward Tomasz Napierala iser_unreg_rdma_mem(iser_pdu, ISER_DIR_OUT); 4734814a0a4SEdward Tomasz Napierala iser_dma_unmap_task_data(iser_pdu, 4744814a0a4SEdward Tomasz Napierala &iser_pdu->data[ISER_DIR_OUT], 4754814a0a4SEdward Tomasz Napierala DMA_TO_DEVICE); 4764814a0a4SEdward Tomasz Napierala } 4774814a0a4SEdward Tomasz Napierala 4784814a0a4SEdward Tomasz Napierala if (likely(tx_desc->mapped)) { 4794814a0a4SEdward Tomasz Napierala ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr, 4804814a0a4SEdward Tomasz Napierala ISER_HEADERS_LEN, DMA_TO_DEVICE); 4814814a0a4SEdward Tomasz Napierala tx_desc->mapped = false; 4824814a0a4SEdward Tomasz Napierala } 4834814a0a4SEdward Tomasz Napierala 4844814a0a4SEdward Tomasz Napierala iser_pdu_free(ic, ip); 4854814a0a4SEdward Tomasz Napierala } 4864814a0a4SEdward Tomasz Napierala 4874814a0a4SEdward Tomasz Napierala static int 48897b84d34SNavdeep Parhar iser_limits(struct icl_drv_limits *idl) 4894814a0a4SEdward Tomasz Napierala { 4909ac7c5a6SHans Petter Selasky 49197b84d34SNavdeep Parhar idl->idl_max_recv_data_segment_length = 128 * 1024; 4929ac7c5a6SHans Petter Selasky idl->idl_max_send_data_segment_length = 128 * 1024; 4939ac7c5a6SHans Petter Selasky idl->idl_max_burst_length = 262144; 4949ac7c5a6SHans Petter Selasky idl->idl_first_burst_length = 65536; 4954814a0a4SEdward Tomasz Napierala 4964814a0a4SEdward Tomasz Napierala return (0); 4974814a0a4SEdward Tomasz Napierala } 4984814a0a4SEdward Tomasz Napierala 4994814a0a4SEdward Tomasz Napierala static int 5004814a0a4SEdward Tomasz Napierala icl_iser_load(void) 5014814a0a4SEdward Tomasz Napierala { 5024814a0a4SEdward Tomasz Napierala int error; 5034814a0a4SEdward Tomasz Napierala 5044814a0a4SEdward Tomasz Napierala ISER_DBG("Starting iSER datamover..."); 5054814a0a4SEdward Tomasz Napierala 5064814a0a4SEdward Tomasz Napierala icl_pdu_zone = uma_zcreate("icl_iser_pdu", sizeof(struct icl_iser_pdu), 5074814a0a4SEdward Tomasz Napierala NULL, NULL, NULL, NULL, 5084814a0a4SEdward Tomasz Napierala UMA_ALIGN_PTR, 0); 5094814a0a4SEdward Tomasz Napierala /* FIXME: Check rc */ 5104814a0a4SEdward Tomasz Napierala 5114814a0a4SEdward Tomasz Napierala refcount_init(&icl_iser_ncons, 0); 5124814a0a4SEdward Tomasz Napierala 51395c4a392SEdward Tomasz Napierala error = icl_register("iser", true, 0, iser_limits, iser_new_conn); 5144814a0a4SEdward Tomasz Napierala KASSERT(error == 0, ("failed to register iser")); 5154814a0a4SEdward Tomasz Napierala 5164814a0a4SEdward Tomasz Napierala memset(&ig, 0, sizeof(struct iser_global)); 5174814a0a4SEdward Tomasz Napierala 5184814a0a4SEdward Tomasz Napierala /* device init is called only after the first addr resolution */ 5194814a0a4SEdward Tomasz Napierala sx_init(&ig.device_list_mutex, "global_device_lock"); 5204814a0a4SEdward Tomasz Napierala INIT_LIST_HEAD(&ig.device_list); 5212b378326SHans Petter Selasky mtx_init(&ig.connlist_mutex, "iser_global_conn_lock", NULL, MTX_DEF); 5224814a0a4SEdward Tomasz Napierala INIT_LIST_HEAD(&ig.connlist); 5234814a0a4SEdward Tomasz Napierala sx_init(&ig.close_conns_mutex, "global_close_conns_lock"); 5244814a0a4SEdward Tomasz Napierala 5254814a0a4SEdward Tomasz Napierala return (error); 5264814a0a4SEdward Tomasz Napierala } 5274814a0a4SEdward Tomasz Napierala 5284814a0a4SEdward Tomasz Napierala static int 5294814a0a4SEdward Tomasz Napierala icl_iser_unload(void) 5304814a0a4SEdward Tomasz Napierala { 5314814a0a4SEdward Tomasz Napierala ISER_DBG("Removing iSER datamover..."); 5324814a0a4SEdward Tomasz Napierala 5334814a0a4SEdward Tomasz Napierala if (icl_iser_ncons != 0) 5344814a0a4SEdward Tomasz Napierala return (EBUSY); 5354814a0a4SEdward Tomasz Napierala 5364814a0a4SEdward Tomasz Napierala sx_destroy(&ig.close_conns_mutex); 5374814a0a4SEdward Tomasz Napierala mtx_destroy(&ig.connlist_mutex); 5384814a0a4SEdward Tomasz Napierala sx_destroy(&ig.device_list_mutex); 5394814a0a4SEdward Tomasz Napierala 54095c4a392SEdward Tomasz Napierala icl_unregister("iser", true); 5414814a0a4SEdward Tomasz Napierala 5424814a0a4SEdward Tomasz Napierala uma_zdestroy(icl_pdu_zone); 5434814a0a4SEdward Tomasz Napierala 5444814a0a4SEdward Tomasz Napierala return (0); 5454814a0a4SEdward Tomasz Napierala } 5464814a0a4SEdward Tomasz Napierala 5474814a0a4SEdward Tomasz Napierala static int 5484814a0a4SEdward Tomasz Napierala icl_iser_modevent(module_t mod, int what, void *arg) 5494814a0a4SEdward Tomasz Napierala { 5504814a0a4SEdward Tomasz Napierala switch (what) { 5514814a0a4SEdward Tomasz Napierala case MOD_LOAD: 5524814a0a4SEdward Tomasz Napierala return (icl_iser_load()); 5534814a0a4SEdward Tomasz Napierala case MOD_UNLOAD: 5544814a0a4SEdward Tomasz Napierala return (icl_iser_unload()); 5554814a0a4SEdward Tomasz Napierala default: 5564814a0a4SEdward Tomasz Napierala return (EINVAL); 5574814a0a4SEdward Tomasz Napierala } 5584814a0a4SEdward Tomasz Napierala } 5594814a0a4SEdward Tomasz Napierala 5604814a0a4SEdward Tomasz Napierala moduledata_t icl_iser_data = { 5614814a0a4SEdward Tomasz Napierala .name = "icl_iser", 5624814a0a4SEdward Tomasz Napierala .evhand = icl_iser_modevent, 5634814a0a4SEdward Tomasz Napierala .priv = 0 5644814a0a4SEdward Tomasz Napierala }; 5654814a0a4SEdward Tomasz Napierala 5664814a0a4SEdward Tomasz Napierala DECLARE_MODULE(icl_iser, icl_iser_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); 5674814a0a4SEdward Tomasz Napierala MODULE_DEPEND(icl_iser, icl, 1, 1, 1); 5684814a0a4SEdward Tomasz Napierala MODULE_DEPEND(icl_iser, ibcore, 1, 1, 1); 5694814a0a4SEdward Tomasz Napierala MODULE_DEPEND(icl_iser, linuxkpi, 1, 1, 1); 5704814a0a4SEdward Tomasz Napierala MODULE_VERSION(icl_iser, 1); 571