14814a0a4SEdward Tomasz Napierala /*-
24814a0a4SEdward Tomasz Napierala * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
34814a0a4SEdward Tomasz Napierala *
44814a0a4SEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without
54814a0a4SEdward Tomasz Napierala * modification, are permitted provided that the following conditions
64814a0a4SEdward Tomasz Napierala * are met:
74814a0a4SEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright
84814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer.
94814a0a4SEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright
104814a0a4SEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the
114814a0a4SEdward Tomasz Napierala * documentation and/or other materials provided with the distribution.
124814a0a4SEdward Tomasz Napierala *
134814a0a4SEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
144814a0a4SEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
154814a0a4SEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
164814a0a4SEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
174814a0a4SEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
184814a0a4SEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
194814a0a4SEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
204814a0a4SEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
214814a0a4SEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
224814a0a4SEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
234814a0a4SEdward Tomasz Napierala * SUCH DAMAGE.
244814a0a4SEdward Tomasz Napierala */
254814a0a4SEdward Tomasz Napierala
264814a0a4SEdward Tomasz Napierala #include "icl_iser.h"
274814a0a4SEdward Tomasz Napierala
284814a0a4SEdward Tomasz Napierala static struct fast_reg_descriptor *
iser_reg_desc_get(struct ib_conn * ib_conn)294814a0a4SEdward Tomasz Napierala iser_reg_desc_get(struct ib_conn *ib_conn)
304814a0a4SEdward Tomasz Napierala {
314814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc;
324814a0a4SEdward Tomasz Napierala
334814a0a4SEdward Tomasz Napierala mtx_lock(&ib_conn->lock);
344814a0a4SEdward Tomasz Napierala desc = list_first_entry(&ib_conn->fastreg.pool,
354814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor, list);
364814a0a4SEdward Tomasz Napierala list_del(&desc->list);
374814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->lock);
384814a0a4SEdward Tomasz Napierala
394814a0a4SEdward Tomasz Napierala return (desc);
404814a0a4SEdward Tomasz Napierala }
414814a0a4SEdward Tomasz Napierala
424814a0a4SEdward Tomasz Napierala static void
iser_reg_desc_put(struct ib_conn * ib_conn,struct fast_reg_descriptor * desc)434814a0a4SEdward Tomasz Napierala iser_reg_desc_put(struct ib_conn *ib_conn,
444814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc)
454814a0a4SEdward Tomasz Napierala {
464814a0a4SEdward Tomasz Napierala mtx_lock(&ib_conn->lock);
474814a0a4SEdward Tomasz Napierala list_add(&desc->list, &ib_conn->fastreg.pool);
484814a0a4SEdward Tomasz Napierala mtx_unlock(&ib_conn->lock);
494814a0a4SEdward Tomasz Napierala }
504814a0a4SEdward Tomasz Napierala
514814a0a4SEdward Tomasz Napierala #define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & ~MASK_4K) == 0)
524814a0a4SEdward Tomasz Napierala
534814a0a4SEdward Tomasz Napierala /**
544814a0a4SEdward Tomasz Napierala * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned
554814a0a4SEdward Tomasz Napierala * for RDMA sub-list of a scatter-gather list of memory buffers, and returns
564814a0a4SEdward Tomasz Napierala * the number of entries which are aligned correctly. Supports the case where
574814a0a4SEdward Tomasz Napierala * consecutive SG elements are actually fragments of the same physcial page.
584814a0a4SEdward Tomasz Napierala */
594814a0a4SEdward Tomasz Napierala static int
iser_data_buf_aligned_len(struct iser_data_buf * data,struct ib_device * ibdev)604814a0a4SEdward Tomasz Napierala iser_data_buf_aligned_len(struct iser_data_buf *data, struct ib_device *ibdev)
614814a0a4SEdward Tomasz Napierala {
624814a0a4SEdward Tomasz Napierala struct scatterlist *sg, *sgl, *next_sg = NULL;
634814a0a4SEdward Tomasz Napierala u64 start_addr, end_addr;
644814a0a4SEdward Tomasz Napierala int i, ret_len, start_check = 0;
654814a0a4SEdward Tomasz Napierala
664814a0a4SEdward Tomasz Napierala if (data->dma_nents == 1)
674814a0a4SEdward Tomasz Napierala return (1);
684814a0a4SEdward Tomasz Napierala
694814a0a4SEdward Tomasz Napierala sgl = data->sgl;
704814a0a4SEdward Tomasz Napierala start_addr = ib_sg_dma_address(ibdev, sgl);
714814a0a4SEdward Tomasz Napierala
724814a0a4SEdward Tomasz Napierala for_each_sg(sgl, sg, data->dma_nents, i) {
734814a0a4SEdward Tomasz Napierala if (start_check && !IS_4K_ALIGNED(start_addr))
744814a0a4SEdward Tomasz Napierala break;
754814a0a4SEdward Tomasz Napierala
764814a0a4SEdward Tomasz Napierala next_sg = sg_next(sg);
774814a0a4SEdward Tomasz Napierala if (!next_sg)
784814a0a4SEdward Tomasz Napierala break;
794814a0a4SEdward Tomasz Napierala
804814a0a4SEdward Tomasz Napierala end_addr = start_addr + ib_sg_dma_len(ibdev, sg);
814814a0a4SEdward Tomasz Napierala start_addr = ib_sg_dma_address(ibdev, next_sg);
824814a0a4SEdward Tomasz Napierala
834814a0a4SEdward Tomasz Napierala if (end_addr == start_addr) {
844814a0a4SEdward Tomasz Napierala start_check = 0;
854814a0a4SEdward Tomasz Napierala continue;
864814a0a4SEdward Tomasz Napierala } else
874814a0a4SEdward Tomasz Napierala start_check = 1;
884814a0a4SEdward Tomasz Napierala
894814a0a4SEdward Tomasz Napierala if (!IS_4K_ALIGNED(end_addr))
904814a0a4SEdward Tomasz Napierala break;
914814a0a4SEdward Tomasz Napierala }
924814a0a4SEdward Tomasz Napierala ret_len = (next_sg) ? i : i+1;
934814a0a4SEdward Tomasz Napierala
944814a0a4SEdward Tomasz Napierala return (ret_len);
954814a0a4SEdward Tomasz Napierala }
964814a0a4SEdward Tomasz Napierala
974814a0a4SEdward Tomasz Napierala void
iser_dma_unmap_task_data(struct icl_iser_pdu * iser_pdu,struct iser_data_buf * data,enum dma_data_direction dir)984814a0a4SEdward Tomasz Napierala iser_dma_unmap_task_data(struct icl_iser_pdu *iser_pdu,
994814a0a4SEdward Tomasz Napierala struct iser_data_buf *data,
1004814a0a4SEdward Tomasz Napierala enum dma_data_direction dir)
1014814a0a4SEdward Tomasz Napierala {
1024814a0a4SEdward Tomasz Napierala struct ib_device *dev;
1034814a0a4SEdward Tomasz Napierala
1044814a0a4SEdward Tomasz Napierala dev = iser_pdu->iser_conn->ib_conn.device->ib_device;
1054814a0a4SEdward Tomasz Napierala ib_dma_unmap_sg(dev, data->sgl, data->size, dir);
1064814a0a4SEdward Tomasz Napierala }
1074814a0a4SEdward Tomasz Napierala
1084814a0a4SEdward Tomasz Napierala static int
iser_reg_dma(struct iser_device * device,struct iser_data_buf * mem,struct iser_mem_reg * reg)1094814a0a4SEdward Tomasz Napierala iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem,
1104814a0a4SEdward Tomasz Napierala struct iser_mem_reg *reg)
1114814a0a4SEdward Tomasz Napierala {
1124814a0a4SEdward Tomasz Napierala struct scatterlist *sg = mem->sgl;
1134814a0a4SEdward Tomasz Napierala
1144814a0a4SEdward Tomasz Napierala reg->sge.lkey = device->mr->lkey;
1154814a0a4SEdward Tomasz Napierala reg->rkey = device->mr->rkey;
1164814a0a4SEdward Tomasz Napierala reg->sge.length = ib_sg_dma_len(device->ib_device, &sg[0]);
1174814a0a4SEdward Tomasz Napierala reg->sge.addr = ib_sg_dma_address(device->ib_device, &sg[0]);
1184814a0a4SEdward Tomasz Napierala
1194814a0a4SEdward Tomasz Napierala return (0);
1204814a0a4SEdward Tomasz Napierala }
1214814a0a4SEdward Tomasz Napierala
1224814a0a4SEdward Tomasz Napierala /**
1234814a0a4SEdward Tomasz Napierala * TODO: This should be a verb
1244814a0a4SEdward Tomasz Napierala * iser_ib_inc_rkey - increments the key portion of the given rkey. Can be used
1254814a0a4SEdward Tomasz Napierala * for calculating a new rkey for type 2 memory windows.
1264814a0a4SEdward Tomasz Napierala * @rkey - the rkey to increment.
1274814a0a4SEdward Tomasz Napierala */
1284814a0a4SEdward Tomasz Napierala static inline u32
iser_ib_inc_rkey(u32 rkey)1294814a0a4SEdward Tomasz Napierala iser_ib_inc_rkey(u32 rkey)
1304814a0a4SEdward Tomasz Napierala {
1314814a0a4SEdward Tomasz Napierala const u32 mask = 0x000000ff;
1324814a0a4SEdward Tomasz Napierala
1334814a0a4SEdward Tomasz Napierala return (((rkey + 1) & mask) | (rkey & ~mask));
1344814a0a4SEdward Tomasz Napierala }
1354814a0a4SEdward Tomasz Napierala
1364814a0a4SEdward Tomasz Napierala static void
iser_inv_rkey(struct ib_send_wr * inv_wr,struct ib_mr * mr)1374814a0a4SEdward Tomasz Napierala iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr)
1384814a0a4SEdward Tomasz Napierala {
1394814a0a4SEdward Tomasz Napierala u32 rkey;
1404814a0a4SEdward Tomasz Napierala
1414814a0a4SEdward Tomasz Napierala memset(inv_wr, 0, sizeof(*inv_wr));
1424814a0a4SEdward Tomasz Napierala inv_wr->opcode = IB_WR_LOCAL_INV;
1434814a0a4SEdward Tomasz Napierala inv_wr->wr_id = ISER_FASTREG_LI_WRID;
1444814a0a4SEdward Tomasz Napierala inv_wr->ex.invalidate_rkey = mr->rkey;
1454814a0a4SEdward Tomasz Napierala
1464814a0a4SEdward Tomasz Napierala rkey = iser_ib_inc_rkey(mr->rkey);
1474814a0a4SEdward Tomasz Napierala ib_update_fast_reg_key(mr, rkey);
1484814a0a4SEdward Tomasz Napierala }
1494814a0a4SEdward Tomasz Napierala
1504814a0a4SEdward Tomasz Napierala static int
iser_fast_reg_mr(struct icl_iser_pdu * iser_pdu,struct iser_data_buf * mem,struct iser_reg_resources * rsc,struct iser_mem_reg * reg)1514814a0a4SEdward Tomasz Napierala iser_fast_reg_mr(struct icl_iser_pdu *iser_pdu,
1524814a0a4SEdward Tomasz Napierala struct iser_data_buf *mem,
1534814a0a4SEdward Tomasz Napierala struct iser_reg_resources *rsc,
1544814a0a4SEdward Tomasz Napierala struct iser_mem_reg *reg)
1554814a0a4SEdward Tomasz Napierala {
1564814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_pdu->iser_conn->ib_conn;
1574814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device;
15841dbd9ddSHans Petter Selasky struct ib_mr *mr = rsc->mr;
15941dbd9ddSHans Petter Selasky struct ib_reg_wr fastreg_wr;
16041dbd9ddSHans Petter Selasky struct ib_send_wr inv_wr;
161*c3987b8eSHans Petter Selasky const struct ib_send_wr *bad_wr;
162*c3987b8eSHans Petter Selasky struct ib_send_wr *wr = NULL;
16341dbd9ddSHans Petter Selasky int ret, n;
1644814a0a4SEdward Tomasz Napierala
1654814a0a4SEdward Tomasz Napierala /* if there a single dma entry, dma mr suffices */
1664814a0a4SEdward Tomasz Napierala if (mem->dma_nents == 1)
1674814a0a4SEdward Tomasz Napierala return iser_reg_dma(device, mem, reg);
1684814a0a4SEdward Tomasz Napierala
1694814a0a4SEdward Tomasz Napierala if (!rsc->mr_valid) {
17041dbd9ddSHans Petter Selasky iser_inv_rkey(&inv_wr, mr);
1714814a0a4SEdward Tomasz Napierala wr = &inv_wr;
1724814a0a4SEdward Tomasz Napierala }
1734814a0a4SEdward Tomasz Napierala
17441dbd9ddSHans Petter Selasky n = ib_map_mr_sg(mr, mem->sg, mem->size, NULL, SIZE_4K);
17541dbd9ddSHans Petter Selasky if (unlikely(n != mem->size)) {
17641dbd9ddSHans Petter Selasky ISER_ERR("failed to map sg (%d/%d)\n", n, mem->size);
17741dbd9ddSHans Petter Selasky return n < 0 ? n : -EINVAL;
17841dbd9ddSHans Petter Selasky }
1794814a0a4SEdward Tomasz Napierala /* Prepare FASTREG WR */
1804814a0a4SEdward Tomasz Napierala memset(&fastreg_wr, 0, sizeof(fastreg_wr));
18141dbd9ddSHans Petter Selasky fastreg_wr.wr.opcode = IB_WR_REG_MR;
18241dbd9ddSHans Petter Selasky fastreg_wr.wr.wr_id = ISER_FASTREG_LI_WRID;
18341dbd9ddSHans Petter Selasky fastreg_wr.wr.num_sge = 0;
18441dbd9ddSHans Petter Selasky fastreg_wr.mr = mr;
18541dbd9ddSHans Petter Selasky fastreg_wr.key = mr->rkey;
18641dbd9ddSHans Petter Selasky fastreg_wr.access = IB_ACCESS_LOCAL_WRITE |
1874814a0a4SEdward Tomasz Napierala IB_ACCESS_REMOTE_WRITE |
18841dbd9ddSHans Petter Selasky IB_ACCESS_REMOTE_READ;
1894814a0a4SEdward Tomasz Napierala
1904814a0a4SEdward Tomasz Napierala if (!wr)
19141dbd9ddSHans Petter Selasky wr = &fastreg_wr.wr;
1924814a0a4SEdward Tomasz Napierala else
19341dbd9ddSHans Petter Selasky wr->next = &fastreg_wr.wr;
1944814a0a4SEdward Tomasz Napierala
1954814a0a4SEdward Tomasz Napierala ret = ib_post_send(ib_conn->qp, wr, &bad_wr);
1964814a0a4SEdward Tomasz Napierala if (ret) {
1974814a0a4SEdward Tomasz Napierala ISER_ERR("fast registration failed, ret:%d", ret);
1984814a0a4SEdward Tomasz Napierala return (ret);
1994814a0a4SEdward Tomasz Napierala }
2004814a0a4SEdward Tomasz Napierala rsc->mr_valid = 0;
2014814a0a4SEdward Tomasz Napierala
20241dbd9ddSHans Petter Selasky reg->sge.lkey = mr->lkey;
20341dbd9ddSHans Petter Selasky reg->rkey = mr->rkey;
20441dbd9ddSHans Petter Selasky reg->sge.addr = mr->iova;
20541dbd9ddSHans Petter Selasky reg->sge.length = mr->length;
2064814a0a4SEdward Tomasz Napierala
2074814a0a4SEdward Tomasz Napierala return (ret);
2084814a0a4SEdward Tomasz Napierala }
2094814a0a4SEdward Tomasz Napierala
2104814a0a4SEdward Tomasz Napierala /**
2114814a0a4SEdward Tomasz Napierala * iser_reg_rdma_mem - Registers memory intended for RDMA,
2124814a0a4SEdward Tomasz Napierala * using Fast Registration WR (if possible) obtaining rkey and va
2134814a0a4SEdward Tomasz Napierala *
2144814a0a4SEdward Tomasz Napierala * returns 0 on success, errno code on failure
2154814a0a4SEdward Tomasz Napierala */
2164814a0a4SEdward Tomasz Napierala int
iser_reg_rdma_mem(struct icl_iser_pdu * iser_pdu,enum iser_data_dir cmd_dir)2174814a0a4SEdward Tomasz Napierala iser_reg_rdma_mem(struct icl_iser_pdu *iser_pdu,
2184814a0a4SEdward Tomasz Napierala enum iser_data_dir cmd_dir)
2194814a0a4SEdward Tomasz Napierala {
2204814a0a4SEdward Tomasz Napierala struct ib_conn *ib_conn = &iser_pdu->iser_conn->ib_conn;
2214814a0a4SEdward Tomasz Napierala struct iser_device *device = ib_conn->device;
2224814a0a4SEdward Tomasz Napierala struct ib_device *ibdev = device->ib_device;
2234814a0a4SEdward Tomasz Napierala struct iser_data_buf *mem = &iser_pdu->data[cmd_dir];
2244814a0a4SEdward Tomasz Napierala struct iser_mem_reg *mem_reg = &iser_pdu->rdma_reg[cmd_dir];
2254814a0a4SEdward Tomasz Napierala struct fast_reg_descriptor *desc = NULL;
2264814a0a4SEdward Tomasz Napierala int err, aligned_len;
2274814a0a4SEdward Tomasz Napierala
2284814a0a4SEdward Tomasz Napierala aligned_len = iser_data_buf_aligned_len(mem, ibdev);
2294814a0a4SEdward Tomasz Napierala if (aligned_len != mem->dma_nents) {
2304814a0a4SEdward Tomasz Napierala ISER_ERR("bounce buffer is not supported");
2314814a0a4SEdward Tomasz Napierala return 1;
2324814a0a4SEdward Tomasz Napierala }
2334814a0a4SEdward Tomasz Napierala
2344814a0a4SEdward Tomasz Napierala if (mem->dma_nents != 1) {
2354814a0a4SEdward Tomasz Napierala desc = iser_reg_desc_get(ib_conn);
2364814a0a4SEdward Tomasz Napierala mem_reg->mem_h = desc;
2374814a0a4SEdward Tomasz Napierala }
2384814a0a4SEdward Tomasz Napierala
2394814a0a4SEdward Tomasz Napierala err = iser_fast_reg_mr(iser_pdu, mem, desc ? &desc->rsc : NULL,
2404814a0a4SEdward Tomasz Napierala mem_reg);
2414814a0a4SEdward Tomasz Napierala if (err)
2424814a0a4SEdward Tomasz Napierala goto err_reg;
2434814a0a4SEdward Tomasz Napierala
2444814a0a4SEdward Tomasz Napierala return (0);
2454814a0a4SEdward Tomasz Napierala
2464814a0a4SEdward Tomasz Napierala err_reg:
2474814a0a4SEdward Tomasz Napierala if (desc)
2484814a0a4SEdward Tomasz Napierala iser_reg_desc_put(ib_conn, desc);
2494814a0a4SEdward Tomasz Napierala
2504814a0a4SEdward Tomasz Napierala return (err);
2514814a0a4SEdward Tomasz Napierala }
2524814a0a4SEdward Tomasz Napierala
2534814a0a4SEdward Tomasz Napierala void
iser_unreg_rdma_mem(struct icl_iser_pdu * iser_pdu,enum iser_data_dir cmd_dir)2544814a0a4SEdward Tomasz Napierala iser_unreg_rdma_mem(struct icl_iser_pdu *iser_pdu,
2554814a0a4SEdward Tomasz Napierala enum iser_data_dir cmd_dir)
2564814a0a4SEdward Tomasz Napierala {
2574814a0a4SEdward Tomasz Napierala struct iser_mem_reg *reg = &iser_pdu->rdma_reg[cmd_dir];
2584814a0a4SEdward Tomasz Napierala
2594814a0a4SEdward Tomasz Napierala if (!reg->mem_h)
2604814a0a4SEdward Tomasz Napierala return;
2614814a0a4SEdward Tomasz Napierala
2624814a0a4SEdward Tomasz Napierala iser_reg_desc_put(&iser_pdu->iser_conn->ib_conn,
2634814a0a4SEdward Tomasz Napierala reg->mem_h);
2644814a0a4SEdward Tomasz Napierala reg->mem_h = NULL;
2654814a0a4SEdward Tomasz Napierala }
2664814a0a4SEdward Tomasz Napierala
2674814a0a4SEdward Tomasz Napierala int
iser_dma_map_task_data(struct icl_iser_pdu * iser_pdu,struct iser_data_buf * data,enum iser_data_dir iser_dir,enum dma_data_direction dma_dir)2684814a0a4SEdward Tomasz Napierala iser_dma_map_task_data(struct icl_iser_pdu *iser_pdu,
2694814a0a4SEdward Tomasz Napierala struct iser_data_buf *data,
2704814a0a4SEdward Tomasz Napierala enum iser_data_dir iser_dir,
2714814a0a4SEdward Tomasz Napierala enum dma_data_direction dma_dir)
2724814a0a4SEdward Tomasz Napierala {
2734814a0a4SEdward Tomasz Napierala struct ib_device *dev;
2744814a0a4SEdward Tomasz Napierala
2754814a0a4SEdward Tomasz Napierala iser_pdu->dir[iser_dir] = 1;
2764814a0a4SEdward Tomasz Napierala dev = iser_pdu->iser_conn->ib_conn.device->ib_device;
2774814a0a4SEdward Tomasz Napierala
2784814a0a4SEdward Tomasz Napierala data->dma_nents = ib_dma_map_sg(dev, data->sgl, data->size, dma_dir);
2794814a0a4SEdward Tomasz Napierala if (data->dma_nents == 0) {
2804814a0a4SEdward Tomasz Napierala ISER_ERR("dma_map_sg failed");
2814814a0a4SEdward Tomasz Napierala return (EINVAL);
2824814a0a4SEdward Tomasz Napierala }
2834814a0a4SEdward Tomasz Napierala
2844814a0a4SEdward Tomasz Napierala return (0);
2854814a0a4SEdward Tomasz Napierala }
286