1 /* 2 * Copyright (c) 2016 Chelsio Communications, Inc. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #include "cxgbit.h" 10 11 static void 12 cxgbit_set_one_ppod(struct cxgbi_pagepod *ppod, 13 struct cxgbi_task_tag_info *ttinfo, 14 struct scatterlist **sg_pp, unsigned int *sg_off) 15 { 16 struct scatterlist *sg = sg_pp ? *sg_pp : NULL; 17 unsigned int offset = sg_off ? *sg_off : 0; 18 dma_addr_t addr = 0UL; 19 unsigned int len = 0; 20 int i; 21 22 memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr)); 23 24 if (sg) { 25 addr = sg_dma_address(sg); 26 len = sg_dma_len(sg); 27 } 28 29 for (i = 0; i < PPOD_PAGES_MAX; i++) { 30 if (sg) { 31 ppod->addr[i] = cpu_to_be64(addr + offset); 32 offset += PAGE_SIZE; 33 if (offset == (len + sg->offset)) { 34 offset = 0; 35 sg = sg_next(sg); 36 if (sg) { 37 addr = sg_dma_address(sg); 38 len = sg_dma_len(sg); 39 } 40 } 41 } else { 42 ppod->addr[i] = 0ULL; 43 } 44 } 45 46 /* 47 * the fifth address needs to be repeated in the next ppod, so do 48 * not move sg 49 */ 50 if (sg_pp) { 51 *sg_pp = sg; 52 *sg_off = offset; 53 } 54 55 if (offset == len) { 56 offset = 0; 57 if (sg) { 58 sg = sg_next(sg); 59 if (sg) 60 addr = sg_dma_address(sg); 61 } 62 } 63 ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL; 64 } 65 66 static struct sk_buff * 67 cxgbit_ppod_init_idata(struct cxgbit_device *cdev, struct cxgbi_ppm *ppm, 68 unsigned int idx, unsigned int npods, unsigned int tid) 69 { 70 struct ulp_mem_io *req; 71 struct ulptx_idata *idata; 72 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit; 73 unsigned int dlen = npods << PPOD_SIZE_SHIFT; 74 unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) + 75 sizeof(struct ulptx_idata) + dlen, 16); 76 struct sk_buff *skb; 77 78 skb = alloc_skb(wr_len, GFP_KERNEL); 79 if (!skb) 80 return NULL; 81 82 req = __skb_put(skb, wr_len); 83 INIT_ULPTX_WR(req, wr_len, 0, tid); 84 req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) | 85 FW_WR_ATOMIC_V(0)); 86 req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) | 87 ULP_MEMIO_ORDER_V(0) | 88 T5_ULP_MEMIO_IMM_V(1)); 89 req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5)); 90 req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5)); 91 req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16)); 92 93 idata = (struct ulptx_idata *)(req + 1); 94 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); 95 idata->len = htonl(dlen); 96 97 return skb; 98 } 99 100 static int 101 cxgbit_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk, 102 struct cxgbi_task_tag_info *ttinfo, unsigned int idx, 103 unsigned int npods, struct scatterlist **sg_pp, 104 unsigned int *sg_off) 105 { 106 struct cxgbit_device *cdev = csk->com.cdev; 107 struct sk_buff *skb; 108 struct ulp_mem_io *req; 109 struct ulptx_idata *idata; 110 struct cxgbi_pagepod *ppod; 111 unsigned int i; 112 113 skb = cxgbit_ppod_init_idata(cdev, ppm, idx, npods, csk->tid); 114 if (!skb) 115 return -ENOMEM; 116 117 req = (struct ulp_mem_io *)skb->data; 118 idata = (struct ulptx_idata *)(req + 1); 119 ppod = (struct cxgbi_pagepod *)(idata + 1); 120 121 for (i = 0; i < npods; i++, ppod++) 122 cxgbit_set_one_ppod(ppod, ttinfo, sg_pp, sg_off); 123 124 __skb_queue_tail(&csk->ppodq, skb); 125 126 return 0; 127 } 128 129 static int 130 cxgbit_ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbit_sock *csk, 131 struct cxgbi_task_tag_info *ttinfo) 132 { 133 unsigned int pidx = ttinfo->idx; 134 unsigned int npods = ttinfo->npods; 135 unsigned int i, cnt; 136 struct scatterlist *sg = ttinfo->sgl; 137 unsigned int offset = 0; 138 int ret = 0; 139 140 for (i = 0; i < npods; i += cnt, pidx += cnt) { 141 cnt = npods - i; 142 143 if (cnt > ULPMEM_IDATA_MAX_NPPODS) 144 cnt = ULPMEM_IDATA_MAX_NPPODS; 145 146 ret = cxgbit_ppod_write_idata(ppm, csk, ttinfo, pidx, cnt, 147 &sg, &offset); 148 if (ret < 0) 149 break; 150 } 151 152 return ret; 153 } 154 155 static int cxgbit_ddp_sgl_check(struct scatterlist *sg, 156 unsigned int nents) 157 { 158 unsigned int last_sgidx = nents - 1; 159 unsigned int i; 160 161 for (i = 0; i < nents; i++, sg = sg_next(sg)) { 162 unsigned int len = sg->length + sg->offset; 163 164 if ((sg->offset & 0x3) || (i && sg->offset) || 165 ((i != last_sgidx) && (len != PAGE_SIZE))) { 166 return -EINVAL; 167 } 168 } 169 170 return 0; 171 } 172 173 static int 174 cxgbit_ddp_reserve(struct cxgbit_sock *csk, struct cxgbi_task_tag_info *ttinfo, 175 unsigned int xferlen) 176 { 177 struct cxgbit_device *cdev = csk->com.cdev; 178 struct cxgbi_ppm *ppm = cdev2ppm(cdev); 179 struct scatterlist *sgl = ttinfo->sgl; 180 unsigned int sgcnt = ttinfo->nents; 181 unsigned int sg_offset = sgl->offset; 182 int ret; 183 184 if ((xferlen < DDP_THRESHOLD) || (!sgcnt)) { 185 pr_debug("ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n", 186 ppm, ppm->tformat.pgsz_idx_dflt, 187 xferlen, ttinfo->nents); 188 return -EINVAL; 189 } 190 191 if (cxgbit_ddp_sgl_check(sgl, sgcnt) < 0) 192 return -EINVAL; 193 194 ttinfo->nr_pages = (xferlen + sgl->offset + 195 (1 << PAGE_SHIFT) - 1) >> PAGE_SHIFT; 196 197 /* 198 * the ddp tag will be used for the ttt in the outgoing r2t pdu 199 */ 200 ret = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx, 201 &ttinfo->tag, 0); 202 if (ret < 0) 203 return ret; 204 ttinfo->npods = ret; 205 206 sgl->offset = 0; 207 ret = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE); 208 sgl->offset = sg_offset; 209 if (!ret) { 210 pr_info("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n", 211 __func__, 0, xferlen, sgcnt); 212 goto rel_ppods; 213 } 214 215 cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset, 216 xferlen, &ttinfo->hdr); 217 218 ret = cxgbit_ddp_set_map(ppm, csk, ttinfo); 219 if (ret < 0) { 220 __skb_queue_purge(&csk->ppodq); 221 dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE); 222 goto rel_ppods; 223 } 224 225 return 0; 226 227 rel_ppods: 228 cxgbi_ppm_ppod_release(ppm, ttinfo->idx); 229 return -EINVAL; 230 } 231 232 void 233 cxgbit_get_r2t_ttt(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 234 struct iscsi_r2t *r2t) 235 { 236 struct cxgbit_sock *csk = conn->context; 237 struct cxgbit_device *cdev = csk->com.cdev; 238 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 239 struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo; 240 int ret = -EINVAL; 241 242 if ((!ccmd->setup_ddp) || 243 (!test_bit(CSK_DDP_ENABLE, &csk->com.flags))) 244 goto out; 245 246 ccmd->setup_ddp = false; 247 248 ttinfo->sgl = cmd->se_cmd.t_data_sg; 249 ttinfo->nents = cmd->se_cmd.t_data_nents; 250 251 ret = cxgbit_ddp_reserve(csk, ttinfo, cmd->se_cmd.data_length); 252 if (ret < 0) { 253 pr_info("csk 0x%p, cmd 0x%p, xfer len %u, sgcnt %u no ddp.\n", 254 csk, cmd, cmd->se_cmd.data_length, ttinfo->nents); 255 256 ttinfo->sgl = NULL; 257 ttinfo->nents = 0; 258 } else { 259 ccmd->release = true; 260 } 261 out: 262 pr_debug("cdev 0x%p, cmd 0x%p, tag 0x%x\n", cdev, cmd, ttinfo->tag); 263 r2t->targ_xfer_tag = ttinfo->tag; 264 } 265 266 void cxgbit_release_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 267 { 268 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 269 270 if (ccmd->release) { 271 struct cxgbi_task_tag_info *ttinfo = &ccmd->ttinfo; 272 273 if (ttinfo->sgl) { 274 struct cxgbit_sock *csk = conn->context; 275 struct cxgbit_device *cdev = csk->com.cdev; 276 struct cxgbi_ppm *ppm = cdev2ppm(cdev); 277 278 /* Abort the TCP conn if DDP is not complete to 279 * avoid any possibility of DDP after freeing 280 * the cmd. 281 */ 282 if (unlikely(cmd->write_data_done != 283 cmd->se_cmd.data_length)) 284 cxgbit_abort_conn(csk); 285 286 cxgbi_ppm_ppod_release(ppm, ttinfo->idx); 287 288 dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl, 289 ttinfo->nents, DMA_FROM_DEVICE); 290 } else { 291 put_page(sg_page(&ccmd->sg)); 292 } 293 294 ccmd->release = false; 295 } 296 } 297 298 int cxgbit_ddp_init(struct cxgbit_device *cdev) 299 { 300 struct cxgb4_lld_info *lldi = &cdev->lldi; 301 struct net_device *ndev = cdev->lldi.ports[0]; 302 struct cxgbi_tag_format tformat; 303 unsigned int ppmax; 304 int ret, i; 305 306 if (!lldi->vr->iscsi.size) { 307 pr_warn("%s, iscsi NOT enabled, check config!\n", ndev->name); 308 return -EACCES; 309 } 310 311 ppmax = lldi->vr->iscsi.size >> PPOD_SIZE_SHIFT; 312 313 memset(&tformat, 0, sizeof(struct cxgbi_tag_format)); 314 for (i = 0; i < 4; i++) 315 tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3)) 316 & 0xF; 317 cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat); 318 319 ret = cxgbi_ppm_init(lldi->iscsi_ppm, cdev->lldi.ports[0], 320 cdev->lldi.pdev, &cdev->lldi, &tformat, 321 ppmax, lldi->iscsi_llimit, 322 lldi->vr->iscsi.start, 2); 323 if (ret >= 0) { 324 struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*lldi->iscsi_ppm); 325 326 if ((ppm->tformat.pgsz_idx_dflt < DDP_PGIDX_MAX) && 327 (ppm->ppmax >= 1024)) 328 set_bit(CDEV_DDP_ENABLE, &cdev->flags); 329 ret = 0; 330 } 331 332 return ret; 333 } 334