1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /******************************************************************************* 3 * SCSI RDMA Protocol lib functions 4 * 5 * Copyright (C) 2006 FUJITA Tomonori <tomof@acm.org> 6 * Copyright (C) 2016 Bryant G. Ly <bryantly@linux.vnet.ibm.com> IBM Corp. 7 * 8 ***********************************************************************/ 9 10 #define pr_fmt(fmt) "libsrp: " fmt 11 12 #include <linux/printk.h> 13 #include <linux/err.h> 14 #include <linux/slab.h> 15 #include <linux/kfifo.h> 16 #include <linux/scatterlist.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/module.h> 19 #include <scsi/srp.h> 20 #include <target/target_core_base.h> 21 #include "libsrp.h" 22 #include "ibmvscsi_tgt.h" 23 24 static int srp_iu_pool_alloc(struct srp_queue *q, size_t max, 25 struct srp_buf **ring) 26 { 27 struct iu_entry *iue; 28 int i; 29 30 q->pool = kcalloc(max, sizeof(struct iu_entry *), GFP_KERNEL); 31 if (!q->pool) 32 return -ENOMEM; 33 q->items = kcalloc(max, sizeof(struct iu_entry), GFP_KERNEL); 34 if (!q->items) 35 goto free_pool; 36 37 spin_lock_init(&q->lock); 38 kfifo_init(&q->queue, (void *)q->pool, max * sizeof(void *)); 39 40 for (i = 0, iue = q->items; i < max; i++) { 41 kfifo_in(&q->queue, (void *)&iue, sizeof(void *)); 42 iue->sbuf = ring[i]; 43 iue++; 44 } 45 return 0; 46 47 free_pool: 48 kfree(q->pool); 49 return -ENOMEM; 50 } 51 52 static void srp_iu_pool_free(struct srp_queue *q) 53 { 54 kfree(q->items); 55 kfree(q->pool); 56 } 57 58 static struct srp_buf **srp_ring_alloc(struct device *dev, 59 size_t max, size_t size) 60 { 61 struct srp_buf **ring; 62 int i; 63 64 ring = kcalloc(max, sizeof(struct srp_buf *), GFP_KERNEL); 65 if (!ring) 66 return NULL; 67 68 for (i = 0; i < max; i++) { 69 ring[i] = kzalloc(sizeof(*ring[i]), GFP_KERNEL); 70 if (!ring[i]) 71 goto out; 72 ring[i]->buf = dma_alloc_coherent(dev, size, &ring[i]->dma, 73 GFP_KERNEL); 74 if (!ring[i]->buf) 75 goto out; 76 } 77 return ring; 78 79 out: 80 for (i = 0; i < max && ring[i]; i++) { 81 if (ring[i]->buf) { 82 dma_free_coherent(dev, size, ring[i]->buf, 83 ring[i]->dma); 84 } 85 kfree(ring[i]); 86 } 87 kfree(ring); 88 89 return NULL; 90 } 91 92 static void srp_ring_free(struct device *dev, struct srp_buf **ring, 93 size_t max, size_t size) 94 { 95 int i; 96 97 for (i = 0; i < max; i++) { 98 dma_free_coherent(dev, size, ring[i]->buf, ring[i]->dma); 99 kfree(ring[i]); 100 } 101 kfree(ring); 102 } 103 104 int srp_target_alloc(struct srp_target *target, struct device *dev, 105 size_t nr, size_t iu_size) 106 { 107 int err; 108 109 spin_lock_init(&target->lock); 110 111 target->dev = dev; 112 113 target->srp_iu_size = iu_size; 114 target->rx_ring_size = nr; 115 target->rx_ring = srp_ring_alloc(target->dev, nr, iu_size); 116 if (!target->rx_ring) 117 return -ENOMEM; 118 err = srp_iu_pool_alloc(&target->iu_queue, nr, target->rx_ring); 119 if (err) 120 goto free_ring; 121 122 dev_set_drvdata(target->dev, target); 123 return 0; 124 125 free_ring: 126 srp_ring_free(target->dev, target->rx_ring, nr, iu_size); 127 return -ENOMEM; 128 } 129 130 void srp_target_free(struct srp_target *target) 131 { 132 dev_set_drvdata(target->dev, NULL); 133 srp_ring_free(target->dev, target->rx_ring, target->rx_ring_size, 134 target->srp_iu_size); 135 srp_iu_pool_free(&target->iu_queue); 136 } 137 138 struct iu_entry *srp_iu_get(struct srp_target *target) 139 { 140 struct iu_entry *iue = NULL; 141 142 if (kfifo_out_locked(&target->iu_queue.queue, (void *)&iue, 143 sizeof(void *), 144 &target->iu_queue.lock) != sizeof(void *)) { 145 WARN_ONCE(1, "unexpected fifo state"); 146 return NULL; 147 } 148 if (!iue) 149 return iue; 150 iue->target = target; 151 iue->flags = 0; 152 return iue; 153 } 154 155 void srp_iu_put(struct iu_entry *iue) 156 { 157 kfifo_in_locked(&iue->target->iu_queue.queue, (void *)&iue, 158 sizeof(void *), &iue->target->iu_queue.lock); 159 } 160 161 static int srp_direct_data(struct ibmvscsis_cmd *cmd, struct srp_direct_buf *md, 162 enum dma_data_direction dir, srp_rdma_t rdma_io, 163 int dma_map, int ext_desc) 164 { 165 struct iu_entry *iue = NULL; 166 struct scatterlist *sg = NULL; 167 int err, nsg = 0, len; 168 169 if (dma_map) { 170 iue = cmd->iue; 171 sg = cmd->se_cmd.t_data_sg; 172 nsg = dma_map_sg(iue->target->dev, sg, cmd->se_cmd.t_data_nents, 173 DMA_BIDIRECTIONAL); 174 if (!nsg) { 175 pr_err("fail to map %p %d\n", iue, 176 cmd->se_cmd.t_data_nents); 177 return 0; 178 } 179 len = min(cmd->se_cmd.data_length, be32_to_cpu(md->len)); 180 } else { 181 len = be32_to_cpu(md->len); 182 } 183 184 err = rdma_io(cmd, sg, nsg, md, 1, dir, len); 185 186 if (dma_map) 187 dma_unmap_sg(iue->target->dev, sg, cmd->se_cmd.t_data_nents, 188 DMA_BIDIRECTIONAL); 189 190 return err; 191 } 192 193 static int srp_indirect_data(struct ibmvscsis_cmd *cmd, struct srp_cmd *srp_cmd, 194 struct srp_indirect_buf *id, 195 enum dma_data_direction dir, srp_rdma_t rdma_io, 196 int dma_map, int ext_desc) 197 { 198 struct iu_entry *iue = NULL; 199 struct srp_direct_buf *md = NULL; 200 struct scatterlist dummy, *sg = NULL; 201 dma_addr_t token = 0; 202 int err = 0; 203 int nmd, nsg = 0, len; 204 205 if (dma_map || ext_desc) { 206 iue = cmd->iue; 207 sg = cmd->se_cmd.t_data_sg; 208 } 209 210 nmd = be32_to_cpu(id->table_desc.len) / sizeof(struct srp_direct_buf); 211 212 if ((dir == DMA_FROM_DEVICE && nmd == srp_cmd->data_in_desc_cnt) || 213 (dir == DMA_TO_DEVICE && nmd == srp_cmd->data_out_desc_cnt)) { 214 md = &id->desc_list[0]; 215 goto rdma; 216 } 217 218 if (ext_desc && dma_map) { 219 md = dma_alloc_coherent(iue->target->dev, 220 be32_to_cpu(id->table_desc.len), 221 &token, GFP_KERNEL); 222 if (!md) { 223 pr_err("Can't get dma memory %u\n", 224 be32_to_cpu(id->table_desc.len)); 225 return -ENOMEM; 226 } 227 228 sg_init_one(&dummy, md, be32_to_cpu(id->table_desc.len)); 229 sg_dma_address(&dummy) = token; 230 sg_dma_len(&dummy) = be32_to_cpu(id->table_desc.len); 231 err = rdma_io(cmd, &dummy, 1, &id->table_desc, 1, DMA_TO_DEVICE, 232 be32_to_cpu(id->table_desc.len)); 233 if (err) { 234 pr_err("Error copying indirect table %d\n", err); 235 goto free_mem; 236 } 237 } else { 238 pr_err("This command uses external indirect buffer\n"); 239 return -EINVAL; 240 } 241 242 rdma: 243 if (dma_map) { 244 nsg = dma_map_sg(iue->target->dev, sg, cmd->se_cmd.t_data_nents, 245 DMA_BIDIRECTIONAL); 246 if (!nsg) { 247 pr_err("fail to map %p %d\n", iue, 248 cmd->se_cmd.t_data_nents); 249 err = -EIO; 250 goto free_mem; 251 } 252 len = min(cmd->se_cmd.data_length, be32_to_cpu(id->len)); 253 } else { 254 len = be32_to_cpu(id->len); 255 } 256 257 err = rdma_io(cmd, sg, nsg, md, nmd, dir, len); 258 259 if (dma_map) 260 dma_unmap_sg(iue->target->dev, sg, cmd->se_cmd.t_data_nents, 261 DMA_BIDIRECTIONAL); 262 263 free_mem: 264 if (token && dma_map) { 265 dma_free_coherent(iue->target->dev, 266 be32_to_cpu(id->table_desc.len), md, token); 267 } 268 return err; 269 } 270 271 static int data_out_desc_size(struct srp_cmd *cmd) 272 { 273 int size = 0; 274 u8 fmt = cmd->buf_fmt >> 4; 275 276 switch (fmt) { 277 case SRP_NO_DATA_DESC: 278 break; 279 case SRP_DATA_DESC_DIRECT: 280 size = sizeof(struct srp_direct_buf); 281 break; 282 case SRP_DATA_DESC_INDIRECT: 283 size = sizeof(struct srp_indirect_buf) + 284 sizeof(struct srp_direct_buf) * cmd->data_out_desc_cnt; 285 break; 286 default: 287 pr_err("client error. Invalid data_out_format %x\n", fmt); 288 break; 289 } 290 return size; 291 } 292 293 /* 294 * TODO: this can be called multiple times for a single command if it 295 * has very long data. 296 */ 297 int srp_transfer_data(struct ibmvscsis_cmd *cmd, struct srp_cmd *srp_cmd, 298 srp_rdma_t rdma_io, int dma_map, int ext_desc) 299 { 300 struct srp_direct_buf *md; 301 struct srp_indirect_buf *id; 302 enum dma_data_direction dir; 303 int offset, err = 0; 304 u8 format; 305 306 if (!cmd->se_cmd.t_data_nents) 307 return 0; 308 309 offset = srp_cmd->add_cdb_len & ~3; 310 311 dir = srp_cmd_direction(srp_cmd); 312 if (dir == DMA_FROM_DEVICE) 313 offset += data_out_desc_size(srp_cmd); 314 315 if (dir == DMA_TO_DEVICE) 316 format = srp_cmd->buf_fmt >> 4; 317 else 318 format = srp_cmd->buf_fmt & ((1U << 4) - 1); 319 320 switch (format) { 321 case SRP_NO_DATA_DESC: 322 break; 323 case SRP_DATA_DESC_DIRECT: 324 md = (struct srp_direct_buf *)(srp_cmd->add_data + offset); 325 err = srp_direct_data(cmd, md, dir, rdma_io, dma_map, ext_desc); 326 break; 327 case SRP_DATA_DESC_INDIRECT: 328 id = (struct srp_indirect_buf *)(srp_cmd->add_data + offset); 329 err = srp_indirect_data(cmd, srp_cmd, id, dir, rdma_io, dma_map, 330 ext_desc); 331 break; 332 default: 333 pr_err("Unknown format %d %x\n", dir, format); 334 err = -EINVAL; 335 } 336 337 return err; 338 } 339 340 u64 srp_data_length(struct srp_cmd *cmd, enum dma_data_direction dir) 341 { 342 struct srp_direct_buf *md; 343 struct srp_indirect_buf *id; 344 u64 len = 0; 345 uint offset = cmd->add_cdb_len & ~3; 346 u8 fmt; 347 348 if (dir == DMA_TO_DEVICE) { 349 fmt = cmd->buf_fmt >> 4; 350 } else { 351 fmt = cmd->buf_fmt & ((1U << 4) - 1); 352 offset += data_out_desc_size(cmd); 353 } 354 355 switch (fmt) { 356 case SRP_NO_DATA_DESC: 357 break; 358 case SRP_DATA_DESC_DIRECT: 359 md = (struct srp_direct_buf *)(cmd->add_data + offset); 360 len = be32_to_cpu(md->len); 361 break; 362 case SRP_DATA_DESC_INDIRECT: 363 id = (struct srp_indirect_buf *)(cmd->add_data + offset); 364 len = be32_to_cpu(id->len); 365 break; 366 default: 367 pr_err("invalid data format %x\n", fmt); 368 break; 369 } 370 return len; 371 } 372 373 int srp_get_desc_table(struct srp_cmd *srp_cmd, enum dma_data_direction *dir, 374 u64 *data_len) 375 { 376 struct srp_indirect_buf *idb; 377 struct srp_direct_buf *db; 378 uint add_cdb_offset; 379 int rc; 380 381 /* 382 * The pointer computations below will only be compiled correctly 383 * if srp_cmd::add_data is declared as s8*, u8*, s8[] or u8[], so check 384 * whether srp_cmd::add_data has been declared as a byte pointer. 385 */ 386 BUILD_BUG_ON(!__same_type(srp_cmd->add_data[0], (s8)0) 387 && !__same_type(srp_cmd->add_data[0], (u8)0)); 388 389 BUG_ON(!dir); 390 BUG_ON(!data_len); 391 392 rc = 0; 393 *data_len = 0; 394 395 *dir = DMA_NONE; 396 397 if (srp_cmd->buf_fmt & 0xf) 398 *dir = DMA_FROM_DEVICE; 399 else if (srp_cmd->buf_fmt >> 4) 400 *dir = DMA_TO_DEVICE; 401 402 add_cdb_offset = srp_cmd->add_cdb_len & ~3; 403 if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) || 404 ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) { 405 db = (struct srp_direct_buf *)(srp_cmd->add_data 406 + add_cdb_offset); 407 *data_len = be32_to_cpu(db->len); 408 } else if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_INDIRECT) || 409 ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_INDIRECT)) { 410 idb = (struct srp_indirect_buf *)(srp_cmd->add_data 411 + add_cdb_offset); 412 413 *data_len = be32_to_cpu(idb->len); 414 } 415 return rc; 416 } 417 418 MODULE_DESCRIPTION("SCSI RDMA Protocol lib functions"); 419 MODULE_AUTHOR("FUJITA Tomonori"); 420 MODULE_LICENSE("GPL"); 421