1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * VDPA simulator for block device.
4 *
5 * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
6 * Copyright (c) 2021, Red Hat Inc. All rights reserved.
7 *
8 */
9
10 #include <linux/init.h>
11 #include <linux/module.h>
12 #include <linux/device.h>
13 #include <linux/kernel.h>
14 #include <linux/blkdev.h>
15 #include <linux/vringh.h>
16 #include <linux/vdpa.h>
17 #include <uapi/linux/virtio_blk.h>
18
19 #include "vdpa_sim.h"
20
21 #define DRV_VERSION "0.1"
22 #define DRV_AUTHOR "Max Gurtovoy <mgurtovoy@nvidia.com>"
23 #define DRV_DESC "vDPA Device Simulator for block device"
24 #define DRV_LICENSE "GPL v2"
25
26 #define VDPASIM_BLK_FEATURES (VDPASIM_FEATURES | \
27 (1ULL << VIRTIO_BLK_F_FLUSH) | \
28 (1ULL << VIRTIO_BLK_F_SIZE_MAX) | \
29 (1ULL << VIRTIO_BLK_F_SEG_MAX) | \
30 (1ULL << VIRTIO_BLK_F_BLK_SIZE) | \
31 (1ULL << VIRTIO_BLK_F_TOPOLOGY) | \
32 (1ULL << VIRTIO_BLK_F_MQ) | \
33 (1ULL << VIRTIO_BLK_F_DISCARD) | \
34 (1ULL << VIRTIO_BLK_F_WRITE_ZEROES))
35
36 #define VDPASIM_BLK_CAPACITY 0x40000
37 #define VDPASIM_BLK_SIZE_MAX 0x1000
38 #define VDPASIM_BLK_SEG_MAX 32
39 #define VDPASIM_BLK_DWZ_MAX_SECTORS UINT_MAX
40
41 /* 1 virtqueue, 1 address space, 1 virtqueue group */
42 #define VDPASIM_BLK_VQ_NUM 1
43 #define VDPASIM_BLK_AS_NUM 1
44 #define VDPASIM_BLK_GROUP_NUM 1
45
46 struct vdpasim_blk {
47 struct vdpasim vdpasim;
48 void *buffer;
49 bool shared_backend;
50 };
51
sim_to_blk(struct vdpasim * vdpasim)52 static struct vdpasim_blk *sim_to_blk(struct vdpasim *vdpasim)
53 {
54 return container_of(vdpasim, struct vdpasim_blk, vdpasim);
55 }
56
57 static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim";
58
59 static bool shared_backend;
60 module_param(shared_backend, bool, 0444);
61 MODULE_PARM_DESC(shared_backend, "Enable the shared backend between virtio-blk devices");
62
63 static void *shared_buffer;
64 /* mutex to synchronize shared_buffer access */
65 static DEFINE_MUTEX(shared_buffer_mutex);
66
vdpasim_blk_buffer_lock(struct vdpasim_blk * blk)67 static void vdpasim_blk_buffer_lock(struct vdpasim_blk *blk)
68 {
69 if (blk->shared_backend)
70 mutex_lock(&shared_buffer_mutex);
71 }
72
vdpasim_blk_buffer_unlock(struct vdpasim_blk * blk)73 static void vdpasim_blk_buffer_unlock(struct vdpasim_blk *blk)
74 {
75 if (blk->shared_backend)
76 mutex_unlock(&shared_buffer_mutex);
77 }
78
vdpasim_blk_check_range(struct vdpasim * vdpasim,u64 start_sector,u64 num_sectors,u64 max_sectors)79 static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector,
80 u64 num_sectors, u64 max_sectors)
81 {
82 if (start_sector >= VDPASIM_BLK_CAPACITY) {
83 dev_dbg(&vdpasim->vdpa.dev,
84 "starting sector exceeds the capacity - start: 0x%llx capacity: 0x%x\n",
85 start_sector, VDPASIM_BLK_CAPACITY);
86 return false;
87 }
88
89 if (num_sectors > max_sectors) {
90 dev_dbg(&vdpasim->vdpa.dev,
91 "number of sectors exceeds the max allowed in a request - num: 0x%llx max: 0x%llx\n",
92 num_sectors, max_sectors);
93 return false;
94 }
95
96 if (num_sectors > VDPASIM_BLK_CAPACITY - start_sector) {
97 dev_dbg(&vdpasim->vdpa.dev,
98 "request exceeds the capacity - start: 0x%llx num: 0x%llx capacity: 0x%x\n",
99 start_sector, num_sectors, VDPASIM_BLK_CAPACITY);
100 return false;
101 }
102
103 return true;
104 }
105
106 /* Returns 'true' if the request is handled (with or without an I/O error)
107 * and the status is correctly written in the last byte of the 'in iov',
108 * 'false' otherwise.
109 */
vdpasim_blk_handle_req(struct vdpasim * vdpasim,struct vdpasim_virtqueue * vq)110 static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim,
111 struct vdpasim_virtqueue *vq)
112 {
113 struct vdpasim_blk *blk = sim_to_blk(vdpasim);
114 size_t pushed = 0, to_pull, to_push;
115 struct virtio_blk_outhdr hdr;
116 bool handled = false;
117 ssize_t bytes;
118 loff_t offset;
119 u64 sector;
120 u8 status;
121 u32 type;
122 int ret;
123
124 ret = vringh_getdesc_iotlb(&vq->vring, &vq->out_iov, &vq->in_iov,
125 &vq->head, GFP_ATOMIC);
126 if (ret != 1)
127 return false;
128
129 if (vq->out_iov.used < 1 || vq->in_iov.used < 1) {
130 dev_dbg(&vdpasim->vdpa.dev, "missing headers - out_iov: %u in_iov %u\n",
131 vq->out_iov.used, vq->in_iov.used);
132 goto err;
133 }
134
135 if (vq->in_iov.iov[vq->in_iov.used - 1].iov_len < 1) {
136 dev_dbg(&vdpasim->vdpa.dev, "request in header too short\n");
137 goto err;
138 }
139
140 /* The last byte is the status and we checked if the last iov has
141 * enough room for it.
142 */
143 to_push = vringh_kiov_length(&vq->in_iov) - 1;
144
145 to_pull = vringh_kiov_length(&vq->out_iov);
146
147 bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &hdr,
148 sizeof(hdr));
149 if (bytes != sizeof(hdr)) {
150 dev_dbg(&vdpasim->vdpa.dev, "request out header too short\n");
151 goto err;
152 }
153
154 to_pull -= bytes;
155
156 type = vdpasim32_to_cpu(vdpasim, hdr.type);
157 sector = vdpasim64_to_cpu(vdpasim, hdr.sector);
158 offset = sector << SECTOR_SHIFT;
159 status = VIRTIO_BLK_S_OK;
160
161 if (type != VIRTIO_BLK_T_IN && type != VIRTIO_BLK_T_OUT &&
162 sector != 0) {
163 dev_dbg(&vdpasim->vdpa.dev,
164 "sector must be 0 for %u request - sector: 0x%llx\n",
165 type, sector);
166 status = VIRTIO_BLK_S_IOERR;
167 goto err_status;
168 }
169
170 switch (type) {
171 case VIRTIO_BLK_T_IN:
172 if (!vdpasim_blk_check_range(vdpasim, sector,
173 to_push >> SECTOR_SHIFT,
174 VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) {
175 status = VIRTIO_BLK_S_IOERR;
176 break;
177 }
178
179 vdpasim_blk_buffer_lock(blk);
180 bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov,
181 blk->buffer + offset, to_push);
182 vdpasim_blk_buffer_unlock(blk);
183 if (bytes < 0) {
184 dev_dbg(&vdpasim->vdpa.dev,
185 "vringh_iov_push_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
186 bytes, offset, to_push);
187 status = VIRTIO_BLK_S_IOERR;
188 break;
189 }
190
191 pushed += bytes;
192 break;
193
194 case VIRTIO_BLK_T_OUT:
195 if (!vdpasim_blk_check_range(vdpasim, sector,
196 to_pull >> SECTOR_SHIFT,
197 VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) {
198 status = VIRTIO_BLK_S_IOERR;
199 break;
200 }
201
202 vdpasim_blk_buffer_lock(blk);
203 bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov,
204 blk->buffer + offset, to_pull);
205 vdpasim_blk_buffer_unlock(blk);
206 if (bytes < 0) {
207 dev_dbg(&vdpasim->vdpa.dev,
208 "vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
209 bytes, offset, to_pull);
210 status = VIRTIO_BLK_S_IOERR;
211 break;
212 }
213 break;
214
215 case VIRTIO_BLK_T_GET_ID:
216 bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov,
217 vdpasim_blk_id,
218 VIRTIO_BLK_ID_BYTES);
219 if (bytes < 0) {
220 dev_dbg(&vdpasim->vdpa.dev,
221 "vringh_iov_push_iotlb() error: %zd\n", bytes);
222 status = VIRTIO_BLK_S_IOERR;
223 break;
224 }
225
226 pushed += bytes;
227 break;
228
229 case VIRTIO_BLK_T_FLUSH:
230 /* nothing to do */
231 break;
232
233 case VIRTIO_BLK_T_DISCARD:
234 case VIRTIO_BLK_T_WRITE_ZEROES: {
235 struct virtio_blk_discard_write_zeroes range;
236 u32 num_sectors, flags;
237
238 if (to_pull != sizeof(range)) {
239 dev_dbg(&vdpasim->vdpa.dev,
240 "discard/write_zeroes header len: 0x%zx [expected: 0x%zx]\n",
241 to_pull, sizeof(range));
242 status = VIRTIO_BLK_S_IOERR;
243 break;
244 }
245
246 bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &range,
247 to_pull);
248 if (bytes < 0) {
249 dev_dbg(&vdpasim->vdpa.dev,
250 "vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n",
251 bytes, offset, to_pull);
252 status = VIRTIO_BLK_S_IOERR;
253 break;
254 }
255
256 sector = le64_to_cpu(range.sector);
257 offset = sector << SECTOR_SHIFT;
258 num_sectors = le32_to_cpu(range.num_sectors);
259 flags = le32_to_cpu(range.flags);
260
261 if (type == VIRTIO_BLK_T_DISCARD && flags != 0) {
262 dev_dbg(&vdpasim->vdpa.dev,
263 "discard unexpected flags set - flags: 0x%x\n",
264 flags);
265 status = VIRTIO_BLK_S_UNSUPP;
266 break;
267 }
268
269 if (type == VIRTIO_BLK_T_WRITE_ZEROES &&
270 flags & ~VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP) {
271 dev_dbg(&vdpasim->vdpa.dev,
272 "write_zeroes unexpected flags set - flags: 0x%x\n",
273 flags);
274 status = VIRTIO_BLK_S_UNSUPP;
275 break;
276 }
277
278 if (!vdpasim_blk_check_range(vdpasim, sector, num_sectors,
279 VDPASIM_BLK_DWZ_MAX_SECTORS)) {
280 status = VIRTIO_BLK_S_IOERR;
281 break;
282 }
283
284 if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
285 vdpasim_blk_buffer_lock(blk);
286 memset(blk->buffer + offset, 0,
287 num_sectors << SECTOR_SHIFT);
288 vdpasim_blk_buffer_unlock(blk);
289 }
290
291 break;
292 }
293 default:
294 dev_dbg(&vdpasim->vdpa.dev,
295 "Unsupported request type %d\n", type);
296 status = VIRTIO_BLK_S_IOERR;
297 break;
298 }
299
300 err_status:
301 /* If some operations fail, we need to skip the remaining bytes
302 * to put the status in the last byte
303 */
304 if (to_push - pushed > 0)
305 vringh_kiov_advance(&vq->in_iov, to_push - pushed);
306
307 /* Last byte is the status */
308 bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov, &status, 1);
309 if (bytes != 1)
310 goto err;
311
312 pushed += bytes;
313
314 /* Make sure data is wrote before advancing index */
315 smp_wmb();
316
317 handled = true;
318
319 err:
320 vringh_complete_iotlb(&vq->vring, vq->head, pushed);
321
322 return handled;
323 }
324
vdpasim_blk_work(struct vdpasim * vdpasim)325 static void vdpasim_blk_work(struct vdpasim *vdpasim)
326 {
327 bool reschedule = false;
328 int i;
329
330 mutex_lock(&vdpasim->mutex);
331
332 if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
333 goto out;
334
335 if (!vdpasim->running)
336 goto out;
337
338 for (i = 0; i < VDPASIM_BLK_VQ_NUM; i++) {
339 struct vdpasim_virtqueue *vq = &vdpasim->vqs[i];
340 int reqs = 0;
341
342 if (!vq->ready)
343 continue;
344
345 while (vdpasim_blk_handle_req(vdpasim, vq)) {
346 /* Make sure used is visible before rasing the interrupt. */
347 smp_wmb();
348
349 local_bh_disable();
350 if (vringh_need_notify_iotlb(&vq->vring) > 0)
351 vringh_notify(&vq->vring);
352 local_bh_enable();
353
354 if (++reqs > 4) {
355 reschedule = true;
356 break;
357 }
358 }
359 }
360 out:
361 mutex_unlock(&vdpasim->mutex);
362
363 if (reschedule)
364 vdpasim_schedule_work(vdpasim);
365 }
366
vdpasim_blk_get_config(struct vdpasim * vdpasim,void * config)367 static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config)
368 {
369 struct virtio_blk_config *blk_config = config;
370
371 memset(config, 0, sizeof(struct virtio_blk_config));
372
373 blk_config->capacity = cpu_to_vdpasim64(vdpasim, VDPASIM_BLK_CAPACITY);
374 blk_config->size_max = cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_SIZE_MAX);
375 blk_config->seg_max = cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_SEG_MAX);
376 blk_config->num_queues = cpu_to_vdpasim16(vdpasim, VDPASIM_BLK_VQ_NUM);
377 blk_config->min_io_size = cpu_to_vdpasim16(vdpasim, 1);
378 blk_config->opt_io_size = cpu_to_vdpasim32(vdpasim, 1);
379 blk_config->blk_size = cpu_to_vdpasim32(vdpasim, SECTOR_SIZE);
380 /* VIRTIO_BLK_F_DISCARD */
381 blk_config->discard_sector_alignment =
382 cpu_to_vdpasim32(vdpasim, SECTOR_SIZE);
383 blk_config->max_discard_sectors =
384 cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS);
385 blk_config->max_discard_seg = cpu_to_vdpasim32(vdpasim, 1);
386 /* VIRTIO_BLK_F_WRITE_ZEROES */
387 blk_config->max_write_zeroes_sectors =
388 cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS);
389 blk_config->max_write_zeroes_seg = cpu_to_vdpasim32(vdpasim, 1);
390
391 }
392
vdpasim_blk_free(struct vdpasim * vdpasim)393 static void vdpasim_blk_free(struct vdpasim *vdpasim)
394 {
395 struct vdpasim_blk *blk = sim_to_blk(vdpasim);
396
397 if (!blk->shared_backend)
398 kvfree(blk->buffer);
399 }
400
401 static struct device *vdpasim_blk_mgmtdev;
402
vdpasim_blk_dev_add(struct vdpa_mgmt_dev * mdev,const char * name,const struct vdpa_dev_set_config * config)403 static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
404 const struct vdpa_dev_set_config *config)
405 {
406 struct vdpasim_dev_attr dev_attr = {};
407 struct vdpasim_blk *blk;
408 struct vdpasim *simdev;
409 int ret;
410
411 dev_attr.mgmt_dev = mdev;
412 dev_attr.name = name;
413 dev_attr.id = VIRTIO_ID_BLOCK;
414 dev_attr.supported_features = VDPASIM_BLK_FEATURES;
415 dev_attr.nvqs = VDPASIM_BLK_VQ_NUM;
416 dev_attr.ngroups = VDPASIM_BLK_GROUP_NUM;
417 dev_attr.nas = VDPASIM_BLK_AS_NUM;
418 dev_attr.alloc_size = sizeof(struct vdpasim_blk);
419 dev_attr.config_size = sizeof(struct virtio_blk_config);
420 dev_attr.get_config = vdpasim_blk_get_config;
421 dev_attr.work_fn = vdpasim_blk_work;
422 dev_attr.free = vdpasim_blk_free;
423
424 simdev = vdpasim_create(&dev_attr, config);
425 if (IS_ERR(simdev))
426 return PTR_ERR(simdev);
427
428 blk = sim_to_blk(simdev);
429 blk->shared_backend = shared_backend;
430
431 if (blk->shared_backend) {
432 blk->buffer = shared_buffer;
433 } else {
434 blk->buffer = kvzalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT,
435 GFP_KERNEL);
436 if (!blk->buffer) {
437 ret = -ENOMEM;
438 goto put_dev;
439 }
440 }
441
442 ret = _vdpa_register_device(&simdev->vdpa, VDPASIM_BLK_VQ_NUM);
443 if (ret)
444 goto put_dev;
445
446 return 0;
447
448 put_dev:
449 put_device(&simdev->vdpa.dev);
450 return ret;
451 }
452
vdpasim_blk_dev_del(struct vdpa_mgmt_dev * mdev,struct vdpa_device * dev)453 static void vdpasim_blk_dev_del(struct vdpa_mgmt_dev *mdev,
454 struct vdpa_device *dev)
455 {
456 struct vdpasim *simdev = container_of(dev, struct vdpasim, vdpa);
457
458 _vdpa_unregister_device(&simdev->vdpa);
459 }
460
461 static const struct vdpa_mgmtdev_ops vdpasim_blk_mgmtdev_ops = {
462 .dev_add = vdpasim_blk_dev_add,
463 .dev_del = vdpasim_blk_dev_del
464 };
465
466 static struct virtio_device_id id_table[] = {
467 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
468 { 0 },
469 };
470
471 static struct vdpa_mgmt_dev mgmt_dev = {
472 .id_table = id_table,
473 .ops = &vdpasim_blk_mgmtdev_ops,
474 };
475
vdpasim_blk_init(void)476 static int __init vdpasim_blk_init(void)
477 {
478 int ret;
479
480 vdpasim_blk_mgmtdev = root_device_register("vdpasim_blk");
481 if (IS_ERR(vdpasim_blk_mgmtdev))
482 return PTR_ERR(vdpasim_blk_mgmtdev);
483
484 mgmt_dev.device = vdpasim_blk_mgmtdev;
485 ret = vdpa_mgmtdev_register(&mgmt_dev);
486 if (ret)
487 goto parent_err;
488
489 if (shared_backend) {
490 shared_buffer = kvzalloc(VDPASIM_BLK_CAPACITY << SECTOR_SHIFT,
491 GFP_KERNEL);
492 if (!shared_buffer) {
493 ret = -ENOMEM;
494 goto mgmt_dev_err;
495 }
496 }
497
498 return 0;
499 mgmt_dev_err:
500 vdpa_mgmtdev_unregister(&mgmt_dev);
501 parent_err:
502 root_device_unregister(vdpasim_blk_mgmtdev);
503
504 return ret;
505 }
506
vdpasim_blk_exit(void)507 static void __exit vdpasim_blk_exit(void)
508 {
509 kvfree(shared_buffer);
510 vdpa_mgmtdev_unregister(&mgmt_dev);
511 root_device_unregister(vdpasim_blk_mgmtdev);
512 }
513
514 module_init(vdpasim_blk_init)
515 module_exit(vdpasim_blk_exit)
516
517 MODULE_VERSION(DRV_VERSION);
518 MODULE_LICENSE(DRV_LICENSE);
519 MODULE_AUTHOR(DRV_AUTHOR);
520 MODULE_DESCRIPTION(DRV_DESC);
521