1 /******************************************************************************
2 *
3 * Back-end of the driver for virtual block devices. This portion of the
4 * driver exports a 'unified' block-device interface that can be accessed
5 * by any operating system that implements a compatible front end. A
6 * reference front-end implementation can be found in:
7 * drivers/block/xen-blkfront.c
8 *
9 * Copyright (c) 2003-2004, Keir Fraser & Steve Hand
10 * Copyright (c) 2005, Christopher Clark
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License version 2
14 * as published by the Free Software Foundation; or, when distributed
15 * separately from the Linux kernel or incorporated into other
16 * software packages, subject to the following license:
17 *
18 * Permission is hereby granted, free of charge, to any person obtaining a copy
19 * of this source file (the "Software"), to deal in the Software without
20 * restriction, including without limitation the rights to use, copy, modify,
21 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
22 * and to permit persons to whom the Software is furnished to do so, subject to
23 * the following conditions:
24 *
25 * The above copyright notice and this permission notice shall be included in
26 * all copies or substantial portions of the Software.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
29 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
31 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
32 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
33 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34 * IN THE SOFTWARE.
35 */
36
37 #define pr_fmt(fmt) "xen-blkback: " fmt
38
39 #include <linux/spinlock.h>
40 #include <linux/kthread.h>
41 #include <linux/list.h>
42 #include <linux/delay.h>
43 #include <linux/freezer.h>
44 #include <linux/bitmap.h>
45
46 #include <xen/events.h>
47 #include <xen/page.h>
48 #include <xen/xen.h>
49 #include <asm/xen/hypervisor.h>
50 #include <asm/xen/hypercall.h>
51 #include <xen/balloon.h>
52 #include <xen/grant_table.h>
53 #include "common.h"
54
55 /*
56 * Maximum number of unused free pages to keep in the internal buffer.
57 * Setting this to a value too low will reduce memory used in each backend,
58 * but can have a performance penalty.
59 *
60 * A sane value is xen_blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST, but can
61 * be set to a lower value that might degrade performance on some intensive
62 * IO workloads.
63 */
64
65 static int max_buffer_pages = 1024;
66 module_param_named(max_buffer_pages, max_buffer_pages, int, 0644);
67 MODULE_PARM_DESC(max_buffer_pages,
68 "Maximum number of free pages to keep in each block backend buffer");
69
70 /*
71 * Maximum number of grants to map persistently in blkback. For maximum
72 * performance this should be the total numbers of grants that can be used
73 * to fill the ring, but since this might become too high, specially with
74 * the use of indirect descriptors, we set it to a value that provides good
75 * performance without using too much memory.
76 *
77 * When the list of persistent grants is full we clean it up using a LRU
78 * algorithm.
79 */
80
81 static int max_pgrants = 1056;
82 module_param_named(max_persistent_grants, max_pgrants, int, 0644);
83 MODULE_PARM_DESC(max_persistent_grants,
84 "Maximum number of grants to map persistently");
85
86 /*
87 * How long a persistent grant is allowed to remain allocated without being in
88 * use. The time is in seconds, 0 means indefinitely long.
89 */
90
91 static unsigned int pgrant_timeout = 60;
92 module_param_named(persistent_grant_unused_seconds, pgrant_timeout,
93 uint, 0644);
94 MODULE_PARM_DESC(persistent_grant_unused_seconds,
95 "Time in seconds an unused persistent grant is allowed to "
96 "remain allocated. Default is 60, 0 means unlimited.");
97
98 /*
99 * Maximum number of rings/queues blkback supports, allow as many queues as there
100 * are CPUs if user has not specified a value.
101 */
102 unsigned int xenblk_max_queues;
103 module_param_named(max_queues, xenblk_max_queues, uint, 0644);
104 MODULE_PARM_DESC(max_queues,
105 "Maximum number of hardware queues per virtual disk." \
106 "By default it is the number of online CPUs.");
107
108 /*
109 * Maximum order of pages to be used for the shared ring between front and
110 * backend, 4KB page granularity is used.
111 */
112 unsigned int xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
113 module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, 0444);
114 MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring");
115 /*
116 * The LRU mechanism to clean the lists of persistent grants needs to
117 * be executed periodically. The time interval between consecutive executions
118 * of the purge mechanism is set in ms.
119 */
120 #define LRU_INTERVAL 100
121
122 /*
123 * When the persistent grants list is full we will remove unused grants
124 * from the list. The percent number of grants to be removed at each LRU
125 * execution.
126 */
127 #define LRU_PERCENT_CLEAN 5
128
129 /* Run-time switchable: /sys/module/blkback/parameters/ */
130 static unsigned int log_stats;
131 module_param(log_stats, int, 0644);
132
133 #define BLKBACK_INVALID_HANDLE (~0)
134
persistent_gnt_timeout(struct persistent_gnt * persistent_gnt)135 static inline bool persistent_gnt_timeout(struct persistent_gnt *persistent_gnt)
136 {
137 return pgrant_timeout && (jiffies - persistent_gnt->last_used >=
138 HZ * pgrant_timeout);
139 }
140
141 #define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page)))
142
143 static int do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags);
144 static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
145 struct blkif_request *req,
146 struct pending_req *pending_req);
147 static void make_response(struct xen_blkif_ring *ring, u64 id,
148 unsigned short op, int st);
149
150 #define foreach_grant_safe(pos, n, rbtree, node) \
151 for ((pos) = container_of(rb_first((rbtree)), typeof(*(pos)), node), \
152 (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL; \
153 &(pos)->node != NULL; \
154 (pos) = container_of(n, typeof(*(pos)), node), \
155 (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL)
156
157
158 /*
159 * We don't need locking around the persistent grant helpers
160 * because blkback uses a single-thread for each backend, so we
161 * can be sure that this functions will never be called recursively.
162 *
163 * The only exception to that is put_persistent_grant, that can be called
164 * from interrupt context (by xen_blkbk_unmap), so we have to use atomic
165 * bit operations to modify the flags of a persistent grant and to count
166 * the number of used grants.
167 */
add_persistent_gnt(struct xen_blkif_ring * ring,struct persistent_gnt * persistent_gnt)168 static int add_persistent_gnt(struct xen_blkif_ring *ring,
169 struct persistent_gnt *persistent_gnt)
170 {
171 struct rb_node **new = NULL, *parent = NULL;
172 struct persistent_gnt *this;
173 struct xen_blkif *blkif = ring->blkif;
174
175 if (ring->persistent_gnt_c >= max_pgrants) {
176 if (!blkif->vbd.overflow_max_grants)
177 blkif->vbd.overflow_max_grants = 1;
178 return -EBUSY;
179 }
180 /* Figure out where to put new node */
181 new = &ring->persistent_gnts.rb_node;
182 while (*new) {
183 this = container_of(*new, struct persistent_gnt, node);
184
185 parent = *new;
186 if (persistent_gnt->gnt < this->gnt)
187 new = &((*new)->rb_left);
188 else if (persistent_gnt->gnt > this->gnt)
189 new = &((*new)->rb_right);
190 else {
191 pr_alert_ratelimited("trying to add a gref that's already in the tree\n");
192 return -EINVAL;
193 }
194 }
195
196 persistent_gnt->active = true;
197 /* Add new node and rebalance tree. */
198 rb_link_node(&(persistent_gnt->node), parent, new);
199 rb_insert_color(&(persistent_gnt->node), &ring->persistent_gnts);
200 ring->persistent_gnt_c++;
201 atomic_inc(&ring->persistent_gnt_in_use);
202 return 0;
203 }
204
get_persistent_gnt(struct xen_blkif_ring * ring,grant_ref_t gref)205 static struct persistent_gnt *get_persistent_gnt(struct xen_blkif_ring *ring,
206 grant_ref_t gref)
207 {
208 struct persistent_gnt *data;
209 struct rb_node *node = NULL;
210
211 node = ring->persistent_gnts.rb_node;
212 while (node) {
213 data = container_of(node, struct persistent_gnt, node);
214
215 if (gref < data->gnt)
216 node = node->rb_left;
217 else if (gref > data->gnt)
218 node = node->rb_right;
219 else {
220 if (data->active) {
221 pr_alert_ratelimited("requesting a grant already in use\n");
222 return NULL;
223 }
224 data->active = true;
225 atomic_inc(&ring->persistent_gnt_in_use);
226 return data;
227 }
228 }
229 return NULL;
230 }
231
put_persistent_gnt(struct xen_blkif_ring * ring,struct persistent_gnt * persistent_gnt)232 static void put_persistent_gnt(struct xen_blkif_ring *ring,
233 struct persistent_gnt *persistent_gnt)
234 {
235 if (!persistent_gnt->active)
236 pr_alert_ratelimited("freeing a grant already unused\n");
237 persistent_gnt->last_used = jiffies;
238 persistent_gnt->active = false;
239 atomic_dec(&ring->persistent_gnt_in_use);
240 }
241
free_persistent_gnts(struct xen_blkif_ring * ring)242 static void free_persistent_gnts(struct xen_blkif_ring *ring)
243 {
244 struct rb_root *root = &ring->persistent_gnts;
245 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
246 struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
247 struct persistent_gnt *persistent_gnt;
248 struct rb_node *n;
249 int segs_to_unmap = 0;
250 struct gntab_unmap_queue_data unmap_data;
251
252 if (RB_EMPTY_ROOT(root))
253 return;
254
255 unmap_data.pages = pages;
256 unmap_data.unmap_ops = unmap;
257 unmap_data.kunmap_ops = NULL;
258
259 foreach_grant_safe(persistent_gnt, n, root, node) {
260 BUG_ON(persistent_gnt->handle ==
261 BLKBACK_INVALID_HANDLE);
262 gnttab_set_unmap_op(&unmap[segs_to_unmap],
263 (unsigned long) pfn_to_kaddr(page_to_pfn(
264 persistent_gnt->page)),
265 GNTMAP_host_map,
266 persistent_gnt->handle);
267
268 pages[segs_to_unmap] = persistent_gnt->page;
269
270 if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST ||
271 !rb_next(&persistent_gnt->node)) {
272
273 unmap_data.count = segs_to_unmap;
274 BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
275
276 gnttab_page_cache_put(&ring->free_pages, pages,
277 segs_to_unmap);
278 segs_to_unmap = 0;
279 }
280
281 rb_erase(&persistent_gnt->node, root);
282 kfree(persistent_gnt);
283 ring->persistent_gnt_c--;
284 }
285
286 BUG_ON(!RB_EMPTY_ROOT(&ring->persistent_gnts));
287 BUG_ON(ring->persistent_gnt_c != 0);
288 }
289
xen_blkbk_unmap_purged_grants(struct work_struct * work)290 void xen_blkbk_unmap_purged_grants(struct work_struct *work)
291 {
292 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
293 struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
294 struct persistent_gnt *persistent_gnt;
295 int segs_to_unmap = 0;
296 struct xen_blkif_ring *ring = container_of(work, typeof(*ring), persistent_purge_work);
297 struct gntab_unmap_queue_data unmap_data;
298
299 unmap_data.pages = pages;
300 unmap_data.unmap_ops = unmap;
301 unmap_data.kunmap_ops = NULL;
302
303 while(!list_empty(&ring->persistent_purge_list)) {
304 persistent_gnt = list_first_entry(&ring->persistent_purge_list,
305 struct persistent_gnt,
306 remove_node);
307 list_del(&persistent_gnt->remove_node);
308
309 gnttab_set_unmap_op(&unmap[segs_to_unmap],
310 vaddr(persistent_gnt->page),
311 GNTMAP_host_map,
312 persistent_gnt->handle);
313
314 pages[segs_to_unmap] = persistent_gnt->page;
315
316 if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
317 unmap_data.count = segs_to_unmap;
318 BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
319 gnttab_page_cache_put(&ring->free_pages, pages,
320 segs_to_unmap);
321 segs_to_unmap = 0;
322 }
323 kfree(persistent_gnt);
324 }
325 if (segs_to_unmap > 0) {
326 unmap_data.count = segs_to_unmap;
327 BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
328 gnttab_page_cache_put(&ring->free_pages, pages, segs_to_unmap);
329 }
330 }
331
purge_persistent_gnt(struct xen_blkif_ring * ring)332 static void purge_persistent_gnt(struct xen_blkif_ring *ring)
333 {
334 struct persistent_gnt *persistent_gnt;
335 struct rb_node *n;
336 unsigned int num_clean, total;
337 bool scan_used = false;
338 struct rb_root *root;
339
340 if (work_busy(&ring->persistent_purge_work)) {
341 pr_alert_ratelimited("Scheduled work from previous purge is still busy, cannot purge list\n");
342 goto out;
343 }
344
345 if (ring->persistent_gnt_c < max_pgrants ||
346 (ring->persistent_gnt_c == max_pgrants &&
347 !ring->blkif->vbd.overflow_max_grants)) {
348 num_clean = 0;
349 } else {
350 num_clean = (max_pgrants / 100) * LRU_PERCENT_CLEAN;
351 num_clean = ring->persistent_gnt_c - max_pgrants + num_clean;
352 num_clean = min(ring->persistent_gnt_c, num_clean);
353 pr_debug("Going to purge at least %u persistent grants\n",
354 num_clean);
355 }
356
357 /*
358 * At this point, we can assure that there will be no calls
359 * to get_persistent_grant (because we are executing this code from
360 * xen_blkif_schedule), there can only be calls to put_persistent_gnt,
361 * which means that the number of currently used grants will go down,
362 * but never up, so we will always be able to remove the requested
363 * number of grants.
364 */
365
366 total = 0;
367
368 BUG_ON(!list_empty(&ring->persistent_purge_list));
369 root = &ring->persistent_gnts;
370 purge_list:
371 foreach_grant_safe(persistent_gnt, n, root, node) {
372 BUG_ON(persistent_gnt->handle ==
373 BLKBACK_INVALID_HANDLE);
374
375 if (persistent_gnt->active)
376 continue;
377 if (!scan_used && !persistent_gnt_timeout(persistent_gnt))
378 continue;
379 if (scan_used && total >= num_clean)
380 continue;
381
382 rb_erase(&persistent_gnt->node, root);
383 list_add(&persistent_gnt->remove_node,
384 &ring->persistent_purge_list);
385 total++;
386 }
387 /*
388 * Check whether we also need to start cleaning
389 * grants that were used since last purge in order to cope
390 * with the requested num
391 */
392 if (!scan_used && total < num_clean) {
393 pr_debug("Still missing %u purged frames\n", num_clean - total);
394 scan_used = true;
395 goto purge_list;
396 }
397
398 if (total) {
399 ring->persistent_gnt_c -= total;
400 ring->blkif->vbd.overflow_max_grants = 0;
401
402 /* We can defer this work */
403 schedule_work(&ring->persistent_purge_work);
404 pr_debug("Purged %u/%u\n", num_clean, total);
405 }
406
407 out:
408 return;
409 }
410
411 /*
412 * Retrieve from the 'pending_reqs' a free pending_req structure to be used.
413 */
alloc_req(struct xen_blkif_ring * ring)414 static struct pending_req *alloc_req(struct xen_blkif_ring *ring)
415 {
416 struct pending_req *req = NULL;
417 unsigned long flags;
418
419 spin_lock_irqsave(&ring->pending_free_lock, flags);
420 if (!list_empty(&ring->pending_free)) {
421 req = list_entry(ring->pending_free.next, struct pending_req,
422 free_list);
423 list_del(&req->free_list);
424 }
425 spin_unlock_irqrestore(&ring->pending_free_lock, flags);
426 return req;
427 }
428
429 /*
430 * Return the 'pending_req' structure back to the freepool. We also
431 * wake up the thread if it was waiting for a free page.
432 */
free_req(struct xen_blkif_ring * ring,struct pending_req * req)433 static void free_req(struct xen_blkif_ring *ring, struct pending_req *req)
434 {
435 unsigned long flags;
436 int was_empty;
437
438 spin_lock_irqsave(&ring->pending_free_lock, flags);
439 was_empty = list_empty(&ring->pending_free);
440 list_add(&req->free_list, &ring->pending_free);
441 spin_unlock_irqrestore(&ring->pending_free_lock, flags);
442 if (was_empty)
443 wake_up(&ring->pending_free_wq);
444 }
445
446 /*
447 * Routines for managing virtual block devices (vbds).
448 */
xen_vbd_translate(struct phys_req * req,struct xen_blkif * blkif,enum req_op operation)449 static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif,
450 enum req_op operation)
451 {
452 struct xen_vbd *vbd = &blkif->vbd;
453 int rc = -EACCES;
454
455 if ((operation != REQ_OP_READ) && vbd->readonly)
456 goto out;
457
458 if (likely(req->nr_sects)) {
459 blkif_sector_t end = req->sector_number + req->nr_sects;
460
461 if (unlikely(end < req->sector_number))
462 goto out;
463 if (unlikely(end > vbd_sz(vbd)))
464 goto out;
465 }
466
467 req->dev = vbd->pdevice;
468 req->bdev = file_bdev(vbd->bdev_file);
469 rc = 0;
470
471 out:
472 return rc;
473 }
474
xen_vbd_resize(struct xen_blkif * blkif)475 static void xen_vbd_resize(struct xen_blkif *blkif)
476 {
477 struct xen_vbd *vbd = &blkif->vbd;
478 struct xenbus_transaction xbt;
479 int err;
480 struct xenbus_device *dev = xen_blkbk_xenbus(blkif->be);
481 unsigned long long new_size = vbd_sz(vbd);
482
483 pr_info("VBD Resize: Domid: %d, Device: (%d, %d)\n",
484 blkif->domid, MAJOR(vbd->pdevice), MINOR(vbd->pdevice));
485 pr_info("VBD Resize: new size %llu\n", new_size);
486 vbd->size = new_size;
487 again:
488 err = xenbus_transaction_start(&xbt);
489 if (err) {
490 pr_warn("Error starting transaction\n");
491 return;
492 }
493 err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu",
494 (unsigned long long)vbd_sz(vbd));
495 if (err) {
496 pr_warn("Error writing new size\n");
497 goto abort;
498 }
499 /*
500 * Write the current state; we will use this to synchronize
501 * the front-end. If the current state is "connected" the
502 * front-end will get the new size information online.
503 */
504 err = xenbus_printf(xbt, dev->nodename, "state", "%d", dev->state);
505 if (err) {
506 pr_warn("Error writing the state\n");
507 goto abort;
508 }
509
510 err = xenbus_transaction_end(xbt, 0);
511 if (err == -EAGAIN)
512 goto again;
513 if (err)
514 pr_warn("Error ending transaction\n");
515 return;
516 abort:
517 xenbus_transaction_end(xbt, 1);
518 }
519
520 /*
521 * Notification from the guest OS.
522 */
blkif_notify_work(struct xen_blkif_ring * ring)523 static void blkif_notify_work(struct xen_blkif_ring *ring)
524 {
525 ring->waiting_reqs = 1;
526 wake_up(&ring->wq);
527 }
528
xen_blkif_be_int(int irq,void * dev_id)529 irqreturn_t xen_blkif_be_int(int irq, void *dev_id)
530 {
531 blkif_notify_work(dev_id);
532 return IRQ_HANDLED;
533 }
534
535 /*
536 * SCHEDULER FUNCTIONS
537 */
538
print_stats(struct xen_blkif_ring * ring)539 static void print_stats(struct xen_blkif_ring *ring)
540 {
541 pr_info("(%s): oo %3llu | rd %4llu | wr %4llu | f %4llu"
542 " | ds %4llu | pg: %4u/%4d\n",
543 current->comm, ring->st_oo_req,
544 ring->st_rd_req, ring->st_wr_req,
545 ring->st_f_req, ring->st_ds_req,
546 ring->persistent_gnt_c, max_pgrants);
547 ring->st_print = jiffies + secs_to_jiffies(10);
548 ring->st_rd_req = 0;
549 ring->st_wr_req = 0;
550 ring->st_oo_req = 0;
551 ring->st_ds_req = 0;
552 }
553
xen_blkif_schedule(void * arg)554 int xen_blkif_schedule(void *arg)
555 {
556 struct xen_blkif_ring *ring = arg;
557 struct xen_blkif *blkif = ring->blkif;
558 struct xen_vbd *vbd = &blkif->vbd;
559 unsigned long timeout;
560 int ret;
561 bool do_eoi;
562 unsigned int eoi_flags = XEN_EOI_FLAG_SPURIOUS;
563
564 set_freezable();
565 while (!kthread_should_stop()) {
566 if (try_to_freeze())
567 continue;
568 if (unlikely(vbd->size != vbd_sz(vbd)))
569 xen_vbd_resize(blkif);
570
571 timeout = msecs_to_jiffies(LRU_INTERVAL);
572
573 timeout = wait_event_interruptible_timeout(
574 ring->wq,
575 ring->waiting_reqs || kthread_should_stop(),
576 timeout);
577 if (timeout == 0)
578 goto purge_gnt_list;
579 timeout = wait_event_interruptible_timeout(
580 ring->pending_free_wq,
581 !list_empty(&ring->pending_free) ||
582 kthread_should_stop(),
583 timeout);
584 if (timeout == 0)
585 goto purge_gnt_list;
586
587 do_eoi = ring->waiting_reqs;
588
589 ring->waiting_reqs = 0;
590 smp_mb(); /* clear flag *before* checking for work */
591
592 ret = do_block_io_op(ring, &eoi_flags);
593 if (ret > 0)
594 ring->waiting_reqs = 1;
595 if (ret == -EACCES)
596 wait_event_interruptible(ring->shutdown_wq,
597 kthread_should_stop());
598
599 if (do_eoi && !ring->waiting_reqs) {
600 xen_irq_lateeoi(ring->irq, eoi_flags);
601 eoi_flags |= XEN_EOI_FLAG_SPURIOUS;
602 }
603
604 purge_gnt_list:
605 if (blkif->vbd.feature_gnt_persistent &&
606 time_after(jiffies, ring->next_lru)) {
607 purge_persistent_gnt(ring);
608 ring->next_lru = jiffies + msecs_to_jiffies(LRU_INTERVAL);
609 }
610
611 /* Shrink the free pages pool if it is too large. */
612 if (time_before(jiffies, blkif->buffer_squeeze_end))
613 gnttab_page_cache_shrink(&ring->free_pages, 0);
614 else
615 gnttab_page_cache_shrink(&ring->free_pages,
616 max_buffer_pages);
617
618 if (log_stats && time_after(jiffies, ring->st_print))
619 print_stats(ring);
620 }
621
622 /* Drain pending purge work */
623 flush_work(&ring->persistent_purge_work);
624
625 if (log_stats)
626 print_stats(ring);
627
628 ring->xenblkd = NULL;
629
630 return 0;
631 }
632
633 /*
634 * Remove persistent grants and empty the pool of free pages
635 */
xen_blkbk_free_caches(struct xen_blkif_ring * ring)636 void xen_blkbk_free_caches(struct xen_blkif_ring *ring)
637 {
638 /* Free all persistent grant pages */
639 free_persistent_gnts(ring);
640
641 /* Since we are shutting down remove all pages from the buffer */
642 gnttab_page_cache_shrink(&ring->free_pages, 0 /* All */);
643 }
644
xen_blkbk_unmap_prepare(struct xen_blkif_ring * ring,struct grant_page ** pages,unsigned int num,struct gnttab_unmap_grant_ref * unmap_ops,struct page ** unmap_pages)645 static unsigned int xen_blkbk_unmap_prepare(
646 struct xen_blkif_ring *ring,
647 struct grant_page **pages,
648 unsigned int num,
649 struct gnttab_unmap_grant_ref *unmap_ops,
650 struct page **unmap_pages)
651 {
652 unsigned int i, invcount = 0;
653
654 for (i = 0; i < num; i++) {
655 if (pages[i]->persistent_gnt != NULL) {
656 put_persistent_gnt(ring, pages[i]->persistent_gnt);
657 continue;
658 }
659 if (pages[i]->handle == BLKBACK_INVALID_HANDLE)
660 continue;
661 unmap_pages[invcount] = pages[i]->page;
662 gnttab_set_unmap_op(&unmap_ops[invcount], vaddr(pages[i]->page),
663 GNTMAP_host_map, pages[i]->handle);
664 pages[i]->handle = BLKBACK_INVALID_HANDLE;
665 invcount++;
666 }
667
668 return invcount;
669 }
670
xen_blkbk_unmap_and_respond_callback(int result,struct gntab_unmap_queue_data * data)671 static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_queue_data *data)
672 {
673 struct pending_req *pending_req = (struct pending_req *)(data->data);
674 struct xen_blkif_ring *ring = pending_req->ring;
675 struct xen_blkif *blkif = ring->blkif;
676
677 /* BUG_ON used to reproduce existing behaviour,
678 but is this the best way to deal with this? */
679 BUG_ON(result);
680
681 gnttab_page_cache_put(&ring->free_pages, data->pages, data->count);
682 make_response(ring, pending_req->id,
683 pending_req->operation, pending_req->status);
684 free_req(ring, pending_req);
685 /*
686 * Make sure the request is freed before releasing blkif,
687 * or there could be a race between free_req and the
688 * cleanup done in xen_blkif_free during shutdown.
689 *
690 * NB: The fact that we might try to wake up pending_free_wq
691 * before drain_complete (in case there's a drain going on)
692 * it's not a problem with our current implementation
693 * because we can assure there's no thread waiting on
694 * pending_free_wq if there's a drain going on, but it has
695 * to be taken into account if the current model is changed.
696 */
697 if (atomic_dec_and_test(&ring->inflight) && atomic_read(&blkif->drain)) {
698 complete(&blkif->drain_complete);
699 }
700 xen_blkif_put(blkif);
701 }
702
xen_blkbk_unmap_and_respond(struct pending_req * req)703 static void xen_blkbk_unmap_and_respond(struct pending_req *req)
704 {
705 struct gntab_unmap_queue_data* work = &req->gnttab_unmap_data;
706 struct xen_blkif_ring *ring = req->ring;
707 struct grant_page **pages = req->segments;
708 unsigned int invcount;
709
710 invcount = xen_blkbk_unmap_prepare(ring, pages, req->nr_segs,
711 req->unmap, req->unmap_pages);
712
713 work->data = req;
714 work->done = xen_blkbk_unmap_and_respond_callback;
715 work->unmap_ops = req->unmap;
716 work->kunmap_ops = NULL;
717 work->pages = req->unmap_pages;
718 work->count = invcount;
719
720 gnttab_unmap_refs_async(&req->gnttab_unmap_data);
721 }
722
723
724 /*
725 * Unmap the grant references.
726 *
727 * This could accumulate ops up to the batch size to reduce the number
728 * of hypercalls, but since this is only used in error paths there's
729 * no real need.
730 */
xen_blkbk_unmap(struct xen_blkif_ring * ring,struct grant_page * pages[],int num)731 static void xen_blkbk_unmap(struct xen_blkif_ring *ring,
732 struct grant_page *pages[],
733 int num)
734 {
735 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
736 struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
737 unsigned int invcount = 0;
738 int ret;
739
740 while (num) {
741 unsigned int batch = min(num, BLKIF_MAX_SEGMENTS_PER_REQUEST);
742
743 invcount = xen_blkbk_unmap_prepare(ring, pages, batch,
744 unmap, unmap_pages);
745 if (invcount) {
746 ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount);
747 BUG_ON(ret);
748 gnttab_page_cache_put(&ring->free_pages, unmap_pages,
749 invcount);
750 }
751 pages += batch;
752 num -= batch;
753 }
754 }
755
xen_blkbk_map(struct xen_blkif_ring * ring,struct grant_page * pages[],int num,bool ro)756 static int xen_blkbk_map(struct xen_blkif_ring *ring,
757 struct grant_page *pages[],
758 int num, bool ro)
759 {
760 struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST];
761 struct page *pages_to_gnt[BLKIF_MAX_SEGMENTS_PER_REQUEST];
762 struct persistent_gnt *persistent_gnt = NULL;
763 phys_addr_t addr = 0;
764 int i, seg_idx, new_map_idx;
765 int segs_to_map = 0;
766 int ret = 0;
767 int last_map = 0, map_until = 0;
768 int use_persistent_gnts;
769 struct xen_blkif *blkif = ring->blkif;
770
771 use_persistent_gnts = (blkif->vbd.feature_gnt_persistent);
772
773 /*
774 * Fill out preq.nr_sects with proper amount of sectors, and setup
775 * assign map[..] with the PFN of the page in our domain with the
776 * corresponding grant reference for each page.
777 */
778 again:
779 for (i = map_until; i < num; i++) {
780 uint32_t flags;
781
782 if (use_persistent_gnts) {
783 persistent_gnt = get_persistent_gnt(
784 ring,
785 pages[i]->gref);
786 }
787
788 if (persistent_gnt) {
789 /*
790 * We are using persistent grants and
791 * the grant is already mapped
792 */
793 pages[i]->page = persistent_gnt->page;
794 pages[i]->persistent_gnt = persistent_gnt;
795 } else {
796 if (gnttab_page_cache_get(&ring->free_pages,
797 &pages[i]->page)) {
798 gnttab_page_cache_put(&ring->free_pages,
799 pages_to_gnt,
800 segs_to_map);
801 ret = -ENOMEM;
802 goto out;
803 }
804 addr = vaddr(pages[i]->page);
805 pages_to_gnt[segs_to_map] = pages[i]->page;
806 pages[i]->persistent_gnt = NULL;
807 flags = GNTMAP_host_map;
808 if (!use_persistent_gnts && ro)
809 flags |= GNTMAP_readonly;
810 gnttab_set_map_op(&map[segs_to_map++], addr,
811 flags, pages[i]->gref,
812 blkif->domid);
813 }
814 map_until = i + 1;
815 if (segs_to_map == BLKIF_MAX_SEGMENTS_PER_REQUEST)
816 break;
817 }
818
819 if (segs_to_map)
820 ret = gnttab_map_refs(map, NULL, pages_to_gnt, segs_to_map);
821
822 /*
823 * Now swizzle the MFN in our domain with the MFN from the other domain
824 * so that when we access vaddr(pending_req,i) it has the contents of
825 * the page from the other domain.
826 */
827 for (seg_idx = last_map, new_map_idx = 0; seg_idx < map_until; seg_idx++) {
828 if (!pages[seg_idx]->persistent_gnt) {
829 /* This is a newly mapped grant */
830 BUG_ON(new_map_idx >= segs_to_map);
831 if (unlikely(map[new_map_idx].status != 0)) {
832 pr_debug("invalid buffer -- could not remap it\n");
833 gnttab_page_cache_put(&ring->free_pages,
834 &pages[seg_idx]->page, 1);
835 pages[seg_idx]->handle = BLKBACK_INVALID_HANDLE;
836 ret |= !ret;
837 goto next;
838 }
839 pages[seg_idx]->handle = map[new_map_idx].handle;
840 } else {
841 continue;
842 }
843 if (use_persistent_gnts &&
844 ring->persistent_gnt_c < max_pgrants) {
845 /*
846 * We are using persistent grants, the grant is
847 * not mapped but we might have room for it.
848 */
849 persistent_gnt = kmalloc_obj(struct persistent_gnt);
850 if (!persistent_gnt) {
851 /*
852 * If we don't have enough memory to
853 * allocate the persistent_gnt struct
854 * map this grant non-persistenly
855 */
856 goto next;
857 }
858 persistent_gnt->gnt = map[new_map_idx].ref;
859 persistent_gnt->handle = map[new_map_idx].handle;
860 persistent_gnt->page = pages[seg_idx]->page;
861 if (add_persistent_gnt(ring,
862 persistent_gnt)) {
863 kfree(persistent_gnt);
864 persistent_gnt = NULL;
865 goto next;
866 }
867 pages[seg_idx]->persistent_gnt = persistent_gnt;
868 pr_debug("grant %u added to the tree of persistent grants, using %u/%u\n",
869 persistent_gnt->gnt, ring->persistent_gnt_c,
870 max_pgrants);
871 goto next;
872 }
873 if (use_persistent_gnts && !blkif->vbd.overflow_max_grants) {
874 blkif->vbd.overflow_max_grants = 1;
875 pr_debug("domain %u, device %#x is using maximum number of persistent grants\n",
876 blkif->domid, blkif->vbd.handle);
877 }
878 /*
879 * We could not map this grant persistently, so use it as
880 * a non-persistent grant.
881 */
882 next:
883 new_map_idx++;
884 }
885 segs_to_map = 0;
886 last_map = map_until;
887 if (!ret && map_until != num)
888 goto again;
889
890 out:
891 for (i = last_map; i < num; i++) {
892 /* Don't zap current batch's valid persistent grants. */
893 if (i >= map_until)
894 pages[i]->persistent_gnt = NULL;
895 pages[i]->handle = BLKBACK_INVALID_HANDLE;
896 }
897
898 return ret;
899 }
900
xen_blkbk_map_seg(struct pending_req * pending_req)901 static int xen_blkbk_map_seg(struct pending_req *pending_req)
902 {
903 int rc;
904
905 rc = xen_blkbk_map(pending_req->ring, pending_req->segments,
906 pending_req->nr_segs,
907 (pending_req->operation != BLKIF_OP_READ));
908
909 return rc;
910 }
911
xen_blkbk_parse_indirect(struct blkif_request * req,struct pending_req * pending_req,struct seg_buf seg[],struct phys_req * preq)912 static int xen_blkbk_parse_indirect(struct blkif_request *req,
913 struct pending_req *pending_req,
914 struct seg_buf seg[],
915 struct phys_req *preq)
916 {
917 struct grant_page **pages = pending_req->indirect_pages;
918 struct xen_blkif_ring *ring = pending_req->ring;
919 int indirect_grefs, rc, n, nseg, i;
920 struct blkif_request_segment *segments = NULL;
921
922 nseg = pending_req->nr_segs;
923 indirect_grefs = INDIRECT_PAGES(nseg);
924 BUG_ON(indirect_grefs > BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST);
925
926 for (i = 0; i < indirect_grefs; i++)
927 pages[i]->gref = req->u.indirect.indirect_grefs[i];
928
929 rc = xen_blkbk_map(ring, pages, indirect_grefs, true);
930 if (rc)
931 goto unmap;
932
933 for (n = 0; n < nseg; n++) {
934 uint8_t first_sect, last_sect;
935
936 if ((n % SEGS_PER_INDIRECT_FRAME) == 0) {
937 /* Map indirect segments */
938 if (segments)
939 kunmap_atomic(segments);
940 segments = kmap_atomic(pages[n/SEGS_PER_INDIRECT_FRAME]->page);
941 }
942 i = n % SEGS_PER_INDIRECT_FRAME;
943
944 pending_req->segments[n]->gref = segments[i].gref;
945
946 first_sect = READ_ONCE(segments[i].first_sect);
947 last_sect = READ_ONCE(segments[i].last_sect);
948 if (last_sect >= (XEN_PAGE_SIZE >> 9) || last_sect < first_sect) {
949 rc = -EINVAL;
950 goto unmap;
951 }
952
953 seg[n].nsec = last_sect - first_sect + 1;
954 seg[n].offset = first_sect << 9;
955 preq->nr_sects += seg[n].nsec;
956 }
957
958 unmap:
959 if (segments)
960 kunmap_atomic(segments);
961 xen_blkbk_unmap(ring, pages, indirect_grefs);
962 return rc;
963 }
964
dispatch_discard_io(struct xen_blkif_ring * ring,struct blkif_request * req)965 static int dispatch_discard_io(struct xen_blkif_ring *ring,
966 struct blkif_request *req)
967 {
968 int err = 0;
969 int status = BLKIF_RSP_OKAY;
970 struct xen_blkif *blkif = ring->blkif;
971 struct block_device *bdev = file_bdev(blkif->vbd.bdev_file);
972 struct phys_req preq;
973
974 xen_blkif_get(blkif);
975
976 preq.sector_number = req->u.discard.sector_number;
977 preq.nr_sects = req->u.discard.nr_sectors;
978
979 err = xen_vbd_translate(&preq, blkif, REQ_OP_WRITE);
980 if (err) {
981 pr_warn("access denied: DISCARD [%llu->%llu] on dev=%04x\n",
982 preq.sector_number,
983 preq.sector_number + preq.nr_sects, blkif->vbd.pdevice);
984 goto fail_response;
985 }
986 ring->st_ds_req++;
987
988 if (blkif->vbd.discard_secure &&
989 (req->u.discard.flag & BLKIF_DISCARD_SECURE))
990 err = blkdev_issue_secure_erase(bdev,
991 req->u.discard.sector_number,
992 req->u.discard.nr_sectors, GFP_KERNEL);
993 else
994 err = blkdev_issue_discard(bdev, req->u.discard.sector_number,
995 req->u.discard.nr_sectors, GFP_KERNEL);
996
997 fail_response:
998 if (err == -EOPNOTSUPP) {
999 pr_debug("discard op failed, not supported\n");
1000 status = BLKIF_RSP_EOPNOTSUPP;
1001 } else if (err)
1002 status = BLKIF_RSP_ERROR;
1003
1004 make_response(ring, req->u.discard.id, req->operation, status);
1005 xen_blkif_put(blkif);
1006 return err;
1007 }
1008
dispatch_other_io(struct xen_blkif_ring * ring,struct blkif_request * req,struct pending_req * pending_req)1009 static int dispatch_other_io(struct xen_blkif_ring *ring,
1010 struct blkif_request *req,
1011 struct pending_req *pending_req)
1012 {
1013 free_req(ring, pending_req);
1014 make_response(ring, req->u.other.id, req->operation,
1015 BLKIF_RSP_EOPNOTSUPP);
1016 return -EIO;
1017 }
1018
xen_blk_drain_io(struct xen_blkif_ring * ring)1019 static void xen_blk_drain_io(struct xen_blkif_ring *ring)
1020 {
1021 struct xen_blkif *blkif = ring->blkif;
1022
1023 atomic_set(&blkif->drain, 1);
1024 do {
1025 if (atomic_read(&ring->inflight) == 0)
1026 break;
1027 wait_for_completion_interruptible_timeout(
1028 &blkif->drain_complete, HZ);
1029
1030 if (!atomic_read(&blkif->drain))
1031 break;
1032 } while (!kthread_should_stop());
1033 atomic_set(&blkif->drain, 0);
1034 }
1035
__end_block_io_op(struct pending_req * pending_req,blk_status_t error)1036 static void __end_block_io_op(struct pending_req *pending_req,
1037 blk_status_t error)
1038 {
1039 /* An error fails the entire request. */
1040 if (pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE &&
1041 error == BLK_STS_NOTSUPP) {
1042 pr_debug("flush diskcache op failed, not supported\n");
1043 xen_blkbk_flush_diskcache(XBT_NIL, pending_req->ring->blkif->be, 0);
1044 pending_req->status = BLKIF_RSP_EOPNOTSUPP;
1045 } else if (pending_req->operation == BLKIF_OP_WRITE_BARRIER &&
1046 error == BLK_STS_NOTSUPP) {
1047 pr_debug("write barrier op failed, not supported\n");
1048 xen_blkbk_barrier(XBT_NIL, pending_req->ring->blkif->be, 0);
1049 pending_req->status = BLKIF_RSP_EOPNOTSUPP;
1050 } else if (error) {
1051 pr_debug("Buffer not up-to-date at end of operation,"
1052 " error=%d\n", error);
1053 pending_req->status = BLKIF_RSP_ERROR;
1054 }
1055
1056 /*
1057 * If all of the bio's have completed it is time to unmap
1058 * the grant references associated with 'request' and provide
1059 * the proper response on the ring.
1060 */
1061 if (atomic_dec_and_test(&pending_req->pendcnt))
1062 xen_blkbk_unmap_and_respond(pending_req);
1063 }
1064
1065 /*
1066 * bio callback.
1067 */
end_block_io_op(struct bio * bio)1068 static void end_block_io_op(struct bio *bio)
1069 {
1070 __end_block_io_op(bio->bi_private, bio->bi_status);
1071 bio_put(bio);
1072 }
1073
blkif_get_x86_32_req(struct blkif_request * dst,const struct blkif_x86_32_request * src)1074 static void blkif_get_x86_32_req(struct blkif_request *dst,
1075 const struct blkif_x86_32_request *src)
1076 {
1077 unsigned int i, n;
1078
1079 dst->operation = READ_ONCE(src->operation);
1080
1081 switch (dst->operation) {
1082 case BLKIF_OP_READ:
1083 case BLKIF_OP_WRITE:
1084 case BLKIF_OP_WRITE_BARRIER:
1085 case BLKIF_OP_FLUSH_DISKCACHE:
1086 dst->u.rw.nr_segments = READ_ONCE(src->u.rw.nr_segments);
1087 dst->u.rw.handle = src->u.rw.handle;
1088 dst->u.rw.id = src->u.rw.id;
1089 dst->u.rw.sector_number = src->u.rw.sector_number;
1090 n = min_t(unsigned int, BLKIF_MAX_SEGMENTS_PER_REQUEST,
1091 dst->u.rw.nr_segments);
1092 for (i = 0; i < n; i++)
1093 dst->u.rw.seg[i] = src->u.rw.seg[i];
1094 break;
1095
1096 case BLKIF_OP_DISCARD:
1097 dst->u.discard.flag = src->u.discard.flag;
1098 dst->u.discard.id = src->u.discard.id;
1099 dst->u.discard.sector_number = src->u.discard.sector_number;
1100 dst->u.discard.nr_sectors = src->u.discard.nr_sectors;
1101 break;
1102
1103 case BLKIF_OP_INDIRECT:
1104 dst->u.indirect.indirect_op = src->u.indirect.indirect_op;
1105 dst->u.indirect.nr_segments =
1106 READ_ONCE(src->u.indirect.nr_segments);
1107 dst->u.indirect.handle = src->u.indirect.handle;
1108 dst->u.indirect.id = src->u.indirect.id;
1109 dst->u.indirect.sector_number = src->u.indirect.sector_number;
1110 n = min(MAX_INDIRECT_PAGES,
1111 INDIRECT_PAGES(dst->u.indirect.nr_segments));
1112 for (i = 0; i < n; i++)
1113 dst->u.indirect.indirect_grefs[i] =
1114 src->u.indirect.indirect_grefs[i];
1115 break;
1116
1117 default:
1118 /*
1119 * Don't know how to translate this op. Only get the
1120 * ID so failure can be reported to the frontend.
1121 */
1122 dst->u.other.id = src->u.other.id;
1123 break;
1124 }
1125 }
1126
blkif_get_x86_64_req(struct blkif_request * dst,const struct blkif_x86_64_request * src)1127 static void blkif_get_x86_64_req(struct blkif_request *dst,
1128 const struct blkif_x86_64_request *src)
1129 {
1130 unsigned int i, n;
1131
1132 dst->operation = READ_ONCE(src->operation);
1133
1134 switch (dst->operation) {
1135 case BLKIF_OP_READ:
1136 case BLKIF_OP_WRITE:
1137 case BLKIF_OP_WRITE_BARRIER:
1138 case BLKIF_OP_FLUSH_DISKCACHE:
1139 dst->u.rw.nr_segments = READ_ONCE(src->u.rw.nr_segments);
1140 dst->u.rw.handle = src->u.rw.handle;
1141 dst->u.rw.id = src->u.rw.id;
1142 dst->u.rw.sector_number = src->u.rw.sector_number;
1143 n = min_t(unsigned int, BLKIF_MAX_SEGMENTS_PER_REQUEST,
1144 dst->u.rw.nr_segments);
1145 for (i = 0; i < n; i++)
1146 dst->u.rw.seg[i] = src->u.rw.seg[i];
1147 break;
1148
1149 case BLKIF_OP_DISCARD:
1150 dst->u.discard.flag = src->u.discard.flag;
1151 dst->u.discard.id = src->u.discard.id;
1152 dst->u.discard.sector_number = src->u.discard.sector_number;
1153 dst->u.discard.nr_sectors = src->u.discard.nr_sectors;
1154 break;
1155
1156 case BLKIF_OP_INDIRECT:
1157 dst->u.indirect.indirect_op = src->u.indirect.indirect_op;
1158 dst->u.indirect.nr_segments =
1159 READ_ONCE(src->u.indirect.nr_segments);
1160 dst->u.indirect.handle = src->u.indirect.handle;
1161 dst->u.indirect.id = src->u.indirect.id;
1162 dst->u.indirect.sector_number = src->u.indirect.sector_number;
1163 n = min(MAX_INDIRECT_PAGES,
1164 INDIRECT_PAGES(dst->u.indirect.nr_segments));
1165 for (i = 0; i < n; i++)
1166 dst->u.indirect.indirect_grefs[i] =
1167 src->u.indirect.indirect_grefs[i];
1168 break;
1169
1170 default:
1171 /*
1172 * Don't know how to translate this op. Only get the
1173 * ID so failure can be reported to the frontend.
1174 */
1175 dst->u.other.id = src->u.other.id;
1176 break;
1177 }
1178 }
1179
1180 /*
1181 * Function to copy the from the ring buffer the 'struct blkif_request'
1182 * (which has the sectors we want, number of them, grant references, etc),
1183 * and transmute it to the block API to hand it over to the proper block disk.
1184 */
1185 static int
__do_block_io_op(struct xen_blkif_ring * ring,unsigned int * eoi_flags)1186 __do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags)
1187 {
1188 union blkif_back_rings *blk_rings = &ring->blk_rings;
1189 struct blkif_request req;
1190 struct pending_req *pending_req;
1191 RING_IDX rc, rp;
1192 int more_to_do = 0;
1193
1194 rc = blk_rings->common.req_cons;
1195 rp = blk_rings->common.sring->req_prod;
1196 rmb(); /* Ensure we see queued requests up to 'rp'. */
1197
1198 if (RING_REQUEST_PROD_OVERFLOW(&blk_rings->common, rp)) {
1199 rc = blk_rings->common.rsp_prod_pvt;
1200 pr_warn("Frontend provided bogus ring requests (%d - %d = %d). Halting ring processing on dev=%04x\n",
1201 rp, rc, rp - rc, ring->blkif->vbd.pdevice);
1202 return -EACCES;
1203 }
1204 while (rc != rp) {
1205
1206 if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc))
1207 break;
1208
1209 /* We've seen a request, so clear spurious eoi flag. */
1210 *eoi_flags &= ~XEN_EOI_FLAG_SPURIOUS;
1211
1212 if (kthread_should_stop()) {
1213 more_to_do = 1;
1214 break;
1215 }
1216
1217 pending_req = alloc_req(ring);
1218 if (NULL == pending_req) {
1219 ring->st_oo_req++;
1220 more_to_do = 1;
1221 break;
1222 }
1223
1224 switch (ring->blkif->blk_protocol) {
1225 case BLKIF_PROTOCOL_NATIVE:
1226 memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req));
1227 break;
1228 case BLKIF_PROTOCOL_X86_32:
1229 blkif_get_x86_32_req(&req, RING_GET_REQUEST(&blk_rings->x86_32, rc));
1230 break;
1231 case BLKIF_PROTOCOL_X86_64:
1232 blkif_get_x86_64_req(&req, RING_GET_REQUEST(&blk_rings->x86_64, rc));
1233 break;
1234 default:
1235 BUG();
1236 }
1237 blk_rings->common.req_cons = ++rc; /* before make_response() */
1238
1239 /* Apply all sanity checks to /private copy/ of request. */
1240 barrier();
1241
1242 switch (req.operation) {
1243 case BLKIF_OP_READ:
1244 case BLKIF_OP_WRITE:
1245 case BLKIF_OP_WRITE_BARRIER:
1246 case BLKIF_OP_FLUSH_DISKCACHE:
1247 case BLKIF_OP_INDIRECT:
1248 if (dispatch_rw_block_io(ring, &req, pending_req))
1249 goto done;
1250 break;
1251 case BLKIF_OP_DISCARD:
1252 free_req(ring, pending_req);
1253 if (dispatch_discard_io(ring, &req))
1254 goto done;
1255 break;
1256 default:
1257 if (dispatch_other_io(ring, &req, pending_req))
1258 goto done;
1259 break;
1260 }
1261
1262 /* Yield point for this unbounded loop. */
1263 cond_resched();
1264 }
1265 done:
1266 return more_to_do;
1267 }
1268
1269 static int
do_block_io_op(struct xen_blkif_ring * ring,unsigned int * eoi_flags)1270 do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags)
1271 {
1272 union blkif_back_rings *blk_rings = &ring->blk_rings;
1273 int more_to_do;
1274
1275 do {
1276 more_to_do = __do_block_io_op(ring, eoi_flags);
1277 if (more_to_do)
1278 break;
1279
1280 RING_FINAL_CHECK_FOR_REQUESTS(&blk_rings->common, more_to_do);
1281 } while (more_to_do);
1282
1283 return more_to_do;
1284 }
1285 /*
1286 * Transmutation of the 'struct blkif_request' to a proper 'struct bio'
1287 * and call the 'submit_bio' to pass it to the underlying storage.
1288 */
dispatch_rw_block_io(struct xen_blkif_ring * ring,struct blkif_request * req,struct pending_req * pending_req)1289 static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
1290 struct blkif_request *req,
1291 struct pending_req *pending_req)
1292 {
1293 struct phys_req preq;
1294 struct seg_buf *seg = pending_req->seg;
1295 unsigned int nseg;
1296 struct bio *bio = NULL;
1297 struct bio **biolist = pending_req->biolist;
1298 int i, nbio = 0;
1299 enum req_op operation;
1300 blk_opf_t operation_flags = 0;
1301 struct blk_plug plug;
1302 bool drain = false;
1303 struct grant_page **pages = pending_req->segments;
1304 unsigned short req_operation;
1305
1306 req_operation = req->operation == BLKIF_OP_INDIRECT ?
1307 req->u.indirect.indirect_op : req->operation;
1308
1309 if ((req->operation == BLKIF_OP_INDIRECT) &&
1310 (req_operation != BLKIF_OP_READ) &&
1311 (req_operation != BLKIF_OP_WRITE)) {
1312 pr_debug("Invalid indirect operation (%u)\n", req_operation);
1313 goto fail_response;
1314 }
1315
1316 switch (req_operation) {
1317 case BLKIF_OP_READ:
1318 ring->st_rd_req++;
1319 operation = REQ_OP_READ;
1320 break;
1321 case BLKIF_OP_WRITE:
1322 ring->st_wr_req++;
1323 operation = REQ_OP_WRITE;
1324 operation_flags = REQ_SYNC | REQ_IDLE;
1325 break;
1326 case BLKIF_OP_WRITE_BARRIER:
1327 drain = true;
1328 fallthrough;
1329 case BLKIF_OP_FLUSH_DISKCACHE:
1330 ring->st_f_req++;
1331 operation = REQ_OP_WRITE;
1332 operation_flags = REQ_PREFLUSH;
1333 break;
1334 default:
1335 operation = 0; /* make gcc happy */
1336 goto fail_response;
1337 break;
1338 }
1339
1340 /* Check that the number of segments is sane. */
1341 nseg = req->operation == BLKIF_OP_INDIRECT ?
1342 req->u.indirect.nr_segments : req->u.rw.nr_segments;
1343
1344 if (unlikely(nseg == 0 && operation_flags != REQ_PREFLUSH) ||
1345 unlikely((req->operation != BLKIF_OP_INDIRECT) &&
1346 (nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) ||
1347 unlikely((req->operation == BLKIF_OP_INDIRECT) &&
1348 (nseg > MAX_INDIRECT_SEGMENTS))) {
1349 pr_debug("Bad number of segments in request (%d)\n", nseg);
1350 /* Haven't submitted any bio's yet. */
1351 goto fail_response;
1352 }
1353
1354 preq.nr_sects = 0;
1355
1356 pending_req->ring = ring;
1357 pending_req->id = req->u.rw.id;
1358 pending_req->operation = req_operation;
1359 pending_req->status = BLKIF_RSP_OKAY;
1360 pending_req->nr_segs = nseg;
1361
1362 if (req->operation != BLKIF_OP_INDIRECT) {
1363 preq.dev = req->u.rw.handle;
1364 preq.sector_number = req->u.rw.sector_number;
1365 for (i = 0; i < nseg; i++) {
1366 pages[i]->gref = req->u.rw.seg[i].gref;
1367 seg[i].nsec = req->u.rw.seg[i].last_sect -
1368 req->u.rw.seg[i].first_sect + 1;
1369 seg[i].offset = (req->u.rw.seg[i].first_sect << 9);
1370 if ((req->u.rw.seg[i].last_sect >= (XEN_PAGE_SIZE >> 9)) ||
1371 (req->u.rw.seg[i].last_sect <
1372 req->u.rw.seg[i].first_sect))
1373 goto fail_response;
1374 preq.nr_sects += seg[i].nsec;
1375 }
1376 } else {
1377 preq.dev = req->u.indirect.handle;
1378 preq.sector_number = req->u.indirect.sector_number;
1379 if (xen_blkbk_parse_indirect(req, pending_req, seg, &preq))
1380 goto fail_response;
1381 }
1382
1383 if (xen_vbd_translate(&preq, ring->blkif, operation) != 0) {
1384 pr_debug("access denied: %s of [%llu,%llu] on dev=%04x\n",
1385 operation == REQ_OP_READ ? "read" : "write",
1386 preq.sector_number,
1387 preq.sector_number + preq.nr_sects,
1388 ring->blkif->vbd.pdevice);
1389 goto fail_response;
1390 }
1391
1392 /*
1393 * This check _MUST_ be done after xen_vbd_translate as the preq.bdev
1394 * is set there.
1395 */
1396 for (i = 0; i < nseg; i++) {
1397 if (((int)preq.sector_number|(int)seg[i].nsec) &
1398 ((bdev_logical_block_size(preq.bdev) >> 9) - 1)) {
1399 pr_debug("Misaligned I/O request from domain %d\n",
1400 ring->blkif->domid);
1401 goto fail_response;
1402 }
1403 }
1404
1405 /* Wait on all outstanding I/O's and once that has been completed
1406 * issue the flush.
1407 */
1408 if (drain)
1409 xen_blk_drain_io(pending_req->ring);
1410
1411 /*
1412 * If we have failed at this point, we need to undo the M2P override,
1413 * set gnttab_set_unmap_op on all of the grant references and perform
1414 * the hypercall to unmap the grants - that is all done in
1415 * xen_blkbk_unmap.
1416 */
1417 if (xen_blkbk_map_seg(pending_req))
1418 goto fail_flush;
1419
1420 /*
1421 * This corresponding xen_blkif_put is done in __end_block_io_op, or
1422 * below (in "!bio") if we are handling a BLKIF_OP_DISCARD.
1423 */
1424 xen_blkif_get(ring->blkif);
1425 atomic_inc(&ring->inflight);
1426
1427 for (i = 0; i < nseg; i++) {
1428 while ((bio == NULL) ||
1429 (bio_add_page(bio,
1430 pages[i]->page,
1431 seg[i].nsec << 9,
1432 seg[i].offset) == 0)) {
1433 bio = bio_alloc(preq.bdev, bio_max_segs(nseg - i),
1434 operation | operation_flags,
1435 GFP_KERNEL);
1436 biolist[nbio++] = bio;
1437 bio->bi_private = pending_req;
1438 bio->bi_end_io = end_block_io_op;
1439 bio->bi_iter.bi_sector = preq.sector_number;
1440 }
1441
1442 preq.sector_number += seg[i].nsec;
1443 }
1444
1445 /* This will be hit if the operation was a flush or discard. */
1446 if (!bio) {
1447 BUG_ON(operation_flags != REQ_PREFLUSH);
1448
1449 bio = bio_alloc(preq.bdev, 0, operation | operation_flags,
1450 GFP_KERNEL);
1451 biolist[nbio++] = bio;
1452 bio->bi_private = pending_req;
1453 bio->bi_end_io = end_block_io_op;
1454 }
1455
1456 atomic_set(&pending_req->pendcnt, nbio);
1457 blk_start_plug(&plug);
1458
1459 for (i = 0; i < nbio; i++)
1460 submit_bio(biolist[i]);
1461
1462 /* Let the I/Os go.. */
1463 blk_finish_plug(&plug);
1464
1465 if (operation == REQ_OP_READ)
1466 ring->st_rd_sect += preq.nr_sects;
1467 else if (operation == REQ_OP_WRITE)
1468 ring->st_wr_sect += preq.nr_sects;
1469
1470 return 0;
1471
1472 fail_flush:
1473 xen_blkbk_unmap(ring, pending_req->segments,
1474 pending_req->nr_segs);
1475 fail_response:
1476 /* Haven't submitted any bio's yet. */
1477 make_response(ring, req->u.rw.id, req_operation, BLKIF_RSP_ERROR);
1478 free_req(ring, pending_req);
1479 msleep(1); /* back off a bit */
1480 return -EIO;
1481 }
1482
1483
1484
1485 /*
1486 * Put a response on the ring on how the operation fared.
1487 */
make_response(struct xen_blkif_ring * ring,u64 id,unsigned short op,int st)1488 static void make_response(struct xen_blkif_ring *ring, u64 id,
1489 unsigned short op, int st)
1490 {
1491 struct blkif_response *resp;
1492 unsigned long flags;
1493 union blkif_back_rings *blk_rings;
1494 int notify;
1495
1496 spin_lock_irqsave(&ring->blk_ring_lock, flags);
1497 blk_rings = &ring->blk_rings;
1498 /* Place on the response ring for the relevant domain. */
1499 switch (ring->blkif->blk_protocol) {
1500 case BLKIF_PROTOCOL_NATIVE:
1501 resp = RING_GET_RESPONSE(&blk_rings->native,
1502 blk_rings->native.rsp_prod_pvt);
1503 break;
1504 case BLKIF_PROTOCOL_X86_32:
1505 resp = RING_GET_RESPONSE(&blk_rings->x86_32,
1506 blk_rings->x86_32.rsp_prod_pvt);
1507 break;
1508 case BLKIF_PROTOCOL_X86_64:
1509 resp = RING_GET_RESPONSE(&blk_rings->x86_64,
1510 blk_rings->x86_64.rsp_prod_pvt);
1511 break;
1512 default:
1513 BUG();
1514 }
1515
1516 resp->id = id;
1517 resp->operation = op;
1518 resp->status = st;
1519
1520 blk_rings->common.rsp_prod_pvt++;
1521 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify);
1522 spin_unlock_irqrestore(&ring->blk_ring_lock, flags);
1523 if (notify)
1524 notify_remote_via_irq(ring->irq);
1525 }
1526
xen_blkif_init(void)1527 static int __init xen_blkif_init(void)
1528 {
1529 int rc = 0;
1530
1531 if (!xen_domain())
1532 return -ENODEV;
1533
1534 if (xen_blkif_max_ring_order > XENBUS_MAX_RING_GRANT_ORDER) {
1535 pr_info("Invalid max_ring_order (%d), will use default max: %d.\n",
1536 xen_blkif_max_ring_order, XENBUS_MAX_RING_GRANT_ORDER);
1537 xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
1538 }
1539
1540 if (xenblk_max_queues == 0)
1541 xenblk_max_queues = num_online_cpus();
1542
1543 rc = xen_blkif_interface_init();
1544 if (rc)
1545 goto failed_init;
1546
1547 rc = xen_blkif_xenbus_init();
1548 if (rc)
1549 goto failed_init;
1550
1551 failed_init:
1552 return rc;
1553 }
1554
1555 module_init(xen_blkif_init);
1556
xen_blkif_fini(void)1557 static void __exit xen_blkif_fini(void)
1558 {
1559 xen_blkif_xenbus_fini();
1560 xen_blkif_interface_fini();
1561 }
1562
1563 module_exit(xen_blkif_fini);
1564
1565 MODULE_DESCRIPTION("Virtual block device back-end driver");
1566 MODULE_LICENSE("Dual BSD/GPL");
1567 MODULE_ALIAS("xen-backend:vbd");
1568