/* * Copyright (c) 1998 Michael Smith <msmith@freebsd.org> * Copyright 2015 Toomas Soome <tsoome@me.com> * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include <sys/cdefs.h> #include <sys/param.h> /* * Simple hashed block cache */ #include <sys/stdint.h> #include <stand.h> #include <string.h> #include <strings.h> #include "bootstrap.h" /* #define BCACHE_DEBUG */ #ifdef BCACHE_DEBUG #define DPRINTF(fmt, args...) printf("%s: " fmt "\n", __func__, ## args) #else #define DPRINTF(fmt, args...) ((void)0) #endif struct bcachectl { daddr_t bc_blkno; int bc_count; }; /* * bcache per device node. cache is allocated on device first open and freed * on last close, to save memory. The issue there is the size; biosdisk * supports up to 31 (0x1f) devices. Classic setup would use single disk * to boot from, but this has changed with zfs. */ struct bcache { struct bcachectl *bcache_ctl; caddr_t bcache_data; size_t bcache_nblks; size_t ra; }; static uint_t bcache_total_nblks; /* set by bcache_init */ static uint_t bcache_blksize; /* set by bcache_init */ static uint_t bcache_numdev; /* set by bcache_add_dev */ /* statistics */ static uint_t bcache_units; /* number of devices with cache */ static uint_t bcache_unit_nblks; /* nblocks per unit */ static uint_t bcache_hits; static uint_t bcache_misses; static uint_t bcache_ops; static uint_t bcache_bypasses; static uint_t bcache_bcount; static uint_t bcache_rablks; #define BHASH(bc, blkno) ((blkno) & ((bc)->bcache_nblks - 1)) #define BCACHE_LOOKUP(bc, blkno) \ ((bc)->bcache_ctl[BHASH((bc), (blkno))].bc_blkno != (blkno)) #define BCACHE_READAHEAD 256 #define BCACHE_MINREADAHEAD 32 static void bcache_invalidate(struct bcache *bc, daddr_t blkno); static void bcache_insert(struct bcache *bc, daddr_t blkno); static void bcache_free_instance(struct bcache *bc); /* * Initialise the cache for (nblks) of (bsize). */ void bcache_init(size_t nblks, size_t bsize) { /* set up control data */ bcache_total_nblks = nblks; bcache_blksize = bsize; } /* * add number of devices to bcache. we have to divide cache space * between the devices, so bcache_add_dev() can be used to set up the * number. The issue is, we need to get the number before actual allocations. * bcache_add_dev() is supposed to be called from device init() call, so the * assumption is, devsw dv_init is called for plain devices first, and * for zfs, last. */ void bcache_add_dev(int devices) { bcache_numdev += devices; } void * bcache_allocate(void) { uint_t i; struct bcache *bc = malloc(sizeof (struct bcache)); int disks = bcache_numdev; if (disks == 0) disks = 1; /* safe guard */ if (bc == NULL) { errno = ENOMEM; return (bc); } /* * the bcache block count must be power of 2 for hash function */ i = fls(disks) - 1; /* highbit - 1 */ if (disks > (1 << i)) /* next power of 2 */ i++; bc->bcache_nblks = bcache_total_nblks >> i; bcache_unit_nblks = bc->bcache_nblks; bc->bcache_data = malloc(bc->bcache_nblks * bcache_blksize); if (bc->bcache_data == NULL) { /* dont error out yet. fall back to 32 blocks and try again */ bc->bcache_nblks = 32; bc->bcache_data = malloc(bc->bcache_nblks * bcache_blksize + sizeof (uint32_t)); } bc->bcache_ctl = malloc(bc->bcache_nblks * sizeof (struct bcachectl)); if ((bc->bcache_data == NULL) || (bc->bcache_ctl == NULL)) { bcache_free_instance(bc); errno = ENOMEM; return (NULL); } /* Flush the cache */ for (i = 0; i < bc->bcache_nblks; i++) { bc->bcache_ctl[i].bc_count = -1; bc->bcache_ctl[i].bc_blkno = -1; } bcache_units++; bc->ra = BCACHE_READAHEAD; /* optimistic read ahead */ return (bc); } void bcache_free(void *cache) { struct bcache *bc = cache; if (bc == NULL) return; bcache_free_instance(bc); bcache_units--; } /* * Handle a write request; write directly to the disk, and populate the * cache with the new values. */ static int write_strategy(void *devdata, int rw, daddr_t blk, size_t size, char *buf, size_t *rsize) { struct bcache_devdata *dd = (struct bcache_devdata *)devdata; struct bcache *bc = dd->dv_cache; daddr_t i, nblk; nblk = size / bcache_blksize; /* Invalidate the blocks being written */ for (i = 0; i < nblk; i++) { bcache_invalidate(bc, blk + i); } /* Write the blocks */ return (dd->dv_strategy(dd->dv_devdata, rw, blk, size, buf, rsize)); } /* * Handle a read request; fill in parts of the request that can * be satisfied by the cache, use the supplied strategy routine to do * device I/O and then use the I/O results to populate the cache. */ static int read_strategy(void *devdata, int rw, daddr_t blk, size_t size, char *buf, size_t *rsize) { struct bcache_devdata *dd = devdata; struct bcache *bc = dd->dv_cache; size_t i, nblk, p_size, r_size, complete, ra; int result; daddr_t p_blk; caddr_t p_buf; if (bc == NULL) { errno = ENODEV; return (-1); } if (rsize != NULL) *rsize = 0; nblk = size / bcache_blksize; if (nblk == 0 && size != 0) nblk++; result = 0; complete = 1; /* Satisfy any cache hits up front, break on first miss */ for (i = 0; i < nblk; i++) { if (BCACHE_LOOKUP(bc, (daddr_t)(blk + i))) { bcache_misses += (nblk - i); complete = 0; if (nblk - i > BCACHE_MINREADAHEAD && bc->ra > BCACHE_MINREADAHEAD) bc->ra >>= 1; /* reduce read ahead */ break; } else { bcache_hits++; } } if (complete) { /* whole set was in cache, return it */ if (bc->ra < BCACHE_READAHEAD) bc->ra <<= 1; /* increase read ahead */ bcopy(bc->bcache_data + (bcache_blksize * BHASH(bc, blk)), buf, size); goto done; } /* * Fill in any misses. From check we have i pointing to first missing * block, read in all remaining blocks + readahead. * We have space at least for nblk - i before bcache wraps. */ p_blk = blk + i; p_buf = bc->bcache_data + (bcache_blksize * BHASH(bc, p_blk)); r_size = bc->bcache_nblks - BHASH(bc, p_blk); /* remaining blocks */ p_size = MIN(r_size, nblk - i); /* read at least those blocks */ /* * The read ahead size setup. * While the read ahead can save us IO, it also can complicate things: * 1. We do not want to read ahead by wrapping around the * bcache end - this would complicate the cache management. * 2. We are using bc->ra as dynamic hint for read ahead size, * detected cache hits will increase the read-ahead block count, * and misses will decrease, see the code above. * 3. The bcache is sized by 512B blocks, however, the underlying device * may have a larger sector size, and we should perform the IO by * taking into account these larger sector sizes. We could solve * this by passing the sector size to bcache_allocate(), or by * using ioctl(), but in this version we are using the constant, * 16 blocks, and are rounding read ahead block count down to * multiple of 16. Using the constant has two reasons, we are not * entirely sure if the BIOS disk interface is providing the * correct value for sector size. And secondly, this way we get * the most conservative setup for the ra. * * The selection of multiple of 16 blocks (8KB) is quite arbitrary, * however, we want to cover CDs (2K) and 4K disks. * bcache_allocate() will always fall back to a minimum of 32 blocks. * Our choice of 16 read ahead blocks will always fit inside the bcache. */ if ((rw & F_NORA) == F_NORA) ra = 0; else ra = bc->bcache_nblks - BHASH(bc, p_blk + p_size); if (ra != 0 && ra != bc->bcache_nblks) { /* do we have RA space? */ ra = MIN(bc->ra, ra - 1); ra = rounddown(ra, 16); /* multiple of 16 blocks */ p_size += ra; } /* invalidate bcache */ for (i = 0; i < p_size; i++) { bcache_invalidate(bc, p_blk + i); } r_size = 0; /* * with read-ahead, it may happen we are attempting to read past * disk end, as bcache has no information about disk size. * in such case we should get partial read if some blocks can be * read or error, if no blocks can be read. * in either case we should return the data in bcache and only * return error if there is no data. */ rw &= F_MASK; result = dd->dv_strategy(dd->dv_devdata, rw, p_blk, p_size * bcache_blksize, p_buf, &r_size); r_size /= bcache_blksize; for (i = 0; i < r_size; i++) bcache_insert(bc, p_blk + i); /* update ra statistics */ if (r_size != 0) { if (r_size < p_size) bcache_rablks += (p_size - r_size); else bcache_rablks += ra; } /* check how much data can we copy */ for (i = 0; i < nblk; i++) { if (BCACHE_LOOKUP(bc, (daddr_t)(blk + i))) break; } if (size > i * bcache_blksize) size = i * bcache_blksize; if (size != 0) { bcopy(bc->bcache_data + (bcache_blksize * BHASH(bc, blk)), buf, size); result = 0; } done: if ((result == 0) && (rsize != NULL)) *rsize = size; return (result); } /* * Requests larger than 1/2 cache size will be bypassed and go * directly to the disk. XXX tune this. */ int bcache_strategy(void *devdata, int rw, daddr_t blk, size_t size, char *buf, size_t *rsize) { struct bcache_devdata *dd = (struct bcache_devdata *)devdata; struct bcache *bc = dd->dv_cache; uint_t bcache_nblks = 0; int nblk, cblk, ret; size_t csize, isize, total; bcache_ops++; if (bc != NULL) bcache_nblks = bc->bcache_nblks; /* bypass large requests, or when the cache is inactive */ if (bc == NULL || ((size * 2 / bcache_blksize) > bcache_nblks)) { DPRINTF("bypass %zu from %jd", size / bcache_blksize, (intmax_t)blk); bcache_bypasses++; rw &= F_MASK; return (dd->dv_strategy(dd->dv_devdata, rw, blk, size, buf, rsize)); } switch (rw & F_MASK) { case F_READ: nblk = size / bcache_blksize; if (size != 0 && nblk == 0) nblk++; /* read at least one block */ ret = 0; total = 0; while (size) { /* # of blocks left */ cblk = bcache_nblks - BHASH(bc, blk); cblk = MIN(cblk, nblk); if (size <= bcache_blksize) csize = size; else csize = cblk * bcache_blksize; ret = read_strategy(devdata, rw, blk, csize, buf + total, &isize); /* * we may have error from read ahead, if we have read * some data return partial read. */ if (ret != 0 || isize == 0) { if (total != 0) ret = 0; break; } blk += isize / bcache_blksize; total += isize; size -= isize; nblk = size / bcache_blksize; } if (rsize) *rsize = total; return (ret); case F_WRITE: return (write_strategy(devdata, F_WRITE, blk, size, buf, rsize)); } return (-1); } /* * Free allocated bcache instance */ static void bcache_free_instance(struct bcache *bc) { if (bc != NULL) { free(bc->bcache_ctl); free(bc->bcache_data); free(bc); } } /* * Insert a block into the cache. */ static void bcache_insert(struct bcache *bc, daddr_t blkno) { uint_t cand; cand = BHASH(bc, blkno); DPRINTF("insert blk %jd -> %u # %d", (intmax_t)blkno, cand, bcache_bcount); bc->bcache_ctl[cand].bc_blkno = blkno; bc->bcache_ctl[cand].bc_count = bcache_bcount++; } /* * Invalidate a block from the cache. */ static void bcache_invalidate(struct bcache *bc, daddr_t blkno) { uint_t i; i = BHASH(bc, blkno); if (bc->bcache_ctl[i].bc_blkno == blkno) { bc->bcache_ctl[i].bc_count = -1; bc->bcache_ctl[i].bc_blkno = -1; DPRINTF("invalidate blk %jd", (intmax_t)blkno); } } COMMAND_SET(bcachestat, "bcachestat", "get disk block cache stats", command_bcache); static int command_bcache(int argc, char *argv[] __unused) { if (argc != 1) { command_errmsg = "wrong number of arguments"; return (CMD_ERROR); } printf("\ncache blocks: %u\n", bcache_total_nblks); printf("cache blocksz: %u\n", bcache_blksize); printf("cache readahead: %u\n", bcache_rablks); printf("unit cache blocks: %u\n", bcache_unit_nblks); printf("cached units: %u\n", bcache_units); printf("%u ops %u bypasses %u hits %u misses\n", bcache_ops, bcache_bypasses, bcache_hits, bcache_misses); return (CMD_OK); }