1 /*- 2 * Copyright (C) 2012 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bio.h> 32 #include <sys/conf.h> 33 #include <sys/fcntl.h> 34 #include <sys/kthread.h> 35 #include <sys/module.h> 36 #include <sys/proc.h> 37 #include <sys/syscallsubr.h> 38 #include <sys/sysctl.h> 39 #include <sys/sysproto.h> 40 #include <sys/systm.h> 41 #include <sys/unistd.h> 42 43 #include <geom/geom.h> 44 45 #include "nvme_private.h" 46 47 struct nvme_io_test_thread { 48 49 uint32_t idx; 50 struct nvme_namespace *ns; 51 enum nvme_nvm_opcode opc; 52 struct timeval start; 53 void *buf; 54 uint32_t size; 55 uint32_t time; 56 uint32_t io_completed; 57 }; 58 59 struct nvme_io_test_internal { 60 61 struct nvme_namespace *ns; 62 enum nvme_nvm_opcode opc; 63 struct timeval start; 64 uint32_t time; 65 uint32_t size; 66 uint32_t td_active; 67 uint32_t td_idx; 68 uint32_t flags; 69 uint32_t io_completed[NVME_TEST_MAX_THREADS]; 70 }; 71 72 static void 73 nvme_ns_bio_test_cb(struct bio *bio) 74 { 75 struct mtx *mtx; 76 77 mtx = mtx_pool_find(mtxpool_sleep, bio); 78 mtx_lock(mtx); 79 wakeup(bio); 80 mtx_unlock(mtx); 81 } 82 83 static void 84 nvme_ns_bio_test(void *arg) 85 { 86 struct nvme_io_test_internal *io_test = arg; 87 struct cdevsw *csw; 88 struct mtx *mtx; 89 struct bio *bio; 90 struct cdev *dev; 91 void *buf; 92 struct timeval t; 93 uint64_t offset; 94 uint32_t idx, io_completed = 0; 95 #if __FreeBSD_version >= 900017 96 int ref; 97 #endif 98 99 buf = malloc(io_test->size, M_NVME, M_NOWAIT); 100 idx = atomic_fetchadd_int(&io_test->td_idx, 1); 101 dev = io_test->ns->cdev; 102 103 offset = idx * 2048 * nvme_ns_get_sector_size(io_test->ns); 104 105 while (1) { 106 107 bio = g_alloc_bio(); 108 109 memset(bio, 0, sizeof(*bio)); 110 bio->bio_cmd = (io_test->opc == NVME_OPC_READ) ? 111 BIO_READ : BIO_WRITE; 112 bio->bio_done = nvme_ns_bio_test_cb; 113 bio->bio_dev = dev; 114 bio->bio_offset = offset; 115 bio->bio_data = buf; 116 bio->bio_bcount = io_test->size; 117 118 if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 119 #if __FreeBSD_version >= 900017 120 csw = dev_refthread(dev, &ref); 121 #else 122 csw = dev_refthread(dev); 123 #endif 124 } else 125 csw = dev->si_devsw; 126 127 mtx = mtx_pool_find(mtxpool_sleep, bio); 128 mtx_lock(mtx); 129 (*csw->d_strategy)(bio); 130 msleep(bio, mtx, PRIBIO, "biotestwait", 0); 131 mtx_unlock(mtx); 132 133 if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 134 #if __FreeBSD_version >= 900017 135 dev_relthread(dev, ref); 136 #else 137 dev_relthread(dev); 138 #endif 139 } 140 141 if ((bio->bio_flags & BIO_ERROR) || (bio->bio_resid > 0)) 142 break; 143 144 g_destroy_bio(bio); 145 146 io_completed++; 147 148 getmicrouptime(&t); 149 timevalsub(&t, &io_test->start); 150 151 if (t.tv_sec >= io_test->time) 152 break; 153 154 offset += io_test->size; 155 if ((offset + io_test->size) > nvme_ns_get_size(io_test->ns)) 156 offset = 0; 157 } 158 159 io_test->io_completed[idx] = io_completed; 160 wakeup_one(io_test); 161 162 free(buf, M_NVME); 163 164 atomic_subtract_int(&io_test->td_active, 1); 165 mb(); 166 167 #if __FreeBSD_version >= 800000 168 kthread_exit(); 169 #else 170 kthread_exit(0); 171 #endif 172 } 173 174 static void 175 nvme_ns_io_test_cb(void *arg, const struct nvme_completion *status) 176 { 177 struct nvme_io_test_thread *tth = arg; 178 struct timeval t; 179 180 tth->io_completed++; 181 182 if (status->sf_sc || status->sf_sct) { 183 printf("%s: error occurred\n", __func__); 184 wakeup_one(tth); 185 return; 186 } 187 188 getmicrouptime(&t); 189 timevalsub(&t, &tth->start); 190 191 if (t.tv_sec >= tth->time) { 192 wakeup_one(tth); 193 return; 194 } 195 196 switch (tth->opc) { 197 case NVME_OPC_WRITE: 198 nvme_ns_cmd_write(tth->ns, tth->buf, tth->idx * 2048, 199 tth->size/nvme_ns_get_sector_size(tth->ns), 200 nvme_ns_io_test_cb, tth); 201 break; 202 case NVME_OPC_READ: 203 nvme_ns_cmd_read(tth->ns, tth->buf, tth->idx * 2048, 204 tth->size/nvme_ns_get_sector_size(tth->ns), 205 nvme_ns_io_test_cb, tth); 206 break; 207 default: 208 break; 209 } 210 } 211 212 static void 213 nvme_ns_io_test(void *arg) 214 { 215 struct nvme_io_test_internal *io_test = arg; 216 struct nvme_io_test_thread *tth; 217 struct nvme_completion cpl; 218 int error; 219 220 tth = malloc(sizeof(*tth), M_NVME, M_NOWAIT | M_ZERO); 221 tth->ns = io_test->ns; 222 tth->opc = io_test->opc; 223 memcpy(&tth->start, &io_test->start, sizeof(tth->start)); 224 tth->buf = malloc(io_test->size, M_NVME, M_NOWAIT); 225 tth->size = io_test->size; 226 tth->time = io_test->time; 227 tth->idx = atomic_fetchadd_int(&io_test->td_idx, 1); 228 229 memset(&cpl, 0, sizeof(cpl)); 230 231 nvme_ns_io_test_cb(tth, &cpl); 232 233 error = tsleep(tth, 0, "test_wait", tth->time*hz*2); 234 235 if (error) 236 printf("%s: error = %d\n", __func__, error); 237 238 io_test->io_completed[tth->idx] = tth->io_completed; 239 wakeup_one(io_test); 240 241 free(tth->buf, M_NVME); 242 free(tth, M_NVME); 243 244 atomic_subtract_int(&io_test->td_active, 1); 245 mb(); 246 247 #if __FreeBSD_version >= 800004 248 kthread_exit(); 249 #else 250 kthread_exit(0); 251 #endif 252 } 253 254 void 255 nvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg) 256 { 257 struct nvme_io_test *io_test; 258 struct nvme_io_test_internal *io_test_internal; 259 void (*fn)(void *); 260 int i; 261 262 io_test = (struct nvme_io_test *)arg; 263 264 if ((io_test->opc != NVME_OPC_READ) && 265 (io_test->opc != NVME_OPC_WRITE)) 266 return; 267 268 if (io_test->size % nvme_ns_get_sector_size(ns)) 269 return; 270 271 io_test_internal = malloc(sizeof(*io_test_internal), M_NVME, 272 M_NOWAIT | M_ZERO); 273 io_test_internal->opc = io_test->opc; 274 io_test_internal->ns = ns; 275 io_test_internal->td_active = io_test->num_threads; 276 io_test_internal->time = io_test->time; 277 io_test_internal->size = io_test->size; 278 io_test_internal->flags = io_test->flags; 279 280 if (cmd == NVME_IO_TEST) 281 fn = nvme_ns_io_test; 282 else 283 fn = nvme_ns_bio_test; 284 285 getmicrouptime(&io_test_internal->start); 286 287 for (i = 0; i < io_test->num_threads; i++) 288 #if __FreeBSD_version >= 800004 289 kthread_add(fn, io_test_internal, 290 NULL, NULL, 0, 0, "nvme_io_test[%d]", i); 291 #else 292 kthread_create(fn, io_test_internal, 293 NULL, 0, 0, "nvme_io_test[%d]", i); 294 #endif 295 296 tsleep(io_test_internal, 0, "nvme_test", io_test->time * 2 * hz); 297 298 while (io_test_internal->td_active > 0) 299 DELAY(10); 300 301 memcpy(io_test->io_completed, io_test_internal->io_completed, 302 sizeof(io_test->io_completed)); 303 304 free(io_test_internal, M_NVME); 305 } 306