1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2012-2013 Intel Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/bio.h> 34 #include <sys/conf.h> 35 #include <sys/fcntl.h> 36 #include <sys/kthread.h> 37 #include <sys/module.h> 38 #include <sys/proc.h> 39 #include <sys/syscallsubr.h> 40 #include <sys/sysctl.h> 41 #include <sys/sysproto.h> 42 #include <sys/systm.h> 43 #include <sys/unistd.h> 44 45 #include <geom/geom.h> 46 47 #include "nvme_private.h" 48 49 struct nvme_io_test_thread { 50 51 uint32_t idx; 52 struct nvme_namespace *ns; 53 enum nvme_nvm_opcode opc; 54 struct timeval start; 55 void *buf; 56 uint32_t size; 57 uint32_t time; 58 uint64_t io_completed; 59 }; 60 61 struct nvme_io_test_internal { 62 63 struct nvme_namespace *ns; 64 enum nvme_nvm_opcode opc; 65 struct timeval start; 66 uint32_t time; 67 uint32_t size; 68 uint32_t td_active; 69 uint32_t td_idx; 70 uint32_t flags; 71 uint64_t io_completed[NVME_TEST_MAX_THREADS]; 72 }; 73 74 static void 75 nvme_ns_bio_test_cb(struct bio *bio) 76 { 77 struct mtx *mtx; 78 79 mtx = mtx_pool_find(mtxpool_sleep, bio); 80 mtx_lock(mtx); 81 wakeup(bio); 82 mtx_unlock(mtx); 83 } 84 85 static void 86 nvme_ns_bio_test(void *arg) 87 { 88 struct nvme_io_test_internal *io_test = arg; 89 struct cdevsw *csw; 90 struct mtx *mtx; 91 struct bio *bio; 92 struct cdev *dev; 93 void *buf; 94 struct timeval t; 95 uint64_t io_completed = 0, offset; 96 uint32_t idx; 97 int ref; 98 99 buf = malloc(io_test->size, M_NVME, M_WAITOK); 100 idx = atomic_fetchadd_int(&io_test->td_idx, 1); 101 dev = io_test->ns->cdev; 102 103 offset = idx * 2048ULL * nvme_ns_get_sector_size(io_test->ns); 104 105 while (1) { 106 107 bio = g_alloc_bio(); 108 109 memset(bio, 0, sizeof(*bio)); 110 bio->bio_cmd = (io_test->opc == NVME_OPC_READ) ? 111 BIO_READ : BIO_WRITE; 112 bio->bio_done = nvme_ns_bio_test_cb; 113 bio->bio_dev = dev; 114 bio->bio_offset = offset; 115 bio->bio_data = buf; 116 bio->bio_bcount = io_test->size; 117 118 if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 119 csw = dev_refthread(dev, &ref); 120 } else 121 csw = dev->si_devsw; 122 123 if (csw == NULL) 124 panic("Unable to retrieve device switch"); 125 mtx = mtx_pool_find(mtxpool_sleep, bio); 126 mtx_lock(mtx); 127 (*csw->d_strategy)(bio); 128 msleep(bio, mtx, PRIBIO, "biotestwait", 0); 129 mtx_unlock(mtx); 130 131 if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 132 dev_relthread(dev, ref); 133 } 134 135 if ((bio->bio_flags & BIO_ERROR) || (bio->bio_resid > 0)) 136 break; 137 138 g_destroy_bio(bio); 139 140 io_completed++; 141 142 getmicrouptime(&t); 143 timevalsub(&t, &io_test->start); 144 145 if (t.tv_sec >= io_test->time) 146 break; 147 148 offset += io_test->size; 149 if ((offset + io_test->size) > nvme_ns_get_size(io_test->ns)) 150 offset = 0; 151 } 152 153 io_test->io_completed[idx] = io_completed; 154 wakeup_one(io_test); 155 156 free(buf, M_NVME); 157 158 atomic_subtract_int(&io_test->td_active, 1); 159 mb(); 160 161 kthread_exit(); 162 } 163 164 static void 165 nvme_ns_io_test_cb(void *arg, const struct nvme_completion *cpl) 166 { 167 struct nvme_io_test_thread *tth = arg; 168 struct timeval t; 169 170 tth->io_completed++; 171 172 if (nvme_completion_is_error(cpl)) { 173 printf("%s: error occurred\n", __func__); 174 wakeup_one(tth); 175 return; 176 } 177 178 getmicrouptime(&t); 179 timevalsub(&t, &tth->start); 180 181 if (t.tv_sec >= tth->time) { 182 wakeup_one(tth); 183 return; 184 } 185 186 switch (tth->opc) { 187 case NVME_OPC_WRITE: 188 nvme_ns_cmd_write(tth->ns, tth->buf, tth->idx * 2048, 189 tth->size/nvme_ns_get_sector_size(tth->ns), 190 nvme_ns_io_test_cb, tth); 191 break; 192 case NVME_OPC_READ: 193 nvme_ns_cmd_read(tth->ns, tth->buf, tth->idx * 2048, 194 tth->size/nvme_ns_get_sector_size(tth->ns), 195 nvme_ns_io_test_cb, tth); 196 break; 197 default: 198 break; 199 } 200 } 201 202 static void 203 nvme_ns_io_test(void *arg) 204 { 205 struct nvme_io_test_internal *io_test = arg; 206 struct nvme_io_test_thread *tth; 207 struct nvme_completion cpl; 208 int error; 209 210 tth = malloc(sizeof(*tth), M_NVME, M_WAITOK | M_ZERO); 211 tth->ns = io_test->ns; 212 tth->opc = io_test->opc; 213 memcpy(&tth->start, &io_test->start, sizeof(tth->start)); 214 tth->buf = malloc(io_test->size, M_NVME, M_WAITOK); 215 tth->size = io_test->size; 216 tth->time = io_test->time; 217 tth->idx = atomic_fetchadd_int(&io_test->td_idx, 1); 218 219 memset(&cpl, 0, sizeof(cpl)); 220 221 nvme_ns_io_test_cb(tth, &cpl); 222 223 error = tsleep(tth, 0, "test_wait", tth->time*hz*2); 224 225 if (error) 226 printf("%s: error = %d\n", __func__, error); 227 228 io_test->io_completed[tth->idx] = tth->io_completed; 229 wakeup_one(io_test); 230 231 free(tth->buf, M_NVME); 232 free(tth, M_NVME); 233 234 atomic_subtract_int(&io_test->td_active, 1); 235 mb(); 236 237 kthread_exit(); 238 } 239 240 void 241 nvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg) 242 { 243 struct nvme_io_test *io_test; 244 struct nvme_io_test_internal *io_test_internal; 245 void (*fn)(void *); 246 int i; 247 248 io_test = (struct nvme_io_test *)arg; 249 250 if ((io_test->opc != NVME_OPC_READ) && 251 (io_test->opc != NVME_OPC_WRITE)) 252 return; 253 254 if (io_test->size % nvme_ns_get_sector_size(ns)) 255 return; 256 257 io_test_internal = malloc(sizeof(*io_test_internal), M_NVME, 258 M_WAITOK | M_ZERO); 259 io_test_internal->opc = io_test->opc; 260 io_test_internal->ns = ns; 261 io_test_internal->td_active = io_test->num_threads; 262 io_test_internal->time = io_test->time; 263 io_test_internal->size = io_test->size; 264 io_test_internal->flags = io_test->flags; 265 266 if (cmd == NVME_IO_TEST) 267 fn = nvme_ns_io_test; 268 else 269 fn = nvme_ns_bio_test; 270 271 getmicrouptime(&io_test_internal->start); 272 273 for (i = 0; i < io_test->num_threads; i++) 274 kthread_add(fn, io_test_internal, 275 NULL, NULL, 0, 0, "nvme_io_test[%d]", i); 276 277 tsleep(io_test_internal, 0, "nvme_test", io_test->time * 2 * hz); 278 279 while (io_test_internal->td_active > 0) 280 DELAY(10); 281 282 memcpy(io_test->io_completed, io_test_internal->io_completed, 283 sizeof(io_test->io_completed)); 284 285 free(io_test_internal, M_NVME); 286 } 287