1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2012-2013 Intel Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/bio.h> 34 #include <sys/conf.h> 35 #include <sys/fcntl.h> 36 #include <sys/kthread.h> 37 #include <sys/module.h> 38 #include <sys/proc.h> 39 #include <sys/syscallsubr.h> 40 #include <sys/sysctl.h> 41 #include <sys/sysproto.h> 42 #include <sys/systm.h> 43 #include <sys/unistd.h> 44 45 #include <geom/geom.h> 46 47 #include "nvme_private.h" 48 49 struct nvme_io_test_thread { 50 51 uint32_t idx; 52 struct nvme_namespace *ns; 53 enum nvme_nvm_opcode opc; 54 struct timeval start; 55 void *buf; 56 uint32_t size; 57 uint32_t time; 58 uint64_t io_completed; 59 }; 60 61 struct nvme_io_test_internal { 62 63 struct nvme_namespace *ns; 64 enum nvme_nvm_opcode opc; 65 struct timeval start; 66 uint32_t time; 67 uint32_t size; 68 uint32_t td_active; 69 uint32_t td_idx; 70 uint32_t flags; 71 uint64_t io_completed[NVME_TEST_MAX_THREADS]; 72 }; 73 74 static void 75 nvme_ns_bio_test_cb(struct bio *bio) 76 { 77 struct mtx *mtx; 78 79 mtx = mtx_pool_find(mtxpool_sleep, bio); 80 mtx_lock(mtx); 81 wakeup(bio); 82 mtx_unlock(mtx); 83 } 84 85 static void 86 nvme_ns_bio_test(void *arg) 87 { 88 struct nvme_io_test_internal *io_test = arg; 89 struct cdevsw *csw; 90 struct mtx *mtx; 91 struct bio *bio; 92 struct cdev *dev; 93 void *buf; 94 struct timeval t; 95 uint64_t io_completed = 0, offset; 96 uint32_t idx; 97 #if __FreeBSD_version >= 900017 98 int ref; 99 #endif 100 101 buf = malloc(io_test->size, M_NVME, M_WAITOK); 102 idx = atomic_fetchadd_int(&io_test->td_idx, 1); 103 dev = io_test->ns->cdev; 104 105 offset = idx * 2048 * nvme_ns_get_sector_size(io_test->ns); 106 107 while (1) { 108 109 bio = g_alloc_bio(); 110 111 memset(bio, 0, sizeof(*bio)); 112 bio->bio_cmd = (io_test->opc == NVME_OPC_READ) ? 113 BIO_READ : BIO_WRITE; 114 bio->bio_done = nvme_ns_bio_test_cb; 115 bio->bio_dev = dev; 116 bio->bio_offset = offset; 117 bio->bio_data = buf; 118 bio->bio_bcount = io_test->size; 119 120 if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 121 #if __FreeBSD_version >= 900017 122 csw = dev_refthread(dev, &ref); 123 #else 124 csw = dev_refthread(dev); 125 #endif 126 } else 127 csw = dev->si_devsw; 128 129 mtx = mtx_pool_find(mtxpool_sleep, bio); 130 mtx_lock(mtx); 131 (*csw->d_strategy)(bio); 132 msleep(bio, mtx, PRIBIO, "biotestwait", 0); 133 mtx_unlock(mtx); 134 135 if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) { 136 #if __FreeBSD_version >= 900017 137 dev_relthread(dev, ref); 138 #else 139 dev_relthread(dev); 140 #endif 141 } 142 143 if ((bio->bio_flags & BIO_ERROR) || (bio->bio_resid > 0)) 144 break; 145 146 g_destroy_bio(bio); 147 148 io_completed++; 149 150 getmicrouptime(&t); 151 timevalsub(&t, &io_test->start); 152 153 if (t.tv_sec >= io_test->time) 154 break; 155 156 offset += io_test->size; 157 if ((offset + io_test->size) > nvme_ns_get_size(io_test->ns)) 158 offset = 0; 159 } 160 161 io_test->io_completed[idx] = io_completed; 162 wakeup_one(io_test); 163 164 free(buf, M_NVME); 165 166 atomic_subtract_int(&io_test->td_active, 1); 167 mb(); 168 169 #if __FreeBSD_version >= 800000 170 kthread_exit(); 171 #else 172 kthread_exit(0); 173 #endif 174 } 175 176 static void 177 nvme_ns_io_test_cb(void *arg, const struct nvme_completion *cpl) 178 { 179 struct nvme_io_test_thread *tth = arg; 180 struct timeval t; 181 182 tth->io_completed++; 183 184 if (nvme_completion_is_error(cpl)) { 185 printf("%s: error occurred\n", __func__); 186 wakeup_one(tth); 187 return; 188 } 189 190 getmicrouptime(&t); 191 timevalsub(&t, &tth->start); 192 193 if (t.tv_sec >= tth->time) { 194 wakeup_one(tth); 195 return; 196 } 197 198 switch (tth->opc) { 199 case NVME_OPC_WRITE: 200 nvme_ns_cmd_write(tth->ns, tth->buf, tth->idx * 2048, 201 tth->size/nvme_ns_get_sector_size(tth->ns), 202 nvme_ns_io_test_cb, tth); 203 break; 204 case NVME_OPC_READ: 205 nvme_ns_cmd_read(tth->ns, tth->buf, tth->idx * 2048, 206 tth->size/nvme_ns_get_sector_size(tth->ns), 207 nvme_ns_io_test_cb, tth); 208 break; 209 default: 210 break; 211 } 212 } 213 214 static void 215 nvme_ns_io_test(void *arg) 216 { 217 struct nvme_io_test_internal *io_test = arg; 218 struct nvme_io_test_thread *tth; 219 struct nvme_completion cpl; 220 int error; 221 222 tth = malloc(sizeof(*tth), M_NVME, M_WAITOK | M_ZERO); 223 tth->ns = io_test->ns; 224 tth->opc = io_test->opc; 225 memcpy(&tth->start, &io_test->start, sizeof(tth->start)); 226 tth->buf = malloc(io_test->size, M_NVME, M_WAITOK); 227 tth->size = io_test->size; 228 tth->time = io_test->time; 229 tth->idx = atomic_fetchadd_int(&io_test->td_idx, 1); 230 231 memset(&cpl, 0, sizeof(cpl)); 232 233 nvme_ns_io_test_cb(tth, &cpl); 234 235 error = tsleep(tth, 0, "test_wait", tth->time*hz*2); 236 237 if (error) 238 printf("%s: error = %d\n", __func__, error); 239 240 io_test->io_completed[tth->idx] = tth->io_completed; 241 wakeup_one(io_test); 242 243 free(tth->buf, M_NVME); 244 free(tth, M_NVME); 245 246 atomic_subtract_int(&io_test->td_active, 1); 247 mb(); 248 249 #if __FreeBSD_version >= 800004 250 kthread_exit(); 251 #else 252 kthread_exit(0); 253 #endif 254 } 255 256 void 257 nvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg) 258 { 259 struct nvme_io_test *io_test; 260 struct nvme_io_test_internal *io_test_internal; 261 void (*fn)(void *); 262 int i; 263 264 io_test = (struct nvme_io_test *)arg; 265 266 if ((io_test->opc != NVME_OPC_READ) && 267 (io_test->opc != NVME_OPC_WRITE)) 268 return; 269 270 if (io_test->size % nvme_ns_get_sector_size(ns)) 271 return; 272 273 io_test_internal = malloc(sizeof(*io_test_internal), M_NVME, 274 M_WAITOK | M_ZERO); 275 io_test_internal->opc = io_test->opc; 276 io_test_internal->ns = ns; 277 io_test_internal->td_active = io_test->num_threads; 278 io_test_internal->time = io_test->time; 279 io_test_internal->size = io_test->size; 280 io_test_internal->flags = io_test->flags; 281 282 if (cmd == NVME_IO_TEST) 283 fn = nvme_ns_io_test; 284 else 285 fn = nvme_ns_bio_test; 286 287 getmicrouptime(&io_test_internal->start); 288 289 for (i = 0; i < io_test->num_threads; i++) 290 #if __FreeBSD_version >= 800004 291 kthread_add(fn, io_test_internal, 292 NULL, NULL, 0, 0, "nvme_io_test[%d]", i); 293 #else 294 kthread_create(fn, io_test_internal, 295 NULL, 0, 0, "nvme_io_test[%d]", i); 296 #endif 297 298 tsleep(io_test_internal, 0, "nvme_test", io_test->time * 2 * hz); 299 300 while (io_test_internal->td_active > 0) 301 DELAY(10); 302 303 memcpy(io_test->io_completed, io_test_internal->io_completed, 304 sizeof(io_test->io_completed)); 305 306 free(io_test_internal, M_NVME); 307 } 308