1#!/bin/sh 2 3# 4# Copyright (c) 2017 Dell EMC Isilon 5# All rights reserved. 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26# SUCH DAMAGE. 27# 28 29# Scenario: backup of a number (> maxvnodes) of small files. 30 31# Test how vnlru impacts open(2). 32 33# FreeBSD 12.0-CURRENT #0 r312620: Mon Jan 23 23:27:46 CET 2017 34# /usr/src/sys/amd64/compile/BENCH amd64 35# ./tvnlru.sh 36# FAIL 328/306502 37# files = 500000, maxvnodes = 500000, ave=0.000018, max=0.000328, elapsed 4 38# files = 500000, maxvnodes = 500000, ave=0.000018, max=0.000155, elapsed 4 39# files = 500000, maxvnodes = 500000, ave=0.000018, max=0.000227, elapsed 4 40# files = 500000, maxvnodes = 500000, ave=0.000014, max=0.000126, elapsed 3 41# 42# files = 1000000, maxvnodes = 500000, ave=0.000035, max=0.205627, elapsed 14 43# files = 1000000, maxvnodes = 500000, ave=0.000033, max=0.205185, elapsed 14 44# files = 1000000, maxvnodes = 500000, ave=0.000038, max=0.306502, elapsed 14 45# files = 1000000, maxvnodes = 500000, ave=0.000037, max=0.205177, elapsed 14 46 47. ../default.cfg 48[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 49[ `sysctl -n hw.physmem` -lt $(( 4 * 1024 * 1024 * 1024)) ] && exit 0 50 51files=1000000 52[ `sysctl -n kern.maxvnodes` -lt $files ] && exit 0 53 54log=/tmp/tvnlru.log 55dir=/tmp 56odir=`pwd` 57cd $dir 58sed '1,/^EOF/d' < $odir/$0 > $dir/tvnlru.c 59mycc -o tvnlru -Wall -Wextra -O0 -g tvnlru.c || exit 1 60rm -f tvnlru.c 61cd $odir 62 63mount | grep -q "on $mntpoint " && umount -f $mntpoint 64[ -c /dev/md$mdstart ] && mdconfig -d -u $mdstart 65mdconfig -a -t swap -s 2g -u $mdstart || exit 1 66newfs -n -b 4096 -f 512 -i 512 md$mdstart > /dev/null 67mount -o async /dev/md$mdstart $mntpoint 68 69ncpu=`sysctl -n hw.ncpu` 70[ $ncpu -lt 4 ] && { rm /tmp/tvnlru; exit 0; } 71ncpu=4 72inodes=`df -i $mntpoint | tail -1 | awk "{print \\$7 - $ncpu - 1}"` 73oldmx=`sysctl -n kern.maxvnodes` 74[ $files -gt $inodes ] && { echo "Disk too small"; files=$inodes; } 75[ $files -gt $oldmx ] && 76 { echo "$files exceed old maxvnods"; files=$oldmx; } 77newmaxvnodes=$((files / 2)) 78trap "sysctl kern.maxvnodes=$oldmx > /dev/null" EXIT SIGINT 79sysctl kern.maxvnodes=$newmaxvnodes > /dev/null 80 81# warmup 82cd $mntpoint 83t1=`/tmp/tvnlru $ncpu $newmaxvnodes $newmaxvnodes $mntpoint 2>/dev/null` 84cd $odir 85umount $mntpoint 86newfs -n -b 4096 -f 512 -i 512 md$mdstart > /dev/null 87mount -o async /dev/md$mdstart $mntpoint 88 89cd $mntpoint 90t1=`/tmp/tvnlru $ncpu $newmaxvnodes $newmaxvnodes $mntpoint 2>$log` 91cd $odir 92umount $mntpoint 93newfs -n -b 4096 -f 512 -i 512 md$mdstart > /dev/null 94mount -o async /dev/md$mdstart $mntpoint 95 96cd $mntpoint 97echo >> $log 98t2=`/tmp/tvnlru $ncpu $files $newmaxvnodes $mntpoint 2>>$log` 99s=$? 100cd $odir 101 102s=0 103for i in `jot 10`; do 104 mount | grep -q "on $mntpoint " || break 105 umount $mntpoint || sleep 2 106done 107mount | grep -q "on $mntpoint " && { s=2; umount -f $mntpoint; } 108mdconfig -d -u $mdstart 109[ $t2 -gt $((t1 * 3)) ] && { s=3; echo "Fail $t1/$t2"; cat $log; } 110rm -rf /tmp/tvnlru /tmp/tvnlru.log 111exit $s 112 113EOF 114#include <sys/param.h> 115#include <sys/mman.h> 116#include <sys/time.h> 117#include <sys/stat.h> 118#include <sys/wait.h> 119 120#include <machine/atomic.h> 121 122#include <err.h> 123#include <errno.h> 124#include <fcntl.h> 125#include <stdio.h> 126#include <stdlib.h> 127#include <time.h> 128#include <unistd.h> 129 130static volatile u_int *share; 131 132#define SYNC 0 133 134static long maxvnodes, parallel, tvnodes, vnodes; 135static char *mp; 136 137static void 138test(int idx) 139{ 140 struct timeval diff, start, stop; 141 time_t st; 142 uint64_t mx, tot, usec; 143 pid_t pid; 144 int fd, i, n; 145 char dir[80], file[80], help[80]; 146 147 atomic_add_int(&share[SYNC], 1); 148 while (share[SYNC] != (unsigned int)parallel) 149 ; 150 151 pid = getpid(); 152 snprintf(dir, sizeof(dir), "d%09ld", (long)pid); 153 if (mkdir(dir, 0700) == -1) 154 err(1, "mkdir(%s)", dir); 155 if (chdir(dir) == -1) 156 err(1, "chdir(%s)", dir); 157 158 for (i = 0; i < vnodes; i++) { 159 snprintf(file, sizeof(file), "f%09d", i); 160 if ((fd = open(file, O_RDWR | O_CREAT, DEFFILEMODE)) == -1) 161 err(1, "open(%s)", file); 162 close(fd); 163 } 164 165 snprintf(help, sizeof(help), "umount %s > /dev/null 2>&1", mp); 166 system(help); /* flush the cache */ 167 168 mx = 0; 169 n = 0; 170 st = time(NULL); 171 tot = 0; 172 for (i = 0; i < vnodes; i++) { 173 snprintf(file, sizeof(file), "f%09d", i); 174 gettimeofday(&start, NULL); 175 if ((fd = open(file, O_RDONLY)) == -1) 176 err(1, "open(%s)", file); 177 gettimeofday(&stop, NULL); 178 timersub(&stop, &start, &diff); 179 usec = ((uint64_t)1000000 * diff.tv_sec + diff.tv_usec); 180 tot += usec; 181 n++; 182 if (mx < usec) 183 mx = usec; 184 close(fd); 185 } 186 fprintf(stderr, 187 "files = %7ld, maxvnodes = %ld, ave=%.6f, max=%.6f, " 188 "elapsed %2ld\n", 189 tvnodes, maxvnodes, (double)tot / 1000000 / n, (double)mx / 190 1000000, time(NULL) - st); 191 share[idx] = mx; 192 193 for (i = 0; i < vnodes; i++) { 194 snprintf(file, sizeof(file), "f%09d", i); 195 if (unlink(file) == -1) 196 err(1, "unlink(%s)", file); 197 } 198 chdir(".."); 199 if (rmdir(dir) == -1) 200 err(1, "rmdir(%s)", dir); 201 202 _exit(0); 203} 204 205int 206main(int argc, char *argv[]) 207{ 208 size_t len; 209 pid_t *pids; 210 int e, i, status; 211 u_int mx; 212 213 if (argc != 5) { 214 fprintf(stderr, "Usage: %s <ncpu> <inodes> <maxvnodes> <mount point>\n", 215 argv[0]); 216 exit(1); 217 } 218 parallel = atol(argv[1]); 219 pids = calloc(parallel, sizeof(pid_t)); 220 tvnodes = atol(argv[2]); 221 vnodes = tvnodes / parallel; 222 maxvnodes = atol(argv[3]); 223 mp = argv[4]; 224 e = 0; 225 len = PAGE_SIZE; 226 if ((share = mmap(NULL, len, PROT_READ | PROT_WRITE, 227 MAP_ANON | MAP_SHARED, -1, 0)) == MAP_FAILED) 228 err(1, "mmap"); 229 230 for (i = 0; i < parallel; i++) { 231 if ((pids[i] = fork()) == 0) 232 test(i + 1); 233 } 234 235 for (i = 0; i < parallel; i++) { 236 if (waitpid(pids[i], &status, 0) == -1) 237 err(1, "waitpid(%d)", pids[i]); 238 e += status == 0 ? 0 : 1; 239 } 240 241 mx = 0; 242 for (i = 0; i < parallel; i++) { 243// fprintf(stderr, "share[%d] = %u\n", i + 1, share[i + 1]); 244 if (mx < share[i + 1]) 245 mx = share[i + 1]; 246 } 247 fprintf(stdout, "%lu\n", (unsigned long)mx); 248 249 return (e); 250} 251