1#!/bin/sh 2 3# 4# Copyright (c) 2018 Dell EMC Isilon 5# All rights reserved. 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26# SUCH DAMAGE. 27# 28 29# Move files between two FSs using sendfile(2). 30# CG checksum errors reported. 31 32# Unrelated gpart page fault seen: 33# https://people.freebsd.org/~pho/stress/log/sendfile13.txt 34# Fixed by r329262 35 36. ../default.cfg 37[ `id -u` -ne 0 ] && echo "Must be root!" && exit 1 38 39dir=/tmp 40odir=`pwd` 41cd $dir 42sed '1,/^EOF/d' < $odir/$0 > $dir/sendfile13.c 43mycc -o sendfile13 -Wall -Wextra -O0 -g sendfile13.c || exit 1 44rm -f sendfile13.c 45cd $odir 46 47set -e 48size="$((`sysctl -n hw.usermem` / 2 / 1024 / 1024 / 1024))" 49size="$((size * 7 / 10))g" 50[ "$size" = "0g" ] && exit 0 51[ "$newfs_flags" = "-U" ] || exit 0 52newfs_flags="-j" 53 54mp1=$mntpoint 55mkdir -p $mp1 56md1=$mdstart 57mount | grep "on $mp1 " | grep -q /dev/md && umount -f $mp1 58[ -c /dev/md$md1 ] && mdconfig -d -u $md1 59mdconfig -a -t swap -s $size -u $md1 60bsdlabel -w md$md1 auto 61newfs $newfs_flags -n md${md1}$part > /dev/null 2>&1 62mount /dev/md${md1}$part $mp1 63 64md2=$((mdstart + 1)) 65mp2=${mntpoint}$md2 66mkdir -p $mp2 67mount | grep "on $mp2 " | grep -q /dev/md && umount -f $mp2 68[ -c /dev/md$md2 ] && mdconfig -d -u $md2 69mdconfig -a -t swap -s $size -u $md2 70bsdlabel -w md$md2 auto 71newfs $newfs_flags -n md${md2}$part > /dev/null 2>&1 72mount /dev/md${md2}$part $mp2 73set +e 74 75free=`df $mp1 | tail -1 | awk '{print $4}'` 76$dir/sendfile13 5432 $mp1 $mp2 $((free / 2)) & 77p1=$! 78$dir/sendfile13 5433 $mp2 $mp1 $((free / 2)) & 79p2=$! 80cd $odir 81s=0 82wait $p1; code=$? 83[ $code -ne 0 ] && { s=$code; echo "$p1 exit status $code"; } 84wait $p2; code=$? 85[ $code -ne 0 ] && { s=$code; echo "$p2 exit status $code"; } 86[ -f sendfile13.core -a $s -eq 0 ] && 87 { ls -l sendfile13.core; mv sendfile13.core /tmp; } 88cd $odir 89 90for i in `jot 6`; do 91 mount | grep -q "on $mp1 " || break 92 umount $mp1 && break || sleep 10 93 [ $i -eq 6 ] && 94 { echo FATAL; fstat -mf $mp1; exit 1; } 95done 96for i in `jot 6`; do 97 mount | grep -q "on $mp2 " || break 98 umount $mp2 && break || sleep 10 99 [ $i -eq 6 ] && 100 { echo FATAL; fstat -mf $mp2; exit 1; } 101done 102checkfs /dev/md${md1}$part || s=1 103checkfs /dev/md${md2}$part || s=1 104mdconfig -d -u $md1 || s=1 105mdconfig -d -u $md2 || s=1 106 107for i in `jot 6`; do 108 mount | grep -q "on $mp2 " || break 109 umount $mp2 && break || sleep 10 110 [ $i -eq 6 ] && 111 { echo FATAL; fstat -mf $mp2; exit 1; } 112done 113checkfs /dev/md${md2}$part || s=1 114 115rm -rf $dir/sendfile13 116exit $s 117 118EOF 119#include <sys/param.h> 120#include <sys/mman.h> 121#include <sys/socket.h> 122#include <sys/stat.h> 123#include <sys/wait.h> 124 125#include <netinet/in.h> 126 127#include <err.h> 128#include <fcntl.h> 129#include <netdb.h> 130#include <signal.h> 131#include <stdio.h> 132#include <stdlib.h> 133#include <string.h> 134#include <time.h> 135#include <unistd.h> 136 137#define BUFSIZE 8192 138#define MAXTHREADS 32 139 140static int files, port; 141static char *fromdir, *todir; 142 143static void 144create(char *path, size_t size) 145{ 146 size_t s; 147 int fd, i, ifd; 148 char *cp, file[128], help[128]; 149 150 i = 0; 151 while (size > 0) { 152 do { 153 s =arc4random() % size + 1; 154 } while (s > 1024 * 1024 * 1024); 155 size -= s; 156 sprintf(file, "%s/f%06d.%06d", path, getpid(), i++); 157 if ((ifd = open("/dev/zero", O_RDONLY)) == -1) 158 err(1, "open(/dev/zero)"); 159 if ((cp = mmap(0, s, PROT_READ, MAP_SHARED, ifd, 0)) == 160 (caddr_t) - 1) 161 err(1, "mmap error for input"); 162 if ((fd = open(file, O_WRONLY | O_CREAT, 0640)) == -1) 163 err(1, "create(%s)", file); 164 if (write(fd, cp, s) != (ssize_t)s) 165 err(1, "write(%s)", file); 166 munmap(cp, s); 167 close(fd); 168 close(ifd); 169 files++; 170 } 171 snprintf(help, sizeof(help), 172 "umount %s 2>&1 | grep -v 'Device busy'", path); 173 system(help); 174#if defined(DEBUG) 175 fprintf(stderr, "%d files created\n", files); 176#endif 177} 178 179static void 180server(void) 181{ 182 pid_t pid; 183 struct sigaction sa; 184 struct sockaddr_in inetaddr, inetpeer; 185 socklen_t len; 186 int tcpsock, msgsock; 187 int *buf, fd, idx, n, on, t __unused; 188 char ofile[128], nfile[128]; 189 190 sa.sa_handler = SIG_IGN; 191 sigemptyset(&sa.sa_mask); 192 sa.sa_flags = 0; 193 if (sigaction(SIGCHLD, &sa, 0) == -1) 194 err(1, "sigaction"); 195 196 on = 1; 197 if ((tcpsock = socket(AF_INET, SOCK_STREAM, 0)) < 0) 198 err(1, "socket(), %s:%d", __FILE__, __LINE__); 199 200 if (setsockopt(tcpsock, 201 SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on)) < 0) 202 err(1, "setsockopt(), %s:%d", __FILE__, __LINE__); 203 204 inetaddr.sin_family = AF_INET; 205 inetaddr.sin_addr.s_addr = INADDR_ANY; 206 inetaddr.sin_port = htons(port); 207 inetaddr.sin_len = sizeof(inetaddr); 208 209 if (bind(tcpsock, 210 (struct sockaddr *)&inetaddr, sizeof (inetaddr)) < 0) 211 err(1, "bind(), %s:%d", __FILE__, __LINE__); 212 213 if (listen(tcpsock, 5) < 0) 214 err(1, "listen(), %s:%d", __FILE__, __LINE__); 215 216 idx = 0; 217 len = sizeof(inetpeer); 218 for (;;) { 219 alarm(120); 220 if ((msgsock = accept(tcpsock, 221 (struct sockaddr *)&inetpeer, &len)) < 0) 222 err(1, "accept(), %s:%d", __FILE__, __LINE__); 223 224 if ((pid = fork()) == 0) { 225 alarm(120); 226 t = 0; 227 if ((buf = malloc(BUFSIZE)) == NULL) 228 err(1, "malloc(%d), %s:%d", BUFSIZE, 229 __FILE__, __LINE__); 230 231 sprintf(ofile, "%s/g%06d.%06d", todir, getpid(), 232 idx); 233 sprintf(nfile, "%s/n%06d.%06d", todir, getpid(), 234 idx); 235 if ((fd = open(ofile, O_RDWR | O_CREAT | O_TRUNC, 236 0640)) == -1) 237 err(1, "open(%s)", ofile); 238 239 for (;;) { 240 if ((n = read(msgsock, buf, BUFSIZE)) < 0) 241 err(1, "read(), %s:%d", __FILE__, 242 __LINE__); 243 t += n; 244 if (n == 0) break; 245 246 if ((write(fd, buf, n)) != n) 247 err(1, "write n=%d", n); 248 } 249 close(msgsock); 250 close(fd); 251 if (rename(ofile, nfile) != 0) 252 err(1, "rename(%s, %s)", ofile, nfile); 253 _exit(0); 254 } 255 close(msgsock); 256 if (++idx == files) 257 break; 258 } 259 260 _exit(0); 261} 262 263static void 264writer(char *inputFile) { 265 struct sockaddr_in inetaddr; 266 struct hostent *hostent; 267 struct stat statb; 268 off_t off = 0; 269 size_t size; 270 int i, fd, on, r, tcpsock; 271 272 alarm(120); 273 on = 1; 274 for (i = 1; i < 5; i++) { 275 if ((tcpsock = socket(AF_INET, SOCK_STREAM, 0)) < 0) 276 err(1, "socket(), %s:%d", __FILE__, __LINE__); 277 278 if (setsockopt(tcpsock, 279 SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on)) < 0) 280 err(1, "setsockopt(), %s:%d", __FILE__, __LINE__); 281 282 size = getpagesize(); 283 if (setsockopt(tcpsock, 284 SOL_SOCKET, SO_SNDBUF, (void *)&size, sizeof(size)) < 0) 285 err(1, "setsockopt(SO_SNDBUF), %s:%d", __FILE__, 286 __LINE__); 287 288 hostent = gethostbyname ("localhost"); 289 memcpy (&inetaddr.sin_addr.s_addr, hostent->h_addr, 290 sizeof (struct in_addr)); 291 292 inetaddr.sin_family = AF_INET; 293 inetaddr.sin_port = htons(port); 294 inetaddr.sin_len = sizeof(inetaddr); 295 296 r = connect(tcpsock, (struct sockaddr *) &inetaddr, 297 sizeof(inetaddr)); 298 if (r == 0) 299 break; 300 sleep(1); 301 close(tcpsock); 302 } 303 if (r < 0) 304 err(1, "connect(), %s:%d", __FILE__, __LINE__); 305 306 if (stat(inputFile, &statb) != 0) 307 err(1, "stat(%s)", inputFile); 308 309 if ((fd = open(inputFile, O_RDWR)) == -1) 310 err(1, "open(%s)", inputFile); 311 312 if (sendfile(fd, tcpsock, 0, statb.st_size, NULL, &off, 313 SF_NOCACHE) == -1) 314 warn("sendfile()"); 315 close(fd); 316 317 return; 318} 319 320static void 321move(int num) 322{ 323 char ifile[128]; 324 325 sprintf(ifile, "%s/f%06d.%06d", fromdir, getpid(), num); 326 writer(ifile); 327 328 if (unlink(ifile) != 0) 329 err(1, "unlink(%s)", ifile); 330} 331 332int 333main(int argc, char *argv[]) 334{ 335 pid_t spid; 336 size_t size; 337 int i, status; 338 339 if (argc != 5) { 340 fprintf(stderr, 341 "Usage %s <port> <from dir> <to dir> <size in k>", 342 argv[0]); 343 exit(1); 344 } 345 port = atoi(argv[1]); 346 fromdir = argv[2]; 347 if (chdir(fromdir) == -1) 348 err(1, "chdir(%s)", fromdir); 349 todir = argv[3]; 350 sscanf(argv[4], "%zd", &size); 351 size = size * 1024; 352 create(fromdir, size); 353 354 if ((spid = fork()) == 0) 355 server(); 356 357 for (i = 0; i < files; i++) { 358 move(i); 359 sleep(10); 360 } 361 if (waitpid(spid, &status, 0) != spid) 362 err(1, "waitpid"); 363 if (status != 0) { 364 if (WIFSIGNALED(status)) 365 fprintf(stderr, 366 "pid %d exit signal %d\n", 367 spid, WTERMSIG(status)); 368 } 369 370 return (status == 0 ? 0 : 1); 371} 372