1#!/bin/sh 2 3# 4# Copyright (c) 2024 Peter Holm <pho@FreeBSD.org> 5# 6# SPDX-License-Identifier: BSD-2-Clause 7# 8 9# Demonstrate issue described in: 10# [Bug 276002] nfscl: data corruption using both copy_file_range and mmap'd I/O 11 12# This version only uses mapped read/write, read(2)/write(2), fstat(2) and ftruncate(2) 13 14# Issue seen: 15# 20241003 10:04:24 all: mmap48.sh 16# 5257c5257 17# < 0244200 80 81 82 83 84 85 86 87 88 89 8a 8b 8c 8d 8e 8f 18# --- 19# > 0244200 80 81 82 83 84 85 86 87 88 ee 8a 8b 8c 8d 8e 8f 20# 256 -rw------- 1 root wheel 262144 Oct 3 10:05 file 21# 256 -rw------- 1 root wheel 262144 Oct 3 10:04 file.orig 22# FAIL mmap48.sh exit code 2 23 24[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 25. ../default.cfg 26set -u 27prog=$(basename "$0" .sh) 28log=/tmp/$prog.log 29serial=/tmp/$prog.serial 30grep -q $mntpoint /etc/exports || 31 { echo "$mntpoint missing from /etc/exports"; exit 0; } 32rpcinfo 2>/dev/null | grep -q mountd || exit 0 33 34cat > /tmp/$prog.c <<EOF 35#include <sys/mman.h> 36#include <sys/stat.h> 37 38#include <assert.h> 39#include <err.h> 40#include <fcntl.h> 41#include <pthread.h> 42#include <pthread_np.h> 43#include <stdio.h> 44#include <stdlib.h> 45#include <string.h> 46#include <unistd.h> 47 48static off_t siz; 49static pthread_mutex_t write_mutex; 50static int fd, go; 51static char *cp; 52 53#define THREADS 100 54 55static void * 56memread(void *arg __unused) 57{ 58 int i; 59 char c; 60 61 if (arc4random() % 100 < 10) 62 return (0); 63 pthread_set_name_np(pthread_self(), __func__); 64 while (go == -1) 65 usleep(50); 66 while (go == 1) { 67 i = arc4random() % siz; 68 c = cp[i]; 69 if (c != 0x77) /* No unused vars here */ 70 usleep(arc4random() % 400); 71 } 72 return (NULL); 73} 74 75static void * 76memwrite(void *arg __unused) 77{ 78 int i; 79 char c; 80 81 if (arc4random() % 100 < 10) 82 return (0); 83 pthread_set_name_np(pthread_self(), __func__); 84 while (go == -1) 85 usleep(50); 86 while (go == 1) { 87 i = arc4random() % siz; 88 pthread_mutex_lock(&write_mutex); 89 c = cp[i]; 90 cp[i] = 0xee; /* This value seems to linger with NFS */ 91 cp[i] = c; 92 pthread_mutex_unlock(&write_mutex); 93 usleep(arc4random() % 400); 94 } 95 return (NULL); 96} 97 98static void * 99wr(void *arg __unused) 100{ 101 off_t pos; 102 int r, s; 103 char buf[1024]; 104 105 if (arc4random() % 100 < 10) 106 return (0); 107 pthread_set_name_np(pthread_self(), __func__); 108 while (go == -1) 109 usleep(50); 110 while (go == 1) { 111 s = arc4random() % sizeof(buf) + 1; 112 pos = arc4random() % (siz - s); 113 pthread_mutex_lock(&write_mutex); 114 if (lseek(fd, pos, SEEK_SET) == -1) 115 err(1, "lseek(%d)", (int)pos); 116 if ((r = read(fd, buf, s)) != s) { 117 fprintf(stderr, "r = %d, s = %d, pos = %d\n", r, s, (int)pos); 118 err(1, "read():2"); 119 } 120 if (lseek(fd, pos, SEEK_SET) == -1) 121 err(1, "lseek(%d)", (int)pos); 122 if (write(fd, buf, s) != s) 123 err(1, "write()"); 124 pthread_mutex_unlock(&write_mutex); 125 usleep(arc4random() % 200); 126 } 127 return (NULL); 128} 129 130/* Both ftruncate() and fdatasync() triggers the problem */ 131 132static void * 133sy(void *arg __unused) 134{ 135 136 if (arc4random() % 100 < 10) 137 return (0); 138 pthread_set_name_np(pthread_self(), __func__); 139 while (go == -1) 140 usleep(50); 141 while (go == 1) { 142 if (fdatasync(fd) == -1) 143 err(1, "fdatasync()"); 144 usleep(arc4random() % 1000); 145 } 146 return (NULL); 147} 148 149static void * 150tr(void *arg __unused) 151{ 152 153 if (arc4random() % 100 < 10) 154 return (0); 155 pthread_set_name_np(pthread_self(), __func__); 156 while (go == -1) 157 usleep(50); 158 while (go == 1) { 159 if (ftruncate(fd, siz) == -1) /* No size change */ 160 err(1, "truncate)"); 161 usleep(arc4random() % 1000); 162 } 163 return (NULL); 164} 165 166static void * 167fs(void *arg __unused) 168{ 169 struct stat st; 170 171 if (arc4random() % 100 < 10) 172 return (0); 173 pthread_set_name_np(pthread_self(), __func__); 174 while (go == -1) 175 usleep(50); 176 while (go == 1) { 177 if (fstat(fd, &st) == -1) 178 err(1, "stat()"); 179 usleep(arc4random() % 1000); 180 } 181 return (NULL); 182} 183 184int 185main(int argc, char *argv[]) 186{ 187 struct stat st; 188 pthread_t tp[THREADS]; 189 int e, i, idx; 190 191 if (argc != 2) { 192 fprintf(stderr, "Usage: %s <file>\n", argv[0]); 193 exit(1); 194 } 195 if ((fd = open(argv[1], O_RDWR)) == -1) 196 err(1, "open(%s)", argv[1]); 197 if (fstat(fd, &st) == -1) 198 err(1, "stat(%s)", argv[1]); 199 siz = st.st_size; 200 cp = mmap(NULL, st.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 201 if (cp == MAP_FAILED) 202 err(1, "mmap()"); 203 204 go = -1; 205 pthread_mutex_init(&write_mutex, NULL); 206 idx = 0; 207 for (i = 0; i < (int)(arc4random() % 3 + 1); i++) { 208 if ((e = pthread_create(&tp[idx++], NULL, memread, NULL)) != 0) 209 errc(1, e, "pthread_create"); 210 } 211 for (i = 0; i < (int)(arc4random() % 3 + 1); i++) { 212 if ((e = pthread_create(&tp[idx++], NULL, memwrite, NULL)) != 0) 213 errc(1, e, "pthread_create"); 214 } 215 for (i = 0; i < (int)(arc4random() % 3 + 1); i++) { 216 if ((e = pthread_create(&tp[idx++], NULL, wr, NULL)) != 0) 217 errc(1, e, "pthread_create"); 218 } 219 for (i = 0; i < (int)(arc4random() % 3 + 1); i++) { 220 if ((e = pthread_create(&tp[idx++], NULL, fs, NULL)) != 0) 221 errc(1, e, "pthread_create"); 222 } 223 for (i = 0; i < (int)(arc4random() % 3 + 1); i++) { 224 if ((e = pthread_create(&tp[idx++], NULL, tr, NULL)) != 0) 225 errc(1, e, "pthread_create"); 226 } 227 for (i = 0; i < (int)(arc4random() % 3 + 1); i++) { 228 if ((e = pthread_create(&tp[idx++], NULL, sy, NULL)) != 0) 229 errc(1, e, "pthread_create"); 230 } 231 assert(idx <= THREADS); 232 233 sleep(1); 234 go = 1; 235 sleep(60); 236 go = 0; 237 for (i = 0; i < idx; i++) 238 pthread_join(tp[i], NULL); 239 if (munmap(cp, siz) == -1) 240 err(1, "munmap()"); 241 close(fd); 242} 243EOF 244mycc -o /tmp/$prog -Wall -Wextra -O0 /tmp/$prog.c -lpthread || exit 1 245 246mycc -o $serial -Wall -Wextra -O2 ../tools/serial.c || exit 1 247mount | grep -q "on $mntpoint " && umount -f $mntpoint 248mdconfig -l | grep -q md$mdstart && mdconfig -d -u $mdstart 249mdconfig -s 5g -u $mdstart 250newfs -n $newfs_flags /dev/md$mdstart > /dev/null 251mount /dev/md$mdstart $mntpoint 252 253mp2=${mntpoint}2 254mkdir -p $mp2 255mount | grep -q "on $mp2 " && umount -f $mp2 256mount -t nfs -o retrycnt=3 127.0.0.1:$mntpoint $mp2 || exit 1 257sleep .2 258 259here=`pwd` 260cd $mp2 261#$here/../testcases/swap/swap -t 5m -i 20 > /dev/null & 262sleep 2 263 264size=262144 265$serial file $size 266cp file file.orig 267 268s=0 269#ktrace -id -f $here/ktrace.out /tmp/$prog file || s=1 270/tmp/$prog file || s=1 271 272while pgrep -q swap; do pkill swap; done 273wait 274if ! cmp -s file.orig file; then 275 od -t x1 file.orig > /var/tmp/$prog.file1 276 od -t x1 file > /var/tmp/$prog.file2 277 diff /var/tmp/$prog.file1 /var/tmp/$prog.file2 > $log 278 head -20 $log 279 rm /var/tmp/$prog.file1 /var/tmp/$prog.file2 280 ls -ls file.orig file 281 s=2 282fi 283 284cd $here 285umount $mp2 286umount $mntpoint 287mdconfig -d -u $mdstart 288rm -f $serial /tmp/$prog /tmp/$prog.c $log 289exit $s 290