1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0 3# 4# Test for cpuset v2 partition root state (PRS) 5# 6# The sched verbose flag can be optionally set so that the console log 7# can be examined for the correct setting of scheduling domain. 8# 9 10skip_test() { 11 echo "$1" 12 echo "Test SKIPPED" 13 exit 4 # ksft_skip 14} 15 16[[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!" 17 18 19# Get wait_inotify location 20WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify 21 22# Find cgroup v2 mount point 23CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}') 24[[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!" 25SUBPARTS_CPUS=$CGROUP2/.__DEBUG__.cpuset.cpus.subpartitions 26CPULIST=$(cat $CGROUP2/cpuset.cpus.effective) 27 28NR_CPUS=$(lscpu | grep "^CPU(s):" | sed -e "s/.*:[[:space:]]*//") 29[[ $NR_CPUS -lt 8 ]] && skip_test "Test needs at least 8 cpus available!" 30 31# Set verbose flag and delay factor 32PROG=$1 33VERBOSE=0 34DELAY_FACTOR=1 35SCHED_DEBUG= 36while [[ "$1" = -* ]] 37do 38 case "$1" in 39 -v) ((VERBOSE++)) 40 # Enable sched/verbose can slow thing down 41 [[ $DELAY_FACTOR -eq 1 ]] && 42 DELAY_FACTOR=2 43 ;; 44 -d) DELAY_FACTOR=$2 45 shift 46 ;; 47 *) echo "Usage: $PROG [-v] [-d <delay-factor>" 48 exit 49 ;; 50 esac 51 shift 52done 53 54# Set sched verbose flag if available when "-v" option is specified 55if [[ $VERBOSE -gt 0 && -d /sys/kernel/debug/sched ]] 56then 57 # Used to restore the original setting during cleanup 58 SCHED_DEBUG=$(cat /sys/kernel/debug/sched/verbose) 59 echo Y > /sys/kernel/debug/sched/verbose 60fi 61 62cd $CGROUP2 63echo +cpuset > cgroup.subtree_control 64 65# 66# If cpuset has been set up and used in child cgroups, we may not be able to 67# create partition under root cgroup because of the CPU exclusivity rule. 68# So we are going to skip the test if this is the case. 69# 70[[ -d test ]] || mkdir test 71echo 0-6 > test/cpuset.cpus 72echo root > test/cpuset.cpus.partition 73cat test/cpuset.cpus.partition | grep -q invalid 74RESULT=$? 75echo member > test/cpuset.cpus.partition 76echo "" > test/cpuset.cpus 77[[ $RESULT -eq 0 ]] && skip_test "Child cgroups are using cpuset!" 78 79cleanup() 80{ 81 online_cpus 82 cd $CGROUP2 83 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1 84 rmdir test > /dev/null 2>&1 85 [[ -n "$SCHED_DEBUG" ]] && 86 echo "$SCHED_DEBUG" > /sys/kernel/debug/sched/verbose 87} 88 89# Pause in ms 90pause() 91{ 92 DELAY=$1 93 LOOP=0 94 while [[ $LOOP -lt $DELAY_FACTOR ]] 95 do 96 sleep $DELAY 97 ((LOOP++)) 98 done 99 return 0 100} 101 102console_msg() 103{ 104 MSG=$1 105 echo "$MSG" 106 echo "" > /dev/console 107 echo "$MSG" > /dev/console 108 pause 0.01 109} 110 111test_partition() 112{ 113 EXPECTED_VAL=$1 114 echo $EXPECTED_VAL > cpuset.cpus.partition 115 [[ $? -eq 0 ]] || exit 1 116 ACTUAL_VAL=$(cat cpuset.cpus.partition) 117 [[ $ACTUAL_VAL != $EXPECTED_VAL ]] && { 118 echo "cpuset.cpus.partition: expect $EXPECTED_VAL, found $ACTUAL_VAL" 119 echo "Test FAILED" 120 exit 1 121 } 122} 123 124test_effective_cpus() 125{ 126 EXPECTED_VAL=$1 127 ACTUAL_VAL=$(cat cpuset.cpus.effective) 128 [[ "$ACTUAL_VAL" != "$EXPECTED_VAL" ]] && { 129 echo "cpuset.cpus.effective: expect '$EXPECTED_VAL', found '$ACTUAL_VAL'" 130 echo "Test FAILED" 131 exit 1 132 } 133} 134 135# Adding current process to cgroup.procs as a test 136test_add_proc() 137{ 138 OUTSTR="$1" 139 ERRMSG=$((echo $$ > cgroup.procs) |& cat) 140 echo $ERRMSG | grep -q "$OUTSTR" 141 [[ $? -ne 0 ]] && { 142 echo "cgroup.procs: expect '$OUTSTR', got '$ERRMSG'" 143 echo "Test FAILED" 144 exit 1 145 } 146 echo $$ > $CGROUP2/cgroup.procs # Move out the task 147} 148 149# 150# Testing the new "isolated" partition root type 151# 152test_isolated() 153{ 154 cd $CGROUP2/test 155 echo 2-3 > cpuset.cpus 156 TYPE=$(cat cpuset.cpus.partition) 157 [[ $TYPE = member ]] || echo member > cpuset.cpus.partition 158 159 console_msg "Change from member to root" 160 test_partition root 161 162 console_msg "Change from root to isolated" 163 test_partition isolated 164 165 console_msg "Change from isolated to member" 166 test_partition member 167 168 console_msg "Change from member to isolated" 169 test_partition isolated 170 171 console_msg "Change from isolated to root" 172 test_partition root 173 174 console_msg "Change from root to member" 175 test_partition member 176 177 # 178 # Testing partition root with no cpu 179 # 180 console_msg "Distribute all cpus to child partition" 181 echo +cpuset > cgroup.subtree_control 182 test_partition root 183 184 mkdir A1 185 cd A1 186 echo 2-3 > cpuset.cpus 187 test_partition root 188 test_effective_cpus 2-3 189 cd .. 190 test_effective_cpus "" 191 192 console_msg "Moving task to partition test" 193 test_add_proc "No space left" 194 cd A1 195 test_add_proc "" 196 cd .. 197 198 console_msg "Shrink and expand child partition" 199 cd A1 200 echo 2 > cpuset.cpus 201 cd .. 202 test_effective_cpus 3 203 cd A1 204 echo 2-3 > cpuset.cpus 205 cd .. 206 test_effective_cpus "" 207 208 # Cleaning up 209 console_msg "Cleaning up" 210 echo $$ > $CGROUP2/cgroup.procs 211 [[ -d A1 ]] && rmdir A1 212} 213 214# 215# Cpuset controller state transition test matrix. 216# 217# Cgroup test hierarchy 218# 219# root -- A1 -- A2 -- A3 220# +- B1 221# 222# P<v> = set cpus.partition (0:member, 1:root, 2:isolated) 223# C<l> = add cpu-list to cpuset.cpus 224# X<l> = add cpu-list to cpuset.cpus.exclusive 225# S<p> = use prefix in subtree_control 226# T = put a task into cgroup 227# O<c>=<v> = Write <v> to CPU online file of <c> 228# 229SETUP_A123_PARTITIONS="C1-3:P1:S+ C2-3:P1:S+ C3:P1" 230TEST_MATRIX=( 231 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 232 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 233 " C0-1 . . C2-3 S+ C4-5 . . 0 A2:0-1" 234 " C0-1 . . C2-3 P1 . . . 0 " 235 " C0-1 . . C2-3 P1:S+ C0-1:P1 . . 0 " 236 " C0-1 . . C2-3 P1:S+ C1:P1 . . 0 " 237 " C0-1:S+ . . C2-3 . . . P1 0 " 238 " C0-1:P1 . . C2-3 S+ C1 . . 0 " 239 " C0-1:P1 . . C2-3 S+ C1:P1 . . 0 " 240 " C0-1:P1 . . C2-3 S+ C1:P1 . P1 0 " 241 " C0-1:P1 . . C2-3 C4-5 . . . 0 A1:4-5" 242 " C0-1:P1 . . C2-3 S+:C4-5 . . . 0 A1:4-5" 243 " C0-1 . . C2-3:P1 . . . C2 0 " 244 " C0-1 . . C2-3:P1 . . . C4-5 0 B1:4-5" 245 "C0-3:P1:S+ C2-3:P1 . . . . . . 0 A1:0-1,A2:2-3" 246 "C0-3:P1:S+ C2-3:P1 . . C1-3 . . . 0 A1:1,A2:2-3" 247 "C2-3:P1:S+ C3:P1 . . C3 . . . 0 A1:,A2:3 A1:P1,A2:P1" 248 "C2-3:P1:S+ C3:P1 . . C3 P0 . . 0 A1:3,A2:3 A1:P1,A2:P0" 249 "C2-3:P1:S+ C2:P1 . . C2-4 . . . 0 A1:3-4,A2:2" 250 "C2-3:P1:S+ C3:P1 . . C3 . . C0-2 0 A1:,B1:0-2 A1:P1,A2:P1" 251 "$SETUP_A123_PARTITIONS . C2-3 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 252 253 # CPU offlining cases: 254 " C0-1 . . C2-3 S+ C4-5 . O2=0 0 A1:0-1,B1:3" 255 "C0-3:P1:S+ C2-3:P1 . . O2=0 . . . 0 A1:0-1,A2:3" 256 "C0-3:P1:S+ C2-3:P1 . . O2=0 O2=1 . . 0 A1:0-1,A2:2-3" 257 "C0-3:P1:S+ C2-3:P1 . . O1=0 . . . 0 A1:0,A2:2-3" 258 "C0-3:P1:S+ C2-3:P1 . . O1=0 O1=1 . . 0 A1:0-1,A2:2-3" 259 "C2-3:P1:S+ C3:P1 . . O3=0 O3=1 . . 0 A1:2,A2:3 A1:P1,A2:P1" 260 "C2-3:P1:S+ C3:P2 . . O3=0 O3=1 . . 0 A1:2,A2:3 A1:P1,A2:P2" 261 "C2-3:P1:S+ C3:P1 . . O2=0 O2=1 . . 0 A1:2,A2:3 A1:P1,A2:P1" 262 "C2-3:P1:S+ C3:P2 . . O2=0 O2=1 . . 0 A1:2,A2:3 A1:P1,A2:P2" 263 "C2-3:P1:S+ C3:P1 . . O2=0 . . . 0 A1:,A2:3 A1:P1,A2:P1" 264 "C2-3:P1:S+ C3:P1 . . O3=0 . . . 0 A1:2,A2: A1:P1,A2:P1" 265 "C2-3:P1:S+ C3:P1 . . T:O2=0 . . . 0 A1:3,A2:3 A1:P1,A2:P-1" 266 "C2-3:P1:S+ C3:P1 . . . T:O3=0 . . 0 A1:2,A2:2 A1:P1,A2:P-1" 267 "$SETUP_A123_PARTITIONS . O1=0 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 268 "$SETUP_A123_PARTITIONS . O2=0 . . . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1" 269 "$SETUP_A123_PARTITIONS . O3=0 . . . 0 A1:1,A2:2,A3: A1:P1,A2:P1,A3:P1" 270 "$SETUP_A123_PARTITIONS . T:O1=0 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" 271 "$SETUP_A123_PARTITIONS . . T:O2=0 . . 0 A1:1,A2:3,A3:3 A1:P1,A2:P1,A3:P-1" 272 "$SETUP_A123_PARTITIONS . . . T:O3=0 . 0 A1:1,A2:2,A3:2 A1:P1,A2:P1,A3:P-1" 273 "$SETUP_A123_PARTITIONS . T:O1=0 O1=1 . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 274 "$SETUP_A123_PARTITIONS . . T:O2=0 O2=1 . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 275 "$SETUP_A123_PARTITIONS . . . T:O3=0 O3=1 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 276 "$SETUP_A123_PARTITIONS . T:O1=0 O2=0 O1=1 . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1" 277 "$SETUP_A123_PARTITIONS . T:O1=0 O2=0 O2=1 . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" 278 279 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 280 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 281 # 282 # Remote partition and cpuset.cpus.exclusive tests 283 # 284 " C0-3:S+ C1-3:S+ C2-3 . X2-3 . . . 0 A1:0-3,A2:1-3,A3:2-3,XA1:2-3" 285 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3:P2 . . 0 A1:0-1,A2:2-3,A3:2-3 A1:P0,A2:P2 2-3" 286 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X3:P2 . . 0 A1:0-2,A2:3,A3:3 A1:P0,A2:P2 3" 287 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2 . 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3" 288 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:C3 . 0 A1:0-2,A2:1-2,A3:3 A1:P0,A3:P2 3" 289 " C0-3:S+ C1-3:S+ C2-3 C2-3 . . . P2 0 A1:0-3,A2:1-3,A3:2-3,B1:2-3 A1:P0,A3:P0,B1:P-2" 290 " C0-3:S+ C1-3:S+ C2-3 C4-5 . . . P2 0 B1:4-5 B1:P2 4-5" 291 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2 P2 0 A3:2-3,B1:4 A3:P2,B1:P2 2-4" 292 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2:C1-3 P2 0 A3:2-3,B1:4 A3:P2,B1:P2 2-4" 293 " C0-3:S+ C1-3:S+ C2-3 C4 X1-3 X1-3:P2 P2 . 0 A2:1,A3:2-3 A2:P2,A3:P2 1-3" 294 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2 P2:C4-5 0 A3:2-3,B1:4-5 A3:P2,B1:P2 2-5" 295 296 # Nested remote/local partition tests 297 " C0-3:S+ C1-3:S+ C2-3 C4-5 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:,A3:2-3,B1:4-5 \ 298 A1:P0,A2:P1,A3:P2,B1:P1 2-3" 299 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:,A3:2-3,B1:4 \ 300 A1:P0,A2:P1,A3:P2,B1:P1 2-4" 301 " C0-3:S+ C1-3:S+ C3 C4 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:2,A3:3,B1:4 \ 302 A1:P0,A2:P1,A3:P2,B1:P1 2-4" 303 " C0-4:S+ C1-4:S+ C2-4 . X2-4 X2-4:P2 X4:P1 . 0 A1:0-1,A2:2-3,A3:4 \ 304 A1:P0,A2:P2,A3:P1 2-4" 305 " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \ 306 . . X5 . . 0 A1:0-4,A2:1-4,A3:2-4 \ 307 A1:P0,A2:P-2,A3:P-1 ." 308 " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \ 309 . . . X1 . 0 A1:0-1,A2:2-4,A3:2-4 \ 310 A1:P0,A2:P2,A3:P-1 2-4" 311 312 # Remote partition offline tests 313 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:O2=0 . 0 A1:0-1,A2:1,A3:3 A1:P0,A3:P2 2-3" 314 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:O2=0 O2=1 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3" 315 " C0-3:S+ C1-3:S+ C3 . X2-3 X2-3 P2:O3=0 . 0 A1:0-2,A2:1-2,A3: A1:P0,A3:P2 3" 316 " C0-3:S+ C1-3:S+ C3 . X2-3 X2-3 T:P2:O3=0 . 0 A1:0-2,A2:1-2,A3:1-2 A1:P0,A3:P-2 3" 317 318 # An invalidated remote partition cannot self-recover from hotplug 319 " C0-3:S+ C1-3:S+ C2 . X2-3 X2-3 T:P2:O2=0 O2=1 0 A1:0-3,A2:1-3,A3:2 A1:P0,A3:P-2" 320 321 # cpus.exclusive.effective clearing test 322 " C0-3:S+ C1-3:S+ C2 . X2-3:X . . . 0 A1:0-3,A2:1-3,A3:2,XA1:" 323 324 # Invalid to valid remote partition transition test 325 " C0-3:S+ C1-3 . . . X3:P2 . . 0 A1:0-3,A2:1-3,XA2: A2:P-2" 326 " C0-3:S+ C1-3:X3:P2 327 . . X2-3 P2 . . 0 A1:0-2,A2:3,XA2:3 A2:P2 3" 328 329 # Invalid to valid local partition direct transition tests 330 " C1-3:S+:P2 C2-3:X1:P2 . . . . . . 0 A1:1-3,XA1:1-3,A2:2-3:XA2: A1:P2,A2:P-2 1-3" 331 " C1-3:S+:P2 C2-3:X1:P2 . . . X3:P2 . . 0 A1:1-2,XA1:1-3,A2:3:XA2:3 A1:P2,A2:P2 1-3" 332 " C0-3:P2 . . C4-6 C0-4 . . . 0 A1:0-4,B1:4-6 A1:P-2,B1:P0" 333 " C0-3:P2 . . C4-6 C0-4:C0-3 . . . 0 A1:0-3,B1:4-6 A1:P2,B1:P0 0-3" 334 " C0-3:P2 . . C3-5:C4-5 . . . . 0 A1:0-3,B1:4-5 A1:P2,B1:P0 0-3" 335 336 # Local partition invalidation tests 337 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \ 338 . . . . . 0 A1:1,A2:2,A3:3 A1:P2,A2:P2,A3:P2 1-3" 339 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \ 340 . . X4 . . 0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3" 341 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \ 342 . . C4 . . 0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3" 343 # Local partition CPU change tests 344 " C0-5:S+:P2 C4-5:S+:P1 . . . C3-5 . . 0 A1:0-2,A2:3-5 A1:P2,A2:P1 0-2" 345 " C0-5:S+:P2 C4-5:S+:P1 . . C1-5 . . . 0 A1:1-3,A2:4-5 A1:P2,A2:P1 1-3" 346 347 # cpus_allowed/exclusive_cpus update tests 348 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \ 349 . C4 . P2 . 0 A1:4,A2:4,XA2:,XA3:,A3:4 \ 350 A1:P0,A3:P-2 ." 351 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \ 352 . X1 . P2 . 0 A1:0-3,A2:1-3,XA1:1,XA2:,XA3:,A3:2-3 \ 353 A1:P0,A3:P-2 ." 354 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \ 355 . . C3 P2 . 0 A1:0-2,A2:0-2,XA2:3,XA3:3,A3:3 \ 356 A1:P0,A3:P2 3" 357 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \ 358 . . X3 P2 . 0 A1:0-2,A2:1-2,XA2:3,XA3:3,A3:3 \ 359 A1:P0,A3:P2 3" 360 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \ 361 . . X3 . . 0 A1:0-3,A2:1-3,XA2:3,XA3:3,A3:2-3 \ 362 A1:P0,A3:P-2 ." 363 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \ 364 . . C3 . . 0 A1:0-3,A2:3,XA2:3,XA3:3,A3:3 \ 365 A1:P0,A3:P-2 ." 366 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \ 367 . C4 . . . 0 A1:4,A2:4,A3:4,XA1:,XA2:,XA3 \ 368 A1:P0,A3:P-2 ." 369 370 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 371 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 372 # 373 # Incorrect change to cpuset.cpus invalidates partition root 374 # 375 # Adding CPUs to partition root that are not in parent's 376 # cpuset.cpus is allowed, but those extra CPUs are ignored. 377 "C2-3:P1:S+ C3:P1 . . . C2-4 . . 0 A1:,A2:2-3 A1:P1,A2:P1" 378 379 # Taking away all CPUs from parent or itself if there are tasks 380 # will make the partition invalid. 381 "C2-3:P1:S+ C3:P1 . . T C2-3 . . 0 A1:2-3,A2:2-3 A1:P1,A2:P-1" 382 " C3:P1:S+ C3 . . T P1 . . 0 A1:3,A2:3 A1:P1,A2:P-1" 383 "$SETUP_A123_PARTITIONS . T:C2-3 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" 384 "$SETUP_A123_PARTITIONS . T:C2-3:C1-3 . . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 385 386 # Changing a partition root to member makes child partitions invalid 387 "C2-3:P1:S+ C3:P1 . . P0 . . . 0 A1:2-3,A2:3 A1:P0,A2:P-1" 388 "$SETUP_A123_PARTITIONS . C2-3 P0 . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P0,A3:P-1" 389 390 # cpuset.cpus can contains cpus not in parent's cpuset.cpus as long 391 # as they overlap. 392 "C2-3:P1:S+ . . . . C3-4:P1 . . 0 A1:2,A2:3 A1:P1,A2:P1" 393 394 # Deletion of CPUs distributed to child cgroup is allowed. 395 "C0-1:P1:S+ C1 . C2-3 C4-5 . . . 0 A1:4-5,A2:4-5" 396 397 # To become a valid partition root, cpuset.cpus must overlap parent's 398 # cpuset.cpus. 399 " C0-1:P1 . . C2-3 S+ C4-5:P1 . . 0 A1:0-1,A2:0-1 A1:P1,A2:P-1" 400 401 # Enabling partition with child cpusets is allowed 402 " C0-1:S+ C1 . C2-3 P1 . . . 0 A1:0-1,A2:1 A1:P1" 403 404 # A partition root with non-partition root parent is invalid, but it 405 # can be made valid if its parent becomes a partition root too. 406 " C0-1:S+ C1 . C2-3 . P2 . . 0 A1:0-1,A2:1 A1:P0,A2:P-2" 407 " C0-1:S+ C1:P2 . C2-3 P1 . . . 0 A1:0,A2:1 A1:P1,A2:P2" 408 409 # A non-exclusive cpuset.cpus change will invalidate partition and its siblings 410 " C0-1:P1 . . C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P0" 411 " C0-1:P1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P-1" 412 " C0-1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P0,B1:P-1" 413 414 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 415 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 416 # Failure cases: 417 418 # A task cannot be added to a partition with no cpu 419 "C2-3:P1:S+ C3:P1 . . O2=0:T . . . 1 A1:,A2:3 A1:P1,A2:P1" 420 421 # Changes to cpuset.cpus.exclusive that violate exclusivity rule is rejected 422 " C0-3 . . C4-5 X0-3 . . X3-5 1 A1:0-3,B1:4-5" 423) 424 425# 426# Write to the cpu online file 427# $1 - <c>=<v> where <c> = cpu number, <v> value to be written 428# 429write_cpu_online() 430{ 431 CPU=${1%=*} 432 VAL=${1#*=} 433 CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online 434 if [[ $VAL -eq 0 ]] 435 then 436 OFFLINE_CPUS="$OFFLINE_CPUS $CPU" 437 else 438 [[ -n "$OFFLINE_CPUS" ]] && { 439 OFFLINE_CPUS=$(echo $CPU $CPU $OFFLINE_CPUS | fmt -1 |\ 440 sort | uniq -u) 441 } 442 fi 443 echo $VAL > $CPUFILE 444 pause 0.01 445} 446 447# 448# Set controller state 449# $1 - cgroup directory 450# $2 - state 451# $3 - showerr 452# 453# The presence of ":" in state means transition from one to the next. 454# 455set_ctrl_state() 456{ 457 TMPMSG=/tmp/.msg_$$ 458 CGRP=$1 459 STATE=$2 460 SHOWERR=${3} 461 CTRL=${CTRL:=$CONTROLLER} 462 HASERR=0 463 REDIRECT="2> $TMPMSG" 464 [[ -z "$STATE" || "$STATE" = '.' ]] && return 0 465 [[ $VERBOSE -gt 0 ]] && SHOWERR=1 466 467 rm -f $TMPMSG 468 for CMD in $(echo $STATE | sed -e "s/:/ /g") 469 do 470 TFILE=$CGRP/cgroup.procs 471 SFILE=$CGRP/cgroup.subtree_control 472 PFILE=$CGRP/cpuset.cpus.partition 473 CFILE=$CGRP/cpuset.cpus 474 XFILE=$CGRP/cpuset.cpus.exclusive 475 S=$(expr substr $CMD 1 1) 476 if [[ $S = S ]] 477 then 478 PREFIX=${CMD#?} 479 COMM="echo ${PREFIX}${CTRL} > $SFILE" 480 eval $COMM $REDIRECT 481 elif [[ $S = X ]] 482 then 483 CPUS=${CMD#?} 484 COMM="echo $CPUS > $XFILE" 485 eval $COMM $REDIRECT 486 elif [[ $S = C ]] 487 then 488 CPUS=${CMD#?} 489 COMM="echo $CPUS > $CFILE" 490 eval $COMM $REDIRECT 491 elif [[ $S = P ]] 492 then 493 VAL=${CMD#?} 494 case $VAL in 495 0) VAL=member 496 ;; 497 1) VAL=root 498 ;; 499 2) VAL=isolated 500 ;; 501 *) 502 echo "Invalid partition state - $VAL" 503 exit 1 504 ;; 505 esac 506 COMM="echo $VAL > $PFILE" 507 eval $COMM $REDIRECT 508 elif [[ $S = O ]] 509 then 510 VAL=${CMD#?} 511 write_cpu_online $VAL 512 elif [[ $S = T ]] 513 then 514 COMM="echo 0 > $TFILE" 515 eval $COMM $REDIRECT 516 fi 517 RET=$? 518 [[ $RET -ne 0 ]] && { 519 [[ -n "$SHOWERR" ]] && { 520 echo "$COMM" 521 cat $TMPMSG 522 } 523 HASERR=1 524 } 525 pause 0.01 526 rm -f $TMPMSG 527 done 528 return $HASERR 529} 530 531set_ctrl_state_noerr() 532{ 533 CGRP=$1 534 STATE=$2 535 [[ -d $CGRP ]] || mkdir $CGRP 536 set_ctrl_state $CGRP $STATE 1 537 [[ $? -ne 0 ]] && { 538 echo "ERROR: Failed to set $2 to cgroup $1!" 539 exit 1 540 } 541} 542 543online_cpus() 544{ 545 [[ -n "OFFLINE_CPUS" ]] && { 546 for C in $OFFLINE_CPUS 547 do 548 write_cpu_online ${C}=1 549 done 550 } 551} 552 553# 554# Return 1 if the list of effective cpus isn't the same as the initial list. 555# 556reset_cgroup_states() 557{ 558 echo 0 > $CGROUP2/cgroup.procs 559 online_cpus 560 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1 561 pause 0.02 562 set_ctrl_state . R- 563 pause 0.01 564} 565 566dump_states() 567{ 568 for DIR in . A1 A1/A2 A1/A2/A3 B1 569 do 570 CPUS=$DIR/cpuset.cpus 571 ECPUS=$DIR/cpuset.cpus.effective 572 XCPUS=$DIR/cpuset.cpus.exclusive 573 XECPUS=$DIR/cpuset.cpus.exclusive.effective 574 PRS=$DIR/cpuset.cpus.partition 575 PCPUS=$DIR/.__DEBUG__.cpuset.cpus.subpartitions 576 [[ -e $CPUS ]] && echo "$CPUS: $(cat $CPUS)" 577 [[ -e $XCPUS ]] && echo "$XCPUS: $(cat $XCPUS)" 578 [[ -e $ECPUS ]] && echo "$ECPUS: $(cat $ECPUS)" 579 [[ -e $XECPUS ]] && echo "$XECPUS: $(cat $XECPUS)" 580 [[ -e $PRS ]] && echo "$PRS: $(cat $PRS)" 581 [[ -e $PCPUS ]] && echo "$PCPUS: $(cat $PCPUS)" 582 done 583} 584 585# 586# Check effective cpus 587# $1 - check string, format: <cgroup>:<cpu-list>[,<cgroup>:<cpu-list>]* 588# 589check_effective_cpus() 590{ 591 CHK_STR=$1 592 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g") 593 do 594 set -- $(echo $CHK | sed -e "s/:/ /g") 595 CGRP=$1 596 CPUS=$2 597 if [[ $CGRP = X* ]] 598 then 599 CGRP=${CGRP#X} 600 FILE=cpuset.cpus.exclusive.effective 601 else 602 FILE=cpuset.cpus.effective 603 fi 604 [[ $CGRP = A2 ]] && CGRP=A1/A2 605 [[ $CGRP = A3 ]] && CGRP=A1/A2/A3 606 [[ -e $CGRP/$FILE ]] || return 1 607 [[ $CPUS = $(cat $CGRP/$FILE) ]] || return 1 608 done 609} 610 611# 612# Check cgroup states 613# $1 - check string, format: <cgroup>:<state>[,<cgroup>:<state>]* 614# 615check_cgroup_states() 616{ 617 CHK_STR=$1 618 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g") 619 do 620 set -- $(echo $CHK | sed -e "s/:/ /g") 621 CGRP=$1 622 STATE=$2 623 FILE= 624 EVAL=$(expr substr $STATE 2 2) 625 [[ $CGRP = A2 ]] && CGRP=A1/A2 626 [[ $CGRP = A3 ]] && CGRP=A1/A2/A3 627 628 case $STATE in 629 P*) FILE=$CGRP/cpuset.cpus.partition 630 ;; 631 *) echo "Unknown state: $STATE!" 632 exit 1 633 ;; 634 esac 635 VAL=$(cat $FILE) 636 637 case "$VAL" in 638 member) VAL=0 639 ;; 640 root) VAL=1 641 ;; 642 isolated) 643 VAL=2 644 ;; 645 "root invalid"*) 646 VAL=-1 647 ;; 648 "isolated invalid"*) 649 VAL=-2 650 ;; 651 esac 652 [[ $EVAL != $VAL ]] && return 1 653 done 654 return 0 655} 656 657# 658# Get isolated (including offline) CPUs by looking at 659# /sys/kernel/debug/sched/domains and compare that with the expected value. 660# 661# Note that a sched domain of just 1 CPU will be considered isolated. 662# 663# $1 - expected isolated cpu list 664# 665check_isolcpus() 666{ 667 EXPECT_VAL=$1 668 ISOLCPUS= 669 LASTISOLCPU= 670 SCHED_DOMAINS=/sys/kernel/debug/sched/domains 671 [[ -d $SCHED_DOMAINS ]] || return 0 672 [[ $EXPECT_VAL = . ]] && EXPECT_VAL= 673 674 for ((CPU=0; CPU < $NR_CPUS; CPU++)) 675 do 676 [[ -n "$(ls ${SCHED_DOMAINS}/cpu$CPU)" ]] && continue 677 678 if [[ -z "$LASTISOLCPU" ]] 679 then 680 ISOLCPUS=$CPU 681 LASTISOLCPU=$CPU 682 elif [[ "$LASTISOLCPU" -eq $((CPU - 1)) ]] 683 then 684 echo $ISOLCPUS | grep -q "\<$LASTISOLCPU\$" 685 if [[ $? -eq 0 ]] 686 then 687 ISOLCPUS=${ISOLCPUS}- 688 fi 689 LASTISOLCPU=$CPU 690 else 691 if [[ $ISOLCPUS = *- ]] 692 then 693 ISOLCPUS=${ISOLCPUS}$LASTISOLCPU 694 fi 695 ISOLCPUS=${ISOLCPUS},$CPU 696 LASTISOLCPU=$CPU 697 fi 698 done 699 [[ "$ISOLCPUS" = *- ]] && ISOLCPUS=${ISOLCPUS}$LASTISOLCPU 700 [[ "$EXPECT_VAL" = "$ISOLCPUS" ]] 701} 702 703test_fail() 704{ 705 TESTNUM=$1 706 TESTTYPE=$2 707 ADDINFO=$3 708 echo "Test $TEST[$TESTNUM] failed $TESTTYPE check!" 709 [[ -n "$ADDINFO" ]] && echo "*** $ADDINFO ***" 710 eval echo \${$TEST[$I]} 711 echo 712 dump_states 713 exit 1 714} 715 716# 717# Run cpuset state transition test 718# $1 - test matrix name 719# 720# This test is somewhat fragile as delays (sleep x) are added in various 721# places to make sure state changes are fully propagated before the next 722# action. These delays may need to be adjusted if running in a slower machine. 723# 724run_state_test() 725{ 726 TEST=$1 727 CONTROLLER=cpuset 728 I=0 729 eval CNT="\${#$TEST[@]}" 730 731 reset_cgroup_states 732 console_msg "Running state transition test ..." 733 734 while [[ $I -lt $CNT ]] 735 do 736 echo "Running test $I ..." > /dev/console 737 [[ $VERBOSE -gt 1 ]] && { 738 echo "" 739 eval echo \${$TEST[$I]} 740 } 741 eval set -- "\${$TEST[$I]}" 742 OLD_A1=$1 743 OLD_A2=$2 744 OLD_A3=$3 745 OLD_B1=$4 746 NEW_A1=$5 747 NEW_A2=$6 748 NEW_A3=$7 749 NEW_B1=$8 750 RESULT=$9 751 ECPUS=${10} 752 STATES=${11} 753 ICPUS=${12} 754 755 set_ctrl_state_noerr B1 $OLD_B1 756 set_ctrl_state_noerr A1 $OLD_A1 757 set_ctrl_state_noerr A1/A2 $OLD_A2 758 set_ctrl_state_noerr A1/A2/A3 $OLD_A3 759 RETVAL=0 760 set_ctrl_state A1 $NEW_A1; ((RETVAL += $?)) 761 set_ctrl_state A1/A2 $NEW_A2; ((RETVAL += $?)) 762 set_ctrl_state A1/A2/A3 $NEW_A3; ((RETVAL += $?)) 763 set_ctrl_state B1 $NEW_B1; ((RETVAL += $?)) 764 765 [[ $RETVAL -ne $RESULT ]] && test_fail $I result 766 767 [[ -n "$ECPUS" && "$ECPUS" != . ]] && { 768 check_effective_cpus $ECPUS 769 [[ $? -ne 0 ]] && test_fail $I "effective CPU" 770 } 771 772 [[ -n "$STATES" && "$STATES" != . ]] && { 773 check_cgroup_states $STATES 774 [[ $? -ne 0 ]] && test_fail $I states 775 } 776 777 # Compare the expected isolated CPUs with the actual ones, 778 # if available 779 [[ -n "$ICPUS" ]] && { 780 check_isolcpus $ICPUS 781 [[ $? -ne 0 ]] && test_fail $I "isolated CPU" \ 782 "Expect $ICPUS, get $ISOLCPUS instead" 783 } 784 reset_cgroup_states 785 # 786 # Check to see if effective cpu list changes 787 # 788 NEWLIST=$(cat cpuset.cpus.effective) 789 RETRY=0 790 while [[ $NEWLIST != $CPULIST && $RETRY -lt 5 ]] 791 do 792 # Wait a bit longer & recheck a few times 793 pause 0.01 794 ((RETRY++)) 795 NEWLIST=$(cat cpuset.cpus.effective) 796 done 797 [[ $NEWLIST != $CPULIST ]] && { 798 echo "Effective cpus changed to $NEWLIST after test $I!" 799 exit 1 800 } 801 [[ $VERBOSE -gt 0 ]] && echo "Test $I done." 802 ((I++)) 803 done 804 echo "All $I tests of $TEST PASSED." 805} 806 807# 808# Wait for inotify event for the given file and read it 809# $1: cgroup file to wait for 810# $2: file to store the read result 811# 812wait_inotify() 813{ 814 CGROUP_FILE=$1 815 OUTPUT_FILE=$2 816 817 $WAIT_INOTIFY $CGROUP_FILE 818 cat $CGROUP_FILE > $OUTPUT_FILE 819} 820 821# 822# Test if inotify events are properly generated when going into and out of 823# invalid partition state. 824# 825test_inotify() 826{ 827 ERR=0 828 PRS=/tmp/.prs_$$ 829 cd $CGROUP2/test 830 [[ -f $WAIT_INOTIFY ]] || { 831 echo "wait_inotify not found, inotify test SKIPPED." 832 return 833 } 834 835 pause 0.01 836 echo 1 > cpuset.cpus 837 echo 0 > cgroup.procs 838 echo root > cpuset.cpus.partition 839 pause 0.01 840 rm -f $PRS 841 wait_inotify $PWD/cpuset.cpus.partition $PRS & 842 pause 0.01 843 set_ctrl_state . "O1=0" 844 pause 0.01 845 check_cgroup_states ".:P-1" 846 if [[ $? -ne 0 ]] 847 then 848 echo "FAILED: Inotify test - partition not invalid" 849 ERR=1 850 elif [[ ! -f $PRS ]] 851 then 852 echo "FAILED: Inotify test - event not generated" 853 ERR=1 854 kill %1 855 elif [[ $(cat $PRS) != "root invalid"* ]] 856 then 857 echo "FAILED: Inotify test - incorrect state" 858 cat $PRS 859 ERR=1 860 fi 861 online_cpus 862 echo member > cpuset.cpus.partition 863 echo 0 > ../cgroup.procs 864 if [[ $ERR -ne 0 ]] 865 then 866 exit 1 867 else 868 echo "Inotify test PASSED" 869 fi 870} 871 872trap cleanup 0 2 3 6 873run_state_test TEST_MATRIX 874test_isolated 875test_inotify 876echo "All tests PASSED." 877