1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0 3# 4# Test for cpuset v2 partition root state (PRS) 5# 6# The sched verbose flag can be optionally set so that the console log 7# can be examined for the correct setting of scheduling domain. 8# 9 10skip_test() { 11 echo "$1" 12 echo "Test SKIPPED" 13 exit 4 # ksft_skip 14} 15 16[[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!" 17 18 19# Get wait_inotify location 20WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify 21 22# Find cgroup v2 mount point 23CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}') 24[[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!" 25SUBPARTS_CPUS=$CGROUP2/.__DEBUG__.cpuset.cpus.subpartitions 26CPULIST=$(cat $CGROUP2/cpuset.cpus.effective) 27 28NR_CPUS=$(lscpu | grep "^CPU(s):" | sed -e "s/.*:[[:space:]]*//") 29[[ $NR_CPUS -lt 8 ]] && skip_test "Test needs at least 8 cpus available!" 30 31# Check to see if /dev/console exists and is writable 32if [[ -c /dev/console && -w /dev/console ]] 33then 34 CONSOLE=/dev/console 35else 36 CONSOLE=/dev/null 37fi 38 39# Set verbose flag and delay factor 40PROG=$1 41VERBOSE=0 42DELAY_FACTOR=1 43SCHED_DEBUG= 44while [[ "$1" = -* ]] 45do 46 case "$1" in 47 -v) ((VERBOSE++)) 48 # Enable sched/verbose can slow thing down 49 [[ $DELAY_FACTOR -eq 1 ]] && 50 DELAY_FACTOR=2 51 ;; 52 -d) DELAY_FACTOR=$2 53 shift 54 ;; 55 *) echo "Usage: $PROG [-v] [-d <delay-factor>" 56 exit 57 ;; 58 esac 59 shift 60done 61 62# Set sched verbose flag if available when "-v" option is specified 63if [[ $VERBOSE -gt 0 && -d /sys/kernel/debug/sched ]] 64then 65 # Used to restore the original setting during cleanup 66 SCHED_DEBUG=$(cat /sys/kernel/debug/sched/verbose) 67 echo Y > /sys/kernel/debug/sched/verbose 68fi 69 70cd $CGROUP2 71echo +cpuset > cgroup.subtree_control 72 73# 74# If cpuset has been set up and used in child cgroups, we may not be able to 75# create partition under root cgroup because of the CPU exclusivity rule. 76# So we are going to skip the test if this is the case. 77# 78[[ -d test ]] || mkdir test 79echo 0-6 > test/cpuset.cpus 80echo root > test/cpuset.cpus.partition 81cat test/cpuset.cpus.partition | grep -q invalid 82RESULT=$? 83echo member > test/cpuset.cpus.partition 84echo "" > test/cpuset.cpus 85[[ $RESULT -eq 0 ]] && skip_test "Child cgroups are using cpuset!" 86 87cleanup() 88{ 89 online_cpus 90 cd $CGROUP2 91 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1 92 rmdir test > /dev/null 2>&1 93 [[ -n "$SCHED_DEBUG" ]] && 94 echo "$SCHED_DEBUG" > /sys/kernel/debug/sched/verbose 95} 96 97# Pause in ms 98pause() 99{ 100 DELAY=$1 101 LOOP=0 102 while [[ $LOOP -lt $DELAY_FACTOR ]] 103 do 104 sleep $DELAY 105 ((LOOP++)) 106 done 107 return 0 108} 109 110console_msg() 111{ 112 MSG=$1 113 echo "$MSG" 114 echo "" > $CONSOLE 115 echo "$MSG" > $CONSOLE 116 pause 0.01 117} 118 119test_partition() 120{ 121 EXPECTED_VAL=$1 122 echo $EXPECTED_VAL > cpuset.cpus.partition 123 [[ $? -eq 0 ]] || exit 1 124 ACTUAL_VAL=$(cat cpuset.cpus.partition) 125 [[ $ACTUAL_VAL != $EXPECTED_VAL ]] && { 126 echo "cpuset.cpus.partition: expect $EXPECTED_VAL, found $ACTUAL_VAL" 127 echo "Test FAILED" 128 exit 1 129 } 130} 131 132test_effective_cpus() 133{ 134 EXPECTED_VAL=$1 135 ACTUAL_VAL=$(cat cpuset.cpus.effective) 136 [[ "$ACTUAL_VAL" != "$EXPECTED_VAL" ]] && { 137 echo "cpuset.cpus.effective: expect '$EXPECTED_VAL', found '$ACTUAL_VAL'" 138 echo "Test FAILED" 139 exit 1 140 } 141} 142 143# Adding current process to cgroup.procs as a test 144test_add_proc() 145{ 146 OUTSTR="$1" 147 ERRMSG=$((echo $$ > cgroup.procs) |& cat) 148 echo $ERRMSG | grep -q "$OUTSTR" 149 [[ $? -ne 0 ]] && { 150 echo "cgroup.procs: expect '$OUTSTR', got '$ERRMSG'" 151 echo "Test FAILED" 152 exit 1 153 } 154 echo $$ > $CGROUP2/cgroup.procs # Move out the task 155} 156 157# 158# Cpuset controller state transition test matrix. 159# 160# Cgroup test hierarchy 161# 162# root -- A1 -- A2 -- A3 163# +- B1 164# 165# P<v> = set cpus.partition (0:member, 1:root, 2:isolated) 166# C<l> = add cpu-list to cpuset.cpus 167# X<l> = add cpu-list to cpuset.cpus.exclusive 168# S<p> = use prefix in subtree_control 169# T = put a task into cgroup 170# O<c>=<v> = Write <v> to CPU online file of <c> 171# 172# ECPUs - effective CPUs of cpusets 173# Pstate - partition root state 174# ISOLCPUS - isolated CPUs (<icpus>[,<icpus2>]) 175# 176# Note that if there are 2 fields in ISOLCPUS, the first one is for 177# sched-debug matching which includes offline CPUs and single-CPU partitions 178# while the second one is for matching cpuset.cpus.isolated. 179# 180SETUP_A123_PARTITIONS="C1-3:P1:S+ C2-3:P1:S+ C3:P1" 181TEST_MATRIX=( 182 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 183 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 184 " C0-1 . . C2-3 S+ C4-5 . . 0 A2:0-1" 185 " C0-1 . . C2-3 P1 . . . 0 " 186 " C0-1 . . C2-3 P1:S+ C0-1:P1 . . 0 " 187 " C0-1 . . C2-3 P1:S+ C1:P1 . . 0 " 188 " C0-1:S+ . . C2-3 . . . P1 0 " 189 " C0-1:P1 . . C2-3 S+ C1 . . 0 " 190 " C0-1:P1 . . C2-3 S+ C1:P1 . . 0 " 191 " C0-1:P1 . . C2-3 S+ C1:P1 . P1 0 " 192 " C0-1:P1 . . C2-3 C4-5 . . . 0 A1:4-5" 193 " C0-1:P1 . . C2-3 S+:C4-5 . . . 0 A1:4-5" 194 " C0-1 . . C2-3:P1 . . . C2 0 " 195 " C0-1 . . C2-3:P1 . . . C4-5 0 B1:4-5" 196 "C0-3:P1:S+ C2-3:P1 . . . . . . 0 A1:0-1,A2:2-3" 197 "C0-3:P1:S+ C2-3:P1 . . C1-3 . . . 0 A1:1,A2:2-3" 198 "C2-3:P1:S+ C3:P1 . . C3 . . . 0 A1:,A2:3 A1:P1,A2:P1" 199 "C2-3:P1:S+ C3:P1 . . C3 P0 . . 0 A1:3,A2:3 A1:P1,A2:P0" 200 "C2-3:P1:S+ C2:P1 . . C2-4 . . . 0 A1:3-4,A2:2" 201 "C2-3:P1:S+ C3:P1 . . C3 . . C0-2 0 A1:,B1:0-2 A1:P1,A2:P1" 202 "$SETUP_A123_PARTITIONS . C2-3 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 203 204 # CPU offlining cases: 205 " C0-1 . . C2-3 S+ C4-5 . O2=0 0 A1:0-1,B1:3" 206 "C0-3:P1:S+ C2-3:P1 . . O2=0 . . . 0 A1:0-1,A2:3" 207 "C0-3:P1:S+ C2-3:P1 . . O2=0 O2=1 . . 0 A1:0-1,A2:2-3" 208 "C0-3:P1:S+ C2-3:P1 . . O1=0 . . . 0 A1:0,A2:2-3" 209 "C0-3:P1:S+ C2-3:P1 . . O1=0 O1=1 . . 0 A1:0-1,A2:2-3" 210 "C2-3:P1:S+ C3:P1 . . O3=0 O3=1 . . 0 A1:2,A2:3 A1:P1,A2:P1" 211 "C2-3:P1:S+ C3:P2 . . O3=0 O3=1 . . 0 A1:2,A2:3 A1:P1,A2:P2" 212 "C2-3:P1:S+ C3:P1 . . O2=0 O2=1 . . 0 A1:2,A2:3 A1:P1,A2:P1" 213 "C2-3:P1:S+ C3:P2 . . O2=0 O2=1 . . 0 A1:2,A2:3 A1:P1,A2:P2" 214 "C2-3:P1:S+ C3:P1 . . O2=0 . . . 0 A1:,A2:3 A1:P1,A2:P1" 215 "C2-3:P1:S+ C3:P1 . . O3=0 . . . 0 A1:2,A2: A1:P1,A2:P1" 216 "C2-3:P1:S+ C3:P1 . . T:O2=0 . . . 0 A1:3,A2:3 A1:P1,A2:P-1" 217 "C2-3:P1:S+ C3:P1 . . . T:O3=0 . . 0 A1:2,A2:2 A1:P1,A2:P-1" 218 "$SETUP_A123_PARTITIONS . O1=0 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 219 "$SETUP_A123_PARTITIONS . O2=0 . . . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1" 220 "$SETUP_A123_PARTITIONS . O3=0 . . . 0 A1:1,A2:2,A3: A1:P1,A2:P1,A3:P1" 221 "$SETUP_A123_PARTITIONS . T:O1=0 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" 222 "$SETUP_A123_PARTITIONS . . T:O2=0 . . 0 A1:1,A2:3,A3:3 A1:P1,A2:P1,A3:P-1" 223 "$SETUP_A123_PARTITIONS . . . T:O3=0 . 0 A1:1,A2:2,A3:2 A1:P1,A2:P1,A3:P-1" 224 "$SETUP_A123_PARTITIONS . T:O1=0 O1=1 . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 225 "$SETUP_A123_PARTITIONS . . T:O2=0 O2=1 . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 226 "$SETUP_A123_PARTITIONS . . . T:O3=0 O3=1 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 227 "$SETUP_A123_PARTITIONS . T:O1=0 O2=0 O1=1 . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1" 228 "$SETUP_A123_PARTITIONS . T:O1=0 O2=0 O2=1 . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" 229 230 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 231 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 232 # 233 # Remote partition and cpuset.cpus.exclusive tests 234 # 235 " C0-3:S+ C1-3:S+ C2-3 . X2-3 . . . 0 A1:0-3,A2:1-3,A3:2-3,XA1:2-3" 236 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3:P2 . . 0 A1:0-1,A2:2-3,A3:2-3 A1:P0,A2:P2 2-3" 237 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X3:P2 . . 0 A1:0-2,A2:3,A3:3 A1:P0,A2:P2 3" 238 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2 . 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3" 239 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:C3 . 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3" 240 " C0-3:S+ C1-3:S+ C2-3 C2-3 . . . P2 0 A1:0-3,A2:1-3,A3:2-3,B1:2-3 A1:P0,A3:P0,B1:P-2" 241 " C0-3:S+ C1-3:S+ C2-3 C4-5 . . . P2 0 B1:4-5 B1:P2 4-5" 242 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2 P2 0 A3:2-3,B1:4 A3:P2,B1:P2 2-4" 243 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2:C1-3 P2 0 A3:2-3,B1:4 A3:P2,B1:P2 2-4" 244 " C0-3:S+ C1-3:S+ C2-3 C4 X1-3 X1-3:P2 P2 . 0 A2:1,A3:2-3 A2:P2,A3:P2 1-3" 245 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2 P2:C4-5 0 A3:2-3,B1:4-5 A3:P2,B1:P2 2-5" 246 " C4:X0-3:S+ X1-3:S+ X2-3 . . P2 . . 0 A1:4,A2:1-3,A3:1-3 A2:P2 1-3" 247 " C4:X0-3:S+ X1-3:S+ X2-3 . . . P2 . 0 A1:4,A2:4,A3:2-3 A3:P2 2-3" 248 249 # Nested remote/local partition tests 250 " C0-3:S+ C1-3:S+ C2-3 C4-5 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:,A3:2-3,B1:4-5 \ 251 A1:P0,A2:P1,A3:P2,B1:P1 2-3" 252 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:,A3:2-3,B1:4 \ 253 A1:P0,A2:P1,A3:P2,B1:P1 2-4,2-3" 254 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3:P1 . P1 0 A1:0-1,A2:2-3,A3:2-3,B1:4 \ 255 A1:P0,A2:P1,A3:P0,B1:P1" 256 " C0-3:S+ C1-3:S+ C3 C4 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:2,A3:3,B1:4 \ 257 A1:P0,A2:P1,A3:P2,B1:P1 2-4,3" 258 " C0-4:S+ C1-4:S+ C2-4 . X2-4 X2-4:P2 X4:P1 . 0 A1:0-1,A2:2-3,A3:4 \ 259 A1:P0,A2:P2,A3:P1 2-4,2-3" 260 " C0-4:S+ C1-4:S+ C2-4 . X2-4 X2-4:P2 X3-4:P1 . 0 A1:0-1,A2:2,A3:3-4 \ 261 A1:P0,A2:P2,A3:P1 2" 262 " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \ 263 . . X5 . . 0 A1:0-4,A2:1-4,A3:2-4 \ 264 A1:P0,A2:P-2,A3:P-1" 265 " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \ 266 . . . X1 . 0 A1:0-1,A2:2-4,A3:2-4 \ 267 A1:P0,A2:P2,A3:P-1 2-4" 268 269 # Remote partition offline tests 270 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:O2=0 . 0 A1:0-1,A2:1,A3:3 A1:P0,A3:P2 2-3" 271 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:O2=0 O2=1 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3" 272 " C0-3:S+ C1-3:S+ C3 . X2-3 X2-3 P2:O3=0 . 0 A1:0-2,A2:1-2,A3: A1:P0,A3:P2 3" 273 " C0-3:S+ C1-3:S+ C3 . X2-3 X2-3 T:P2:O3=0 . 0 A1:0-2,A2:1-2,A3:1-2 A1:P0,A3:P-2 3," 274 275 # An invalidated remote partition cannot self-recover from hotplug 276 " C0-3:S+ C1-3:S+ C2 . X2-3 X2-3 T:P2:O2=0 O2=1 0 A1:0-3,A2:1-3,A3:2 A1:P0,A3:P-2" 277 278 # cpus.exclusive.effective clearing test 279 " C0-3:S+ C1-3:S+ C2 . X2-3:X . . . 0 A1:0-3,A2:1-3,A3:2,XA1:" 280 281 # Invalid to valid remote partition transition test 282 " C0-3:S+ C1-3 . . . X3:P2 . . 0 A1:0-3,A2:1-3,XA2: A2:P-2" 283 " C0-3:S+ C1-3:X3:P2 284 . . X2-3 P2 . . 0 A1:0-2,A2:3,XA2:3 A2:P2 3" 285 286 # Invalid to valid local partition direct transition tests 287 " C1-3:S+:P2 X4:P2 . . . . . . 0 A1:1-3,XA1:1-3,A2:1-3:XA2: A1:P2,A2:P-2 1-3" 288 " C1-3:S+:P2 X4:P2 . . . X3:P2 . . 0 A1:1-2,XA1:1-3,A2:3:XA2:3 A1:P2,A2:P2 1-3" 289 " C0-3:P2 . . C4-6 C0-4 . . . 0 A1:0-4,B1:4-6 A1:P-2,B1:P0" 290 " C0-3:P2 . . C4-6 C0-4:C0-3 . . . 0 A1:0-3,B1:4-6 A1:P2,B1:P0 0-3" 291 " C0-3:P2 . . C3-5:C4-5 . . . . 0 A1:0-3,B1:4-5 A1:P2,B1:P0 0-3" 292 293 # Local partition invalidation tests 294 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \ 295 . . . . . 0 A1:1,A2:2,A3:3 A1:P2,A2:P2,A3:P2 1-3" 296 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \ 297 . . X4 . . 0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3" 298 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \ 299 . . C4:X . . 0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3" 300 # Local partition CPU change tests 301 " C0-5:S+:P2 C4-5:S+:P1 . . . C3-5 . . 0 A1:0-2,A2:3-5 A1:P2,A2:P1 0-2" 302 " C0-5:S+:P2 C4-5:S+:P1 . . C1-5 . . . 0 A1:1-3,A2:4-5 A1:P2,A2:P1 1-3" 303 304 # cpus_allowed/exclusive_cpus update tests 305 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \ 306 . X:C4 . P2 . 0 A1:4,A2:4,XA2:,XA3:,A3:4 \ 307 A1:P0,A3:P-2" 308 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \ 309 . X1 . P2 . 0 A1:0-3,A2:1-3,XA1:1,XA2:,XA3:,A3:2-3 \ 310 A1:P0,A3:P-2" 311 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \ 312 . . X3 P2 . 0 A1:0-2,A2:1-2,XA2:3,XA3:3,A3:3 \ 313 A1:P0,A3:P2 3" 314 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \ 315 . . X3 . . 0 A1:0-3,A2:1-3,XA2:3,XA3:3,A3:2-3 \ 316 A1:P0,A3:P-2" 317 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \ 318 . X4 . . . 0 A1:0-3,A2:1-3,A3:2-3,XA1:4,XA2:,XA3 \ 319 A1:P0,A3:P-2" 320 321 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 322 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 323 # 324 # Incorrect change to cpuset.cpus invalidates partition root 325 # 326 # Adding CPUs to partition root that are not in parent's 327 # cpuset.cpus is allowed, but those extra CPUs are ignored. 328 "C2-3:P1:S+ C3:P1 . . . C2-4 . . 0 A1:,A2:2-3 A1:P1,A2:P1" 329 330 # Taking away all CPUs from parent or itself if there are tasks 331 # will make the partition invalid. 332 "C2-3:P1:S+ C3:P1 . . T C2-3 . . 0 A1:2-3,A2:2-3 A1:P1,A2:P-1" 333 " C3:P1:S+ C3 . . T P1 . . 0 A1:3,A2:3 A1:P1,A2:P-1" 334 "$SETUP_A123_PARTITIONS . T:C2-3 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" 335 "$SETUP_A123_PARTITIONS . T:C2-3:C1-3 . . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 336 337 # Changing a partition root to member makes child partitions invalid 338 "C2-3:P1:S+ C3:P1 . . P0 . . . 0 A1:2-3,A2:3 A1:P0,A2:P-1" 339 "$SETUP_A123_PARTITIONS . C2-3 P0 . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P0,A3:P-1" 340 341 # cpuset.cpus can contains cpus not in parent's cpuset.cpus as long 342 # as they overlap. 343 "C2-3:P1:S+ . . . . C3-4:P1 . . 0 A1:2,A2:3 A1:P1,A2:P1" 344 345 # Deletion of CPUs distributed to child cgroup is allowed. 346 "C0-1:P1:S+ C1 . C2-3 C4-5 . . . 0 A1:4-5,A2:4-5" 347 348 # To become a valid partition root, cpuset.cpus must overlap parent's 349 # cpuset.cpus. 350 " C0-1:P1 . . C2-3 S+ C4-5:P1 . . 0 A1:0-1,A2:0-1 A1:P1,A2:P-1" 351 352 # Enabling partition with child cpusets is allowed 353 " C0-1:S+ C1 . C2-3 P1 . . . 0 A1:0-1,A2:1 A1:P1" 354 355 # A partition root with non-partition root parent is invalid, but it 356 # can be made valid if its parent becomes a partition root too. 357 " C0-1:S+ C1 . C2-3 . P2 . . 0 A1:0-1,A2:1 A1:P0,A2:P-2" 358 " C0-1:S+ C1:P2 . C2-3 P1 . . . 0 A1:0,A2:1 A1:P1,A2:P2" 359 360 # A non-exclusive cpuset.cpus change will invalidate partition and its siblings 361 " C0-1:P1 . . C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P0" 362 " C0-1:P1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P-1" 363 " C0-1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P0,B1:P-1" 364 365 # cpuset.cpus can overlap with sibling cpuset.cpus.exclusive but not subsumed by it 366 " C0-3 . . C4-5 X5 . . . 0 A1:0-3,B1:4-5" 367 368 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 369 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 370 # Failure cases: 371 372 # A task cannot be added to a partition with no cpu 373 "C2-3:P1:S+ C3:P1 . . O2=0:T . . . 1 A1:,A2:3 A1:P1,A2:P1" 374 375 # Changes to cpuset.cpus.exclusive that violate exclusivity rule is rejected 376 " C0-3 . . C4-5 X0-3 . . X3-5 1 A1:0-3,B1:4-5" 377 378 # cpuset.cpus cannot be a subset of sibling cpuset.cpus.exclusive 379 " C0-3 . . C4-5 X3-5 . . . 1 A1:0-3,B1:4-5" 380) 381 382# 383# Write to the cpu online file 384# $1 - <c>=<v> where <c> = cpu number, <v> value to be written 385# 386write_cpu_online() 387{ 388 CPU=${1%=*} 389 VAL=${1#*=} 390 CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online 391 if [[ $VAL -eq 0 ]] 392 then 393 OFFLINE_CPUS="$OFFLINE_CPUS $CPU" 394 else 395 [[ -n "$OFFLINE_CPUS" ]] && { 396 OFFLINE_CPUS=$(echo $CPU $CPU $OFFLINE_CPUS | fmt -1 |\ 397 sort | uniq -u) 398 } 399 fi 400 echo $VAL > $CPUFILE 401 pause 0.05 402} 403 404# 405# Set controller state 406# $1 - cgroup directory 407# $2 - state 408# $3 - showerr 409# 410# The presence of ":" in state means transition from one to the next. 411# 412set_ctrl_state() 413{ 414 TMPMSG=/tmp/.msg_$$ 415 CGRP=$1 416 STATE=$2 417 SHOWERR=${3} 418 CTRL=${CTRL:=$CONTROLLER} 419 HASERR=0 420 REDIRECT="2> $TMPMSG" 421 [[ -z "$STATE" || "$STATE" = '.' ]] && return 0 422 [[ $VERBOSE -gt 0 ]] && SHOWERR=1 423 424 rm -f $TMPMSG 425 for CMD in $(echo $STATE | sed -e "s/:/ /g") 426 do 427 TFILE=$CGRP/cgroup.procs 428 SFILE=$CGRP/cgroup.subtree_control 429 PFILE=$CGRP/cpuset.cpus.partition 430 CFILE=$CGRP/cpuset.cpus 431 XFILE=$CGRP/cpuset.cpus.exclusive 432 S=$(expr substr $CMD 1 1) 433 if [[ $S = S ]] 434 then 435 PREFIX=${CMD#?} 436 COMM="echo ${PREFIX}${CTRL} > $SFILE" 437 eval $COMM $REDIRECT 438 elif [[ $S = X ]] 439 then 440 CPUS=${CMD#?} 441 COMM="echo $CPUS > $XFILE" 442 eval $COMM $REDIRECT 443 elif [[ $S = C ]] 444 then 445 CPUS=${CMD#?} 446 COMM="echo $CPUS > $CFILE" 447 eval $COMM $REDIRECT 448 elif [[ $S = P ]] 449 then 450 VAL=${CMD#?} 451 case $VAL in 452 0) VAL=member 453 ;; 454 1) VAL=root 455 ;; 456 2) VAL=isolated 457 ;; 458 *) 459 echo "Invalid partition state - $VAL" 460 exit 1 461 ;; 462 esac 463 COMM="echo $VAL > $PFILE" 464 eval $COMM $REDIRECT 465 elif [[ $S = O ]] 466 then 467 VAL=${CMD#?} 468 write_cpu_online $VAL 469 elif [[ $S = T ]] 470 then 471 COMM="echo 0 > $TFILE" 472 eval $COMM $REDIRECT 473 fi 474 RET=$? 475 [[ $RET -ne 0 ]] && { 476 [[ -n "$SHOWERR" ]] && { 477 echo "$COMM" 478 cat $TMPMSG 479 } 480 HASERR=1 481 } 482 pause 0.01 483 rm -f $TMPMSG 484 done 485 return $HASERR 486} 487 488set_ctrl_state_noerr() 489{ 490 CGRP=$1 491 STATE=$2 492 [[ -d $CGRP ]] || mkdir $CGRP 493 set_ctrl_state $CGRP $STATE 1 494 [[ $? -ne 0 ]] && { 495 echo "ERROR: Failed to set $2 to cgroup $1!" 496 exit 1 497 } 498} 499 500online_cpus() 501{ 502 [[ -n "OFFLINE_CPUS" ]] && { 503 for C in $OFFLINE_CPUS 504 do 505 write_cpu_online ${C}=1 506 done 507 } 508} 509 510# 511# Return 1 if the list of effective cpus isn't the same as the initial list. 512# 513reset_cgroup_states() 514{ 515 echo 0 > $CGROUP2/cgroup.procs 516 online_cpus 517 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1 518 pause 0.02 519 set_ctrl_state . R- 520 pause 0.01 521} 522 523dump_states() 524{ 525 for DIR in . A1 A1/A2 A1/A2/A3 B1 526 do 527 CPUS=$DIR/cpuset.cpus 528 ECPUS=$DIR/cpuset.cpus.effective 529 XCPUS=$DIR/cpuset.cpus.exclusive 530 XECPUS=$DIR/cpuset.cpus.exclusive.effective 531 PRS=$DIR/cpuset.cpus.partition 532 PCPUS=$DIR/.__DEBUG__.cpuset.cpus.subpartitions 533 ISCPUS=$DIR/cpuset.cpus.isolated 534 [[ -e $CPUS ]] && echo "$CPUS: $(cat $CPUS)" 535 [[ -e $XCPUS ]] && echo "$XCPUS: $(cat $XCPUS)" 536 [[ -e $ECPUS ]] && echo "$ECPUS: $(cat $ECPUS)" 537 [[ -e $XECPUS ]] && echo "$XECPUS: $(cat $XECPUS)" 538 [[ -e $PRS ]] && echo "$PRS: $(cat $PRS)" 539 [[ -e $PCPUS ]] && echo "$PCPUS: $(cat $PCPUS)" 540 [[ -e $ISCPUS ]] && echo "$ISCPUS: $(cat $ISCPUS)" 541 done 542} 543 544# 545# Check effective cpus 546# $1 - check string, format: <cgroup>:<cpu-list>[,<cgroup>:<cpu-list>]* 547# 548check_effective_cpus() 549{ 550 CHK_STR=$1 551 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g") 552 do 553 set -- $(echo $CHK | sed -e "s/:/ /g") 554 CGRP=$1 555 CPUS=$2 556 if [[ $CGRP = X* ]] 557 then 558 CGRP=${CGRP#X} 559 FILE=cpuset.cpus.exclusive.effective 560 else 561 FILE=cpuset.cpus.effective 562 fi 563 [[ $CGRP = A2 ]] && CGRP=A1/A2 564 [[ $CGRP = A3 ]] && CGRP=A1/A2/A3 565 [[ -e $CGRP/$FILE ]] || return 1 566 [[ $CPUS = $(cat $CGRP/$FILE) ]] || return 1 567 done 568} 569 570# 571# Check cgroup states 572# $1 - check string, format: <cgroup>:<state>[,<cgroup>:<state>]* 573# 574check_cgroup_states() 575{ 576 CHK_STR=$1 577 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g") 578 do 579 set -- $(echo $CHK | sed -e "s/:/ /g") 580 CGRP=$1 581 CGRP_DIR=$CGRP 582 STATE=$2 583 FILE= 584 EVAL=$(expr substr $STATE 2 2) 585 [[ $CGRP = A2 ]] && CGRP_DIR=A1/A2 586 [[ $CGRP = A3 ]] && CGRP_DIR=A1/A2/A3 587 588 case $STATE in 589 P*) FILE=$CGRP_DIR/cpuset.cpus.partition 590 ;; 591 *) echo "Unknown state: $STATE!" 592 exit 1 593 ;; 594 esac 595 VAL=$(cat $FILE) 596 597 case "$VAL" in 598 member) VAL=0 599 ;; 600 root) VAL=1 601 ;; 602 isolated) 603 VAL=2 604 ;; 605 "root invalid"*) 606 VAL=-1 607 ;; 608 "isolated invalid"*) 609 VAL=-2 610 ;; 611 esac 612 [[ $EVAL != $VAL ]] && return 1 613 614 # 615 # For root partition, dump sched-domains info to console if 616 # verbose mode set for manual comparison with sched debug info. 617 # 618 [[ $VAL -eq 1 && $VERBOSE -gt 0 ]] && { 619 DOMS=$(cat $CGRP_DIR/cpuset.cpus.effective) 620 [[ -n "$DOMS" ]] && 621 echo " [$CGRP] sched-domain: $DOMS" > $CONSOLE 622 } 623 done 624 return 0 625} 626 627# 628# Get isolated (including offline) CPUs by looking at 629# /sys/kernel/debug/sched/domains and cpuset.cpus.isolated control file, 630# if available, and compare that with the expected value. 631# 632# Note that isolated CPUs from the sched/domains context include offline 633# CPUs as well as CPUs in non-isolated 1-CPU partition. Those CPUs may 634# not be included in the cpuset.cpus.isolated control file which contains 635# only CPUs in isolated partitions. 636# 637# $1 - expected isolated cpu list(s) <isolcpus1>{,<isolcpus2>} 638# <isolcpus1> - expected sched/domains value 639# <isolcpus2> - cpuset.cpus.isolated value = <isolcpus1> if not defined 640# 641check_isolcpus() 642{ 643 EXPECT_VAL=$1 644 ISOLCPUS= 645 LASTISOLCPU= 646 SCHED_DOMAINS=/sys/kernel/debug/sched/domains 647 ISCPUS=${CGROUP2}/cpuset.cpus.isolated 648 if [[ $EXPECT_VAL = . ]] 649 then 650 EXPECT_VAL= 651 EXPECT_VAL2= 652 elif [[ $(expr $EXPECT_VAL : ".*,.*") > 0 ]] 653 then 654 set -- $(echo $EXPECT_VAL | sed -e "s/,/ /g") 655 EXPECT_VAL=$1 656 EXPECT_VAL2=$2 657 else 658 EXPECT_VAL2=$EXPECT_VAL 659 fi 660 661 # 662 # Check the debug isolated cpumask, if present 663 # 664 [[ -f $ISCPUS ]] && { 665 ISOLCPUS=$(cat $ISCPUS) 666 [[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && { 667 # Take a 50ms pause and try again 668 pause 0.05 669 ISOLCPUS=$(cat $ISCPUS) 670 } 671 [[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && return 1 672 ISOLCPUS= 673 } 674 675 # 676 # Use the sched domain in debugfs to check isolated CPUs, if available 677 # 678 [[ -d $SCHED_DOMAINS ]] || return 0 679 680 for ((CPU=0; CPU < $NR_CPUS; CPU++)) 681 do 682 [[ -n "$(ls ${SCHED_DOMAINS}/cpu$CPU)" ]] && continue 683 684 if [[ -z "$LASTISOLCPU" ]] 685 then 686 ISOLCPUS=$CPU 687 LASTISOLCPU=$CPU 688 elif [[ "$LASTISOLCPU" -eq $((CPU - 1)) ]] 689 then 690 echo $ISOLCPUS | grep -q "\<$LASTISOLCPU\$" 691 if [[ $? -eq 0 ]] 692 then 693 ISOLCPUS=${ISOLCPUS}- 694 fi 695 LASTISOLCPU=$CPU 696 else 697 if [[ $ISOLCPUS = *- ]] 698 then 699 ISOLCPUS=${ISOLCPUS}$LASTISOLCPU 700 fi 701 ISOLCPUS=${ISOLCPUS},$CPU 702 LASTISOLCPU=$CPU 703 fi 704 done 705 [[ "$ISOLCPUS" = *- ]] && ISOLCPUS=${ISOLCPUS}$LASTISOLCPU 706 [[ "$EXPECT_VAL" = "$ISOLCPUS" ]] 707} 708 709test_fail() 710{ 711 TESTNUM=$1 712 TESTTYPE=$2 713 ADDINFO=$3 714 echo "Test $TEST[$TESTNUM] failed $TESTTYPE check!" 715 [[ -n "$ADDINFO" ]] && echo "*** $ADDINFO ***" 716 eval echo \${$TEST[$I]} 717 echo 718 dump_states 719 exit 1 720} 721 722# 723# Check to see if there are unexpected isolated CPUs left 724# 725null_isolcpus_check() 726{ 727 [[ $VERBOSE -gt 0 ]] || return 0 728 # Retry a few times before printing error 729 RETRY=0 730 while [[ $RETRY -lt 8 ]] 731 do 732 pause 0.02 733 check_isolcpus "." 734 [[ $? -eq 0 ]] && return 0 735 ((RETRY++)) 736 done 737 echo "Unexpected isolated CPUs: $ISOLCPUS" 738 dump_states 739 exit 1 740} 741 742# 743# Run cpuset state transition test 744# $1 - test matrix name 745# 746# This test is somewhat fragile as delays (sleep x) are added in various 747# places to make sure state changes are fully propagated before the next 748# action. These delays may need to be adjusted if running in a slower machine. 749# 750run_state_test() 751{ 752 TEST=$1 753 CONTROLLER=cpuset 754 I=0 755 eval CNT="\${#$TEST[@]}" 756 757 reset_cgroup_states 758 console_msg "Running state transition test ..." 759 760 while [[ $I -lt $CNT ]] 761 do 762 echo "Running test $I ..." > $CONSOLE 763 [[ $VERBOSE -gt 1 ]] && { 764 echo "" 765 eval echo \${$TEST[$I]} 766 } 767 eval set -- "\${$TEST[$I]}" 768 OLD_A1=$1 769 OLD_A2=$2 770 OLD_A3=$3 771 OLD_B1=$4 772 NEW_A1=$5 773 NEW_A2=$6 774 NEW_A3=$7 775 NEW_B1=$8 776 RESULT=$9 777 ECPUS=${10} 778 STATES=${11} 779 ICPUS=${12} 780 781 set_ctrl_state_noerr B1 $OLD_B1 782 set_ctrl_state_noerr A1 $OLD_A1 783 set_ctrl_state_noerr A1/A2 $OLD_A2 784 set_ctrl_state_noerr A1/A2/A3 $OLD_A3 785 RETVAL=0 786 set_ctrl_state A1 $NEW_A1; ((RETVAL += $?)) 787 set_ctrl_state A1/A2 $NEW_A2; ((RETVAL += $?)) 788 set_ctrl_state A1/A2/A3 $NEW_A3; ((RETVAL += $?)) 789 set_ctrl_state B1 $NEW_B1; ((RETVAL += $?)) 790 791 [[ $RETVAL -ne $RESULT ]] && test_fail $I result 792 793 [[ -n "$ECPUS" && "$ECPUS" != . ]] && { 794 check_effective_cpus $ECPUS 795 [[ $? -ne 0 ]] && test_fail $I "effective CPU" 796 } 797 798 [[ -n "$STATES" && "$STATES" != . ]] && { 799 check_cgroup_states $STATES 800 [[ $? -ne 0 ]] && test_fail $I states 801 } 802 803 # Compare the expected isolated CPUs with the actual ones, 804 # if available 805 [[ -n "$ICPUS" ]] && { 806 check_isolcpus $ICPUS 807 [[ $? -ne 0 ]] && test_fail $I "isolated CPU" \ 808 "Expect $ICPUS, get $ISOLCPUS instead" 809 } 810 reset_cgroup_states 811 # 812 # Check to see if effective cpu list changes 813 # 814 NEWLIST=$(cat cpuset.cpus.effective) 815 RETRY=0 816 while [[ $NEWLIST != $CPULIST && $RETRY -lt 8 ]] 817 do 818 # Wait a bit longer & recheck a few times 819 pause 0.02 820 ((RETRY++)) 821 NEWLIST=$(cat cpuset.cpus.effective) 822 done 823 [[ $NEWLIST != $CPULIST ]] && { 824 echo "Effective cpus changed to $NEWLIST after test $I!" 825 exit 1 826 } 827 null_isolcpus_check 828 [[ $VERBOSE -gt 0 ]] && echo "Test $I done." 829 ((I++)) 830 done 831 echo "All $I tests of $TEST PASSED." 832} 833 834# 835# Testing the new "isolated" partition root type 836# 837test_isolated() 838{ 839 cd $CGROUP2/test 840 echo 2-3 > cpuset.cpus 841 TYPE=$(cat cpuset.cpus.partition) 842 [[ $TYPE = member ]] || echo member > cpuset.cpus.partition 843 844 console_msg "Change from member to root" 845 test_partition root 846 847 console_msg "Change from root to isolated" 848 test_partition isolated 849 850 console_msg "Change from isolated to member" 851 test_partition member 852 853 console_msg "Change from member to isolated" 854 test_partition isolated 855 856 console_msg "Change from isolated to root" 857 test_partition root 858 859 console_msg "Change from root to member" 860 test_partition member 861 862 # 863 # Testing partition root with no cpu 864 # 865 console_msg "Distribute all cpus to child partition" 866 echo +cpuset > cgroup.subtree_control 867 test_partition root 868 869 mkdir A1 870 cd A1 871 echo 2-3 > cpuset.cpus 872 test_partition root 873 test_effective_cpus 2-3 874 cd .. 875 test_effective_cpus "" 876 877 console_msg "Moving task to partition test" 878 test_add_proc "No space left" 879 cd A1 880 test_add_proc "" 881 cd .. 882 883 console_msg "Shrink and expand child partition" 884 cd A1 885 echo 2 > cpuset.cpus 886 cd .. 887 test_effective_cpus 3 888 cd A1 889 echo 2-3 > cpuset.cpus 890 cd .. 891 test_effective_cpus "" 892 893 # Cleaning up 894 console_msg "Cleaning up" 895 echo $$ > $CGROUP2/cgroup.procs 896 [[ -d A1 ]] && rmdir A1 897 null_isolcpus_check 898} 899 900# 901# Wait for inotify event for the given file and read it 902# $1: cgroup file to wait for 903# $2: file to store the read result 904# 905wait_inotify() 906{ 907 CGROUP_FILE=$1 908 OUTPUT_FILE=$2 909 910 $WAIT_INOTIFY $CGROUP_FILE 911 cat $CGROUP_FILE > $OUTPUT_FILE 912} 913 914# 915# Test if inotify events are properly generated when going into and out of 916# invalid partition state. 917# 918test_inotify() 919{ 920 ERR=0 921 PRS=/tmp/.prs_$$ 922 cd $CGROUP2/test 923 [[ -f $WAIT_INOTIFY ]] || { 924 echo "wait_inotify not found, inotify test SKIPPED." 925 return 926 } 927 928 pause 0.01 929 echo 1 > cpuset.cpus 930 echo 0 > cgroup.procs 931 echo root > cpuset.cpus.partition 932 pause 0.01 933 rm -f $PRS 934 wait_inotify $PWD/cpuset.cpus.partition $PRS & 935 pause 0.01 936 set_ctrl_state . "O1=0" 937 pause 0.01 938 check_cgroup_states ".:P-1" 939 if [[ $? -ne 0 ]] 940 then 941 echo "FAILED: Inotify test - partition not invalid" 942 ERR=1 943 elif [[ ! -f $PRS ]] 944 then 945 echo "FAILED: Inotify test - event not generated" 946 ERR=1 947 kill %1 948 elif [[ $(cat $PRS) != "root invalid"* ]] 949 then 950 echo "FAILED: Inotify test - incorrect state" 951 cat $PRS 952 ERR=1 953 fi 954 online_cpus 955 echo member > cpuset.cpus.partition 956 echo 0 > ../cgroup.procs 957 if [[ $ERR -ne 0 ]] 958 then 959 exit 1 960 else 961 echo "Inotify test PASSED" 962 fi 963} 964 965trap cleanup 0 2 3 6 966run_state_test TEST_MATRIX 967test_isolated 968test_inotify 969echo "All tests PASSED." 970