1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0 3# 4# Test for cpuset v2 partition root state (PRS) 5# 6# The sched verbose flag can be optionally set so that the console log 7# can be examined for the correct setting of scheduling domain. 8# 9 10skip_test() { 11 echo "$1" 12 echo "Test SKIPPED" 13 exit 4 # ksft_skip 14} 15 16[[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!" 17 18 19# Get wait_inotify location 20WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify 21 22# Find cgroup v2 mount point 23CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}') 24[[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!" 25SUBPARTS_CPUS=$CGROUP2/.__DEBUG__.cpuset.cpus.subpartitions 26CPULIST=$(cat $CGROUP2/cpuset.cpus.effective) 27 28NR_CPUS=$(lscpu | grep "^CPU(s):" | sed -e "s/.*:[[:space:]]*//") 29[[ $NR_CPUS -lt 8 ]] && skip_test "Test needs at least 8 cpus available!" 30 31# Check to see if /dev/console exists and is writable 32if [[ -c /dev/console && -w /dev/console ]] 33then 34 CONSOLE=/dev/console 35else 36 CONSOLE=/dev/null 37fi 38 39# Set verbose flag and delay factor 40PROG=$1 41VERBOSE=0 42DELAY_FACTOR=1 43SCHED_DEBUG= 44while [[ "$1" = -* ]] 45do 46 case "$1" in 47 -v) ((VERBOSE++)) 48 # Enable sched/verbose can slow thing down 49 [[ $DELAY_FACTOR -eq 1 ]] && 50 DELAY_FACTOR=2 51 ;; 52 -d) DELAY_FACTOR=$2 53 shift 54 ;; 55 *) echo "Usage: $PROG [-v] [-d <delay-factor>" 56 exit 57 ;; 58 esac 59 shift 60done 61 62# Set sched verbose flag if available when "-v" option is specified 63if [[ $VERBOSE -gt 0 && -d /sys/kernel/debug/sched ]] 64then 65 # Used to restore the original setting during cleanup 66 SCHED_DEBUG=$(cat /sys/kernel/debug/sched/verbose) 67 echo Y > /sys/kernel/debug/sched/verbose 68fi 69 70cd $CGROUP2 71echo +cpuset > cgroup.subtree_control 72 73# 74# If cpuset has been set up and used in child cgroups, we may not be able to 75# create partition under root cgroup because of the CPU exclusivity rule. 76# So we are going to skip the test if this is the case. 77# 78[[ -d test ]] || mkdir test 79echo 0-6 > test/cpuset.cpus 80echo root > test/cpuset.cpus.partition 81cat test/cpuset.cpus.partition | grep -q invalid 82RESULT=$? 83echo member > test/cpuset.cpus.partition 84echo "" > test/cpuset.cpus 85[[ $RESULT -eq 0 ]] && skip_test "Child cgroups are using cpuset!" 86 87# 88# If isolated CPUs have been reserved at boot time (as shown in 89# cpuset.cpus.isolated), these isolated CPUs should be outside of CPUs 0-8 90# that will be used by this script for testing purpose. If not, some of 91# the tests may fail incorrectly. These pre-isolated CPUs should stay in 92# an isolated state throughout the testing process for now. 93# 94BOOT_ISOLCPUS=$(cat $CGROUP2/cpuset.cpus.isolated) 95if [[ -n "$BOOT_ISOLCPUS" ]] 96then 97 [[ $(echo $BOOT_ISOLCPUS | sed -e "s/[,-].*//") -le 8 ]] && 98 skip_test "Pre-isolated CPUs ($BOOT_ISOLCPUS) overlap CPUs to be tested" 99 echo "Pre-isolated CPUs: $BOOT_ISOLCPUS" 100fi 101cleanup() 102{ 103 online_cpus 104 cd $CGROUP2 105 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1 106 rmdir test > /dev/null 2>&1 107 [[ -n "$SCHED_DEBUG" ]] && 108 echo "$SCHED_DEBUG" > /sys/kernel/debug/sched/verbose 109} 110 111# Pause in ms 112pause() 113{ 114 DELAY=$1 115 LOOP=0 116 while [[ $LOOP -lt $DELAY_FACTOR ]] 117 do 118 sleep $DELAY 119 ((LOOP++)) 120 done 121 return 0 122} 123 124console_msg() 125{ 126 MSG=$1 127 echo "$MSG" 128 echo "" > $CONSOLE 129 echo "$MSG" > $CONSOLE 130 pause 0.01 131} 132 133test_partition() 134{ 135 EXPECTED_VAL=$1 136 echo $EXPECTED_VAL > cpuset.cpus.partition 137 [[ $? -eq 0 ]] || exit 1 138 ACTUAL_VAL=$(cat cpuset.cpus.partition) 139 [[ $ACTUAL_VAL != $EXPECTED_VAL ]] && { 140 echo "cpuset.cpus.partition: expect $EXPECTED_VAL, found $ACTUAL_VAL" 141 echo "Test FAILED" 142 exit 1 143 } 144} 145 146test_effective_cpus() 147{ 148 EXPECTED_VAL=$1 149 ACTUAL_VAL=$(cat cpuset.cpus.effective) 150 [[ "$ACTUAL_VAL" != "$EXPECTED_VAL" ]] && { 151 echo "cpuset.cpus.effective: expect '$EXPECTED_VAL', found '$ACTUAL_VAL'" 152 echo "Test FAILED" 153 exit 1 154 } 155} 156 157# Adding current process to cgroup.procs as a test 158test_add_proc() 159{ 160 OUTSTR="$1" 161 ERRMSG=$((echo $$ > cgroup.procs) |& cat) 162 echo $ERRMSG | grep -q "$OUTSTR" 163 [[ $? -ne 0 ]] && { 164 echo "cgroup.procs: expect '$OUTSTR', got '$ERRMSG'" 165 echo "Test FAILED" 166 exit 1 167 } 168 echo $$ > $CGROUP2/cgroup.procs # Move out the task 169} 170 171# 172# Cpuset controller state transition test matrix. 173# 174# Cgroup test hierarchy 175# 176# root -- A1 -- A2 -- A3 177# +- B1 178# 179# P<v> = set cpus.partition (0:member, 1:root, 2:isolated) 180# C<l> = add cpu-list to cpuset.cpus 181# X<l> = add cpu-list to cpuset.cpus.exclusive 182# S<p> = use prefix in subtree_control 183# T = put a task into cgroup 184# O<c>=<v> = Write <v> to CPU online file of <c> 185# 186# ECPUs - effective CPUs of cpusets 187# Pstate - partition root state 188# ISOLCPUS - isolated CPUs (<icpus>[,<icpus2>]) 189# 190# Note that if there are 2 fields in ISOLCPUS, the first one is for 191# sched-debug matching which includes offline CPUs and single-CPU partitions 192# while the second one is for matching cpuset.cpus.isolated. 193# 194SETUP_A123_PARTITIONS="C1-3:P1:S+ C2-3:P1:S+ C3:P1" 195TEST_MATRIX=( 196 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 197 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 198 " C0-1 . . C2-3 S+ C4-5 . . 0 A2:0-1" 199 " C0-1 . . C2-3 P1 . . . 0 " 200 " C0-1 . . C2-3 P1:S+ C0-1:P1 . . 0 " 201 " C0-1 . . C2-3 P1:S+ C1:P1 . . 0 " 202 " C0-1:S+ . . C2-3 . . . P1 0 " 203 " C0-1:P1 . . C2-3 S+ C1 . . 0 " 204 " C0-1:P1 . . C2-3 S+ C1:P1 . . 0 " 205 " C0-1:P1 . . C2-3 S+ C1:P1 . P1 0 " 206 " C0-1:P1 . . C2-3 C4-5 . . . 0 A1:4-5" 207 " C0-1:P1 . . C2-3 S+:C4-5 . . . 0 A1:4-5" 208 " C0-1 . . C2-3:P1 . . . C2 0 " 209 " C0-1 . . C2-3:P1 . . . C4-5 0 B1:4-5" 210 "C0-3:P1:S+ C2-3:P1 . . . . . . 0 A1:0-1,A2:2-3" 211 "C0-3:P1:S+ C2-3:P1 . . C1-3 . . . 0 A1:1,A2:2-3" 212 "C2-3:P1:S+ C3:P1 . . C3 . . . 0 A1:,A2:3 A1:P1,A2:P1" 213 "C2-3:P1:S+ C3:P1 . . C3 P0 . . 0 A1:3,A2:3 A1:P1,A2:P0" 214 "C2-3:P1:S+ C2:P1 . . C2-4 . . . 0 A1:3-4,A2:2" 215 "C2-3:P1:S+ C3:P1 . . C3 . . C0-2 0 A1:,B1:0-2 A1:P1,A2:P1" 216 "$SETUP_A123_PARTITIONS . C2-3 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 217 218 # CPU offlining cases: 219 " C0-1 . . C2-3 S+ C4-5 . O2=0 0 A1:0-1,B1:3" 220 "C0-3:P1:S+ C2-3:P1 . . O2=0 . . . 0 A1:0-1,A2:3" 221 "C0-3:P1:S+ C2-3:P1 . . O2=0 O2=1 . . 0 A1:0-1,A2:2-3" 222 "C0-3:P1:S+ C2-3:P1 . . O1=0 . . . 0 A1:0,A2:2-3" 223 "C0-3:P1:S+ C2-3:P1 . . O1=0 O1=1 . . 0 A1:0-1,A2:2-3" 224 "C2-3:P1:S+ C3:P1 . . O3=0 O3=1 . . 0 A1:2,A2:3 A1:P1,A2:P1" 225 "C2-3:P1:S+ C3:P2 . . O3=0 O3=1 . . 0 A1:2,A2:3 A1:P1,A2:P2" 226 "C2-3:P1:S+ C3:P1 . . O2=0 O2=1 . . 0 A1:2,A2:3 A1:P1,A2:P1" 227 "C2-3:P1:S+ C3:P2 . . O2=0 O2=1 . . 0 A1:2,A2:3 A1:P1,A2:P2" 228 "C2-3:P1:S+ C3:P1 . . O2=0 . . . 0 A1:,A2:3 A1:P1,A2:P1" 229 "C2-3:P1:S+ C3:P1 . . O3=0 . . . 0 A1:2,A2: A1:P1,A2:P1" 230 "C2-3:P1:S+ C3:P1 . . T:O2=0 . . . 0 A1:3,A2:3 A1:P1,A2:P-1" 231 "C2-3:P1:S+ C3:P1 . . . T:O3=0 . . 0 A1:2,A2:2 A1:P1,A2:P-1" 232 "$SETUP_A123_PARTITIONS . O1=0 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 233 "$SETUP_A123_PARTITIONS . O2=0 . . . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1" 234 "$SETUP_A123_PARTITIONS . O3=0 . . . 0 A1:1,A2:2,A3: A1:P1,A2:P1,A3:P1" 235 "$SETUP_A123_PARTITIONS . T:O1=0 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" 236 "$SETUP_A123_PARTITIONS . . T:O2=0 . . 0 A1:1,A2:3,A3:3 A1:P1,A2:P1,A3:P-1" 237 "$SETUP_A123_PARTITIONS . . . T:O3=0 . 0 A1:1,A2:2,A3:2 A1:P1,A2:P1,A3:P-1" 238 "$SETUP_A123_PARTITIONS . T:O1=0 O1=1 . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 239 "$SETUP_A123_PARTITIONS . . T:O2=0 O2=1 . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 240 "$SETUP_A123_PARTITIONS . . . T:O3=0 O3=1 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 241 "$SETUP_A123_PARTITIONS . T:O1=0 O2=0 O1=1 . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1" 242 "$SETUP_A123_PARTITIONS . T:O1=0 O2=0 O2=1 . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" 243 244 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 245 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 246 # 247 # Remote partition and cpuset.cpus.exclusive tests 248 # 249 " C0-3:S+ C1-3:S+ C2-3 . X2-3 . . . 0 A1:0-3,A2:1-3,A3:2-3,XA1:2-3" 250 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3:P2 . . 0 A1:0-1,A2:2-3,A3:2-3 A1:P0,A2:P2 2-3" 251 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X3:P2 . . 0 A1:0-2,A2:3,A3:3 A1:P0,A2:P2 3" 252 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2 . 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3" 253 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:C3 . 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3" 254 " C0-3:S+ C1-3:S+ C2-3 C2-3 . . . P2 0 A1:0-3,A2:1-3,A3:2-3,B1:2-3 A1:P0,A3:P0,B1:P-2" 255 " C0-3:S+ C1-3:S+ C2-3 C4-5 . . . P2 0 B1:4-5 B1:P2 4-5" 256 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2 P2 0 A3:2-3,B1:4 A3:P2,B1:P2 2-4" 257 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2:C1-3 P2 0 A3:2-3,B1:4 A3:P2,B1:P2 2-4" 258 " C0-3:S+ C1-3:S+ C2-3 C4 X1-3 X1-3:P2 P2 . 0 A2:1,A3:2-3 A2:P2,A3:P2 1-3" 259 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3 X2-3:P2 P2:C4-5 0 A3:2-3,B1:4-5 A3:P2,B1:P2 2-5" 260 " C4:X0-3:S+ X1-3:S+ X2-3 . . P2 . . 0 A1:4,A2:1-3,A3:1-3 A2:P2 1-3" 261 " C4:X0-3:S+ X1-3:S+ X2-3 . . . P2 . 0 A1:4,A2:4,A3:2-3 A3:P2 2-3" 262 263 # Nested remote/local partition tests 264 " C0-3:S+ C1-3:S+ C2-3 C4-5 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:,A3:2-3,B1:4-5 \ 265 A1:P0,A2:P1,A3:P2,B1:P1 2-3" 266 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:,A3:2-3,B1:4 \ 267 A1:P0,A2:P1,A3:P2,B1:P1 2-4,2-3" 268 " C0-3:S+ C1-3:S+ C2-3 C4 X2-3 X2-3:P1 . P1 0 A1:0-1,A2:2-3,A3:2-3,B1:4 \ 269 A1:P0,A2:P1,A3:P0,B1:P1" 270 " C0-3:S+ C1-3:S+ C3 C4 X2-3 X2-3:P1 P2 P1 0 A1:0-1,A2:2,A3:3,B1:4 \ 271 A1:P0,A2:P1,A3:P2,B1:P1 2-4,3" 272 " C0-4:S+ C1-4:S+ C2-4 . X2-4 X2-4:P2 X4:P1 . 0 A1:0-1,A2:2-3,A3:4 \ 273 A1:P0,A2:P2,A3:P1 2-4,2-3" 274 " C0-4:S+ C1-4:S+ C2-4 . X2-4 X2-4:P2 X3-4:P1 . 0 A1:0-1,A2:2,A3:3-4 \ 275 A1:P0,A2:P2,A3:P1 2" 276 " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \ 277 . . X5 . . 0 A1:0-4,A2:1-4,A3:2-4 \ 278 A1:P0,A2:P-2,A3:P-1" 279 " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \ 280 . . . X1 . 0 A1:0-1,A2:2-4,A3:2-4 \ 281 A1:P0,A2:P2,A3:P-1 2-4" 282 283 # Remote partition offline tests 284 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:O2=0 . 0 A1:0-1,A2:1,A3:3 A1:P0,A3:P2 2-3" 285 " C0-3:S+ C1-3:S+ C2-3 . X2-3 X2-3 X2-3:P2:O2=0 O2=1 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3" 286 " C0-3:S+ C1-3:S+ C3 . X2-3 X2-3 P2:O3=0 . 0 A1:0-2,A2:1-2,A3: A1:P0,A3:P2 3" 287 " C0-3:S+ C1-3:S+ C3 . X2-3 X2-3 T:P2:O3=0 . 0 A1:0-2,A2:1-2,A3:1-2 A1:P0,A3:P-2 3," 288 289 # An invalidated remote partition cannot self-recover from hotplug 290 " C0-3:S+ C1-3:S+ C2 . X2-3 X2-3 T:P2:O2=0 O2=1 0 A1:0-3,A2:1-3,A3:2 A1:P0,A3:P-2" 291 292 # cpus.exclusive.effective clearing test 293 " C0-3:S+ C1-3:S+ C2 . X2-3:X . . . 0 A1:0-3,A2:1-3,A3:2,XA1:" 294 295 # Invalid to valid remote partition transition test 296 " C0-3:S+ C1-3 . . . X3:P2 . . 0 A1:0-3,A2:1-3,XA2: A2:P-2" 297 " C0-3:S+ C1-3:X3:P2 298 . . X2-3 P2 . . 0 A1:0-2,A2:3,XA2:3 A2:P2 3" 299 300 # Invalid to valid local partition direct transition tests 301 " C1-3:S+:P2 X4:P2 . . . . . . 0 A1:1-3,XA1:1-3,A2:1-3:XA2: A1:P2,A2:P-2 1-3" 302 " C1-3:S+:P2 X4:P2 . . . X3:P2 . . 0 A1:1-2,XA1:1-3,A2:3:XA2:3 A1:P2,A2:P2 1-3" 303 " C0-3:P2 . . C4-6 C0-4 . . . 0 A1:0-4,B1:4-6 A1:P-2,B1:P0" 304 " C0-3:P2 . . C4-6 C0-4:C0-3 . . . 0 A1:0-3,B1:4-6 A1:P2,B1:P0 0-3" 305 " C0-3:P2 . . C3-5:C4-5 . . . . 0 A1:0-3,B1:4-5 A1:P2,B1:P0 0-3" 306 307 # Local partition invalidation tests 308 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \ 309 . . . . . 0 A1:1,A2:2,A3:3 A1:P2,A2:P2,A3:P2 1-3" 310 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \ 311 . . X4 . . 0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3" 312 " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \ 313 . . C4:X . . 0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3" 314 # Local partition CPU change tests 315 " C0-5:S+:P2 C4-5:S+:P1 . . . C3-5 . . 0 A1:0-2,A2:3-5 A1:P2,A2:P1 0-2" 316 " C0-5:S+:P2 C4-5:S+:P1 . . C1-5 . . . 0 A1:1-3,A2:4-5 A1:P2,A2:P1 1-3" 317 318 # cpus_allowed/exclusive_cpus update tests 319 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \ 320 . X:C4 . P2 . 0 A1:4,A2:4,XA2:,XA3:,A3:4 \ 321 A1:P0,A3:P-2" 322 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \ 323 . X1 . P2 . 0 A1:0-3,A2:1-3,XA1:1,XA2:,XA3:,A3:2-3 \ 324 A1:P0,A3:P-2" 325 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \ 326 . . X3 P2 . 0 A1:0-2,A2:1-2,XA2:3,XA3:3,A3:3 \ 327 A1:P0,A3:P2 3" 328 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \ 329 . . X3 . . 0 A1:0-3,A2:1-3,XA2:3,XA3:3,A3:2-3 \ 330 A1:P0,A3:P-2" 331 " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \ 332 . X4 . . . 0 A1:0-3,A2:1-3,A3:2-3,XA1:4,XA2:,XA3 \ 333 A1:P0,A3:P-2" 334 335 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 336 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 337 # 338 # Incorrect change to cpuset.cpus[.exclusive] invalidates partition root 339 # 340 # Adding CPUs to partition root that are not in parent's 341 # cpuset.cpus is allowed, but those extra CPUs are ignored. 342 "C2-3:P1:S+ C3:P1 . . . C2-4 . . 0 A1:,A2:2-3 A1:P1,A2:P1" 343 344 # Taking away all CPUs from parent or itself if there are tasks 345 # will make the partition invalid. 346 "C2-3:P1:S+ C3:P1 . . T C2-3 . . 0 A1:2-3,A2:2-3 A1:P1,A2:P-1" 347 " C3:P1:S+ C3 . . T P1 . . 0 A1:3,A2:3 A1:P1,A2:P-1" 348 "$SETUP_A123_PARTITIONS . T:C2-3 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" 349 "$SETUP_A123_PARTITIONS . T:C2-3:C1-3 . . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" 350 351 # Changing a partition root to member makes child partitions invalid 352 "C2-3:P1:S+ C3:P1 . . P0 . . . 0 A1:2-3,A2:3 A1:P0,A2:P-1" 353 "$SETUP_A123_PARTITIONS . C2-3 P0 . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P0,A3:P-1" 354 355 # cpuset.cpus can contains cpus not in parent's cpuset.cpus as long 356 # as they overlap. 357 "C2-3:P1:S+ . . . . C3-4:P1 . . 0 A1:2,A2:3 A1:P1,A2:P1" 358 359 # Deletion of CPUs distributed to child cgroup is allowed. 360 "C0-1:P1:S+ C1 . C2-3 C4-5 . . . 0 A1:4-5,A2:4-5" 361 362 # To become a valid partition root, cpuset.cpus must overlap parent's 363 # cpuset.cpus. 364 " C0-1:P1 . . C2-3 S+ C4-5:P1 . . 0 A1:0-1,A2:0-1 A1:P1,A2:P-1" 365 366 # Enabling partition with child cpusets is allowed 367 " C0-1:S+ C1 . C2-3 P1 . . . 0 A1:0-1,A2:1 A1:P1" 368 369 # A partition root with non-partition root parent is invalid, but it 370 # can be made valid if its parent becomes a partition root too. 371 " C0-1:S+ C1 . C2-3 . P2 . . 0 A1:0-1,A2:1 A1:P0,A2:P-2" 372 " C0-1:S+ C1:P2 . C2-3 P1 . . . 0 A1:0,A2:1 A1:P1,A2:P2" 373 374 # A non-exclusive cpuset.cpus change will invalidate partition and its siblings 375 " C0-1:P1 . . C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P0" 376 " C0-1:P1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P-1" 377 " C0-1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P0,B1:P-1" 378 379 # cpuset.cpus can overlap with sibling cpuset.cpus.exclusive but not subsumed by it 380 " C0-3 . . C4-5 X5 . . . 0 A1:0-3,B1:4-5" 381 382 # Child partition root that try to take all CPUs from parent partition 383 # with tasks will remain invalid. 384 " C1-4:P1:S+ P1 . . . . . . 0 A1:1-4,A2:1-4 A1:P1,A2:P-1" 385 " C1-4:P1:S+ P1 . . . C1-4 . . 0 A1,A2:1-4 A1:P1,A2:P1" 386 " C1-4:P1:S+ P1 . . T C1-4 . . 0 A1:1-4,A2:1-4 A1:P1,A2:P-1" 387 388 # Clearing of cpuset.cpus with a preset cpuset.cpus.exclusive shouldn't 389 # affect cpuset.cpus.exclusive.effective. 390 " C1-4:X3:S+ C1:X3 . . . C . . 0 A2:1-4,XA2:3" 391 392 # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS 393 # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- 394 # Failure cases: 395 396 # A task cannot be added to a partition with no cpu 397 "C2-3:P1:S+ C3:P1 . . O2=0:T . . . 1 A1:,A2:3 A1:P1,A2:P1" 398 399 # Changes to cpuset.cpus.exclusive that violate exclusivity rule is rejected 400 " C0-3 . . C4-5 X0-3 . . X3-5 1 A1:0-3,B1:4-5" 401 402 # cpuset.cpus cannot be a subset of sibling cpuset.cpus.exclusive 403 " C0-3 . . C4-5 X3-5 . . . 1 A1:0-3,B1:4-5" 404) 405 406# 407# Write to the cpu online file 408# $1 - <c>=<v> where <c> = cpu number, <v> value to be written 409# 410write_cpu_online() 411{ 412 CPU=${1%=*} 413 VAL=${1#*=} 414 CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online 415 if [[ $VAL -eq 0 ]] 416 then 417 OFFLINE_CPUS="$OFFLINE_CPUS $CPU" 418 else 419 [[ -n "$OFFLINE_CPUS" ]] && { 420 OFFLINE_CPUS=$(echo $CPU $CPU $OFFLINE_CPUS | fmt -1 |\ 421 sort | uniq -u) 422 } 423 fi 424 echo $VAL > $CPUFILE 425 pause 0.05 426} 427 428# 429# Set controller state 430# $1 - cgroup directory 431# $2 - state 432# $3 - showerr 433# 434# The presence of ":" in state means transition from one to the next. 435# 436set_ctrl_state() 437{ 438 TMPMSG=/tmp/.msg_$$ 439 CGRP=$1 440 STATE=$2 441 SHOWERR=${3} 442 CTRL=${CTRL:=$CONTROLLER} 443 HASERR=0 444 REDIRECT="2> $TMPMSG" 445 [[ -z "$STATE" || "$STATE" = '.' ]] && return 0 446 [[ $VERBOSE -gt 0 ]] && SHOWERR=1 447 448 rm -f $TMPMSG 449 for CMD in $(echo $STATE | sed -e "s/:/ /g") 450 do 451 TFILE=$CGRP/cgroup.procs 452 SFILE=$CGRP/cgroup.subtree_control 453 PFILE=$CGRP/cpuset.cpus.partition 454 CFILE=$CGRP/cpuset.cpus 455 XFILE=$CGRP/cpuset.cpus.exclusive 456 S=$(expr substr $CMD 1 1) 457 if [[ $S = S ]] 458 then 459 PREFIX=${CMD#?} 460 COMM="echo ${PREFIX}${CTRL} > $SFILE" 461 eval $COMM $REDIRECT 462 elif [[ $S = X ]] 463 then 464 CPUS=${CMD#?} 465 COMM="echo $CPUS > $XFILE" 466 eval $COMM $REDIRECT 467 elif [[ $S = C ]] 468 then 469 CPUS=${CMD#?} 470 COMM="echo $CPUS > $CFILE" 471 eval $COMM $REDIRECT 472 elif [[ $S = P ]] 473 then 474 VAL=${CMD#?} 475 case $VAL in 476 0) VAL=member 477 ;; 478 1) VAL=root 479 ;; 480 2) VAL=isolated 481 ;; 482 *) 483 echo "Invalid partition state - $VAL" 484 exit 1 485 ;; 486 esac 487 COMM="echo $VAL > $PFILE" 488 eval $COMM $REDIRECT 489 elif [[ $S = O ]] 490 then 491 VAL=${CMD#?} 492 write_cpu_online $VAL 493 elif [[ $S = T ]] 494 then 495 COMM="echo 0 > $TFILE" 496 eval $COMM $REDIRECT 497 fi 498 RET=$? 499 [[ $RET -ne 0 ]] && { 500 [[ -n "$SHOWERR" ]] && { 501 echo "$COMM" 502 cat $TMPMSG 503 } 504 HASERR=1 505 } 506 pause 0.01 507 rm -f $TMPMSG 508 done 509 return $HASERR 510} 511 512set_ctrl_state_noerr() 513{ 514 CGRP=$1 515 STATE=$2 516 [[ -d $CGRP ]] || mkdir $CGRP 517 set_ctrl_state $CGRP $STATE 1 518 [[ $? -ne 0 ]] && { 519 echo "ERROR: Failed to set $2 to cgroup $1!" 520 exit 1 521 } 522} 523 524online_cpus() 525{ 526 [[ -n "OFFLINE_CPUS" ]] && { 527 for C in $OFFLINE_CPUS 528 do 529 write_cpu_online ${C}=1 530 done 531 } 532} 533 534# 535# Return 1 if the list of effective cpus isn't the same as the initial list. 536# 537reset_cgroup_states() 538{ 539 echo 0 > $CGROUP2/cgroup.procs 540 online_cpus 541 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1 542 pause 0.02 543 set_ctrl_state . R- 544 pause 0.01 545} 546 547dump_states() 548{ 549 for DIR in . A1 A1/A2 A1/A2/A3 B1 550 do 551 CPUS=$DIR/cpuset.cpus 552 ECPUS=$DIR/cpuset.cpus.effective 553 XCPUS=$DIR/cpuset.cpus.exclusive 554 XECPUS=$DIR/cpuset.cpus.exclusive.effective 555 PRS=$DIR/cpuset.cpus.partition 556 PCPUS=$DIR/.__DEBUG__.cpuset.cpus.subpartitions 557 ISCPUS=$DIR/cpuset.cpus.isolated 558 [[ -e $CPUS ]] && echo "$CPUS: $(cat $CPUS)" 559 [[ -e $XCPUS ]] && echo "$XCPUS: $(cat $XCPUS)" 560 [[ -e $ECPUS ]] && echo "$ECPUS: $(cat $ECPUS)" 561 [[ -e $XECPUS ]] && echo "$XECPUS: $(cat $XECPUS)" 562 [[ -e $PRS ]] && echo "$PRS: $(cat $PRS)" 563 [[ -e $PCPUS ]] && echo "$PCPUS: $(cat $PCPUS)" 564 [[ -e $ISCPUS ]] && echo "$ISCPUS: $(cat $ISCPUS)" 565 done 566} 567 568# 569# Check effective cpus 570# $1 - check string, format: <cgroup>:<cpu-list>[,<cgroup>:<cpu-list>]* 571# 572check_effective_cpus() 573{ 574 CHK_STR=$1 575 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g") 576 do 577 set -- $(echo $CHK | sed -e "s/:/ /g") 578 CGRP=$1 579 CPUS=$2 580 if [[ $CGRP = X* ]] 581 then 582 CGRP=${CGRP#X} 583 FILE=cpuset.cpus.exclusive.effective 584 else 585 FILE=cpuset.cpus.effective 586 fi 587 [[ $CGRP = A2 ]] && CGRP=A1/A2 588 [[ $CGRP = A3 ]] && CGRP=A1/A2/A3 589 [[ -e $CGRP/$FILE ]] || return 1 590 [[ $CPUS = $(cat $CGRP/$FILE) ]] || return 1 591 done 592} 593 594# 595# Check cgroup states 596# $1 - check string, format: <cgroup>:<state>[,<cgroup>:<state>]* 597# 598check_cgroup_states() 599{ 600 CHK_STR=$1 601 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g") 602 do 603 set -- $(echo $CHK | sed -e "s/:/ /g") 604 CGRP=$1 605 CGRP_DIR=$CGRP 606 STATE=$2 607 FILE= 608 EVAL=$(expr substr $STATE 2 2) 609 [[ $CGRP = A2 ]] && CGRP_DIR=A1/A2 610 [[ $CGRP = A3 ]] && CGRP_DIR=A1/A2/A3 611 612 case $STATE in 613 P*) FILE=$CGRP_DIR/cpuset.cpus.partition 614 ;; 615 *) echo "Unknown state: $STATE!" 616 exit 1 617 ;; 618 esac 619 VAL=$(cat $FILE) 620 621 case "$VAL" in 622 member) VAL=0 623 ;; 624 root) VAL=1 625 ;; 626 isolated) 627 VAL=2 628 ;; 629 "root invalid"*) 630 VAL=-1 631 ;; 632 "isolated invalid"*) 633 VAL=-2 634 ;; 635 esac 636 [[ $EVAL != $VAL ]] && return 1 637 638 # 639 # For root partition, dump sched-domains info to console if 640 # verbose mode set for manual comparison with sched debug info. 641 # 642 [[ $VAL -eq 1 && $VERBOSE -gt 0 ]] && { 643 DOMS=$(cat $CGRP_DIR/cpuset.cpus.effective) 644 [[ -n "$DOMS" ]] && 645 echo " [$CGRP] sched-domain: $DOMS" > $CONSOLE 646 } 647 done 648 return 0 649} 650 651# 652# Get isolated (including offline) CPUs by looking at 653# /sys/kernel/debug/sched/domains and cpuset.cpus.isolated control file, 654# if available, and compare that with the expected value. 655# 656# Note that isolated CPUs from the sched/domains context include offline 657# CPUs as well as CPUs in non-isolated 1-CPU partition. Those CPUs may 658# not be included in the cpuset.cpus.isolated control file which contains 659# only CPUs in isolated partitions as well as those that are isolated at 660# boot time. 661# 662# $1 - expected isolated cpu list(s) <isolcpus1>{,<isolcpus2>} 663# <isolcpus1> - expected sched/domains value 664# <isolcpus2> - cpuset.cpus.isolated value = <isolcpus1> if not defined 665# 666check_isolcpus() 667{ 668 EXPECT_VAL=$1 669 ISOLCPUS= 670 LASTISOLCPU= 671 SCHED_DOMAINS=/sys/kernel/debug/sched/domains 672 ISCPUS=${CGROUP2}/cpuset.cpus.isolated 673 if [[ $EXPECT_VAL = . ]] 674 then 675 EXPECT_VAL= 676 EXPECT_VAL2= 677 elif [[ $(expr $EXPECT_VAL : ".*,.*") > 0 ]] 678 then 679 set -- $(echo $EXPECT_VAL | sed -e "s/,/ /g") 680 EXPECT_VAL=$1 681 EXPECT_VAL2=$2 682 else 683 EXPECT_VAL2=$EXPECT_VAL 684 fi 685 686 # 687 # Appending pre-isolated CPUs 688 # Even though CPU #8 isn't used for testing, it can't be pre-isolated 689 # to make appending those CPUs easier. 690 # 691 [[ -n "$BOOT_ISOLCPUS" ]] && { 692 EXPECT_VAL=${EXPECT_VAL:+${EXPECT_VAL},}${BOOT_ISOLCPUS} 693 EXPECT_VAL2=${EXPECT_VAL2:+${EXPECT_VAL2},}${BOOT_ISOLCPUS} 694 } 695 696 # 697 # Check cpuset.cpus.isolated cpumask 698 # 699 [[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && { 700 # Take a 50ms pause and try again 701 pause 0.05 702 ISOLCPUS=$(cat $ISCPUS) 703 } 704 [[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && return 1 705 ISOLCPUS= 706 707 # 708 # Use the sched domain in debugfs to check isolated CPUs, if available 709 # 710 [[ -d $SCHED_DOMAINS ]] || return 0 711 712 for ((CPU=0; CPU < $NR_CPUS; CPU++)) 713 do 714 [[ -n "$(ls ${SCHED_DOMAINS}/cpu$CPU)" ]] && continue 715 716 if [[ -z "$LASTISOLCPU" ]] 717 then 718 ISOLCPUS=$CPU 719 LASTISOLCPU=$CPU 720 elif [[ "$LASTISOLCPU" -eq $((CPU - 1)) ]] 721 then 722 echo $ISOLCPUS | grep -q "\<$LASTISOLCPU\$" 723 if [[ $? -eq 0 ]] 724 then 725 ISOLCPUS=${ISOLCPUS}- 726 fi 727 LASTISOLCPU=$CPU 728 else 729 if [[ $ISOLCPUS = *- ]] 730 then 731 ISOLCPUS=${ISOLCPUS}$LASTISOLCPU 732 fi 733 ISOLCPUS=${ISOLCPUS},$CPU 734 LASTISOLCPU=$CPU 735 fi 736 done 737 [[ "$ISOLCPUS" = *- ]] && ISOLCPUS=${ISOLCPUS}$LASTISOLCPU 738 739 [[ "$EXPECT_VAL" = "$ISOLCPUS" ]] 740} 741 742test_fail() 743{ 744 TESTNUM=$1 745 TESTTYPE=$2 746 ADDINFO=$3 747 echo "Test $TEST[$TESTNUM] failed $TESTTYPE check!" 748 [[ -n "$ADDINFO" ]] && echo "*** $ADDINFO ***" 749 eval echo \${$TEST[$I]} 750 echo 751 dump_states 752 exit 1 753} 754 755# 756# Check to see if there are unexpected isolated CPUs left beyond the boot 757# time isolated ones. 758# 759null_isolcpus_check() 760{ 761 [[ $VERBOSE -gt 0 ]] || return 0 762 # Retry a few times before printing error 763 RETRY=0 764 while [[ $RETRY -lt 8 ]] 765 do 766 pause 0.02 767 check_isolcpus "." 768 [[ $? -eq 0 ]] && return 0 769 ((RETRY++)) 770 done 771 echo "Unexpected isolated CPUs: $ISOLCPUS" 772 dump_states 773 exit 1 774} 775 776# 777# Run cpuset state transition test 778# $1 - test matrix name 779# 780# This test is somewhat fragile as delays (sleep x) are added in various 781# places to make sure state changes are fully propagated before the next 782# action. These delays may need to be adjusted if running in a slower machine. 783# 784run_state_test() 785{ 786 TEST=$1 787 CONTROLLER=cpuset 788 I=0 789 eval CNT="\${#$TEST[@]}" 790 791 reset_cgroup_states 792 console_msg "Running state transition test ..." 793 794 while [[ $I -lt $CNT ]] 795 do 796 echo "Running test $I ..." > $CONSOLE 797 [[ $VERBOSE -gt 1 ]] && { 798 echo "" 799 eval echo \${$TEST[$I]} 800 } 801 eval set -- "\${$TEST[$I]}" 802 OLD_A1=$1 803 OLD_A2=$2 804 OLD_A3=$3 805 OLD_B1=$4 806 NEW_A1=$5 807 NEW_A2=$6 808 NEW_A3=$7 809 NEW_B1=$8 810 RESULT=$9 811 ECPUS=${10} 812 STATES=${11} 813 ICPUS=${12} 814 815 set_ctrl_state_noerr B1 $OLD_B1 816 set_ctrl_state_noerr A1 $OLD_A1 817 set_ctrl_state_noerr A1/A2 $OLD_A2 818 set_ctrl_state_noerr A1/A2/A3 $OLD_A3 819 RETVAL=0 820 set_ctrl_state A1 $NEW_A1; ((RETVAL += $?)) 821 set_ctrl_state A1/A2 $NEW_A2; ((RETVAL += $?)) 822 set_ctrl_state A1/A2/A3 $NEW_A3; ((RETVAL += $?)) 823 set_ctrl_state B1 $NEW_B1; ((RETVAL += $?)) 824 825 [[ $RETVAL -ne $RESULT ]] && test_fail $I result 826 827 [[ -n "$ECPUS" && "$ECPUS" != . ]] && { 828 check_effective_cpus $ECPUS 829 [[ $? -ne 0 ]] && test_fail $I "effective CPU" 830 } 831 832 [[ -n "$STATES" && "$STATES" != . ]] && { 833 check_cgroup_states $STATES 834 [[ $? -ne 0 ]] && test_fail $I states 835 } 836 837 # Compare the expected isolated CPUs with the actual ones, 838 # if available 839 [[ -n "$ICPUS" ]] && { 840 check_isolcpus $ICPUS 841 [[ $? -ne 0 ]] && { 842 [[ -n "$BOOT_ISOLCPUS" ]] && ICPUS=${ICPUS},${BOOT_ISOLCPUS} 843 test_fail $I "isolated CPU" \ 844 "Expect $ICPUS, get $ISOLCPUS instead" 845 } 846 } 847 reset_cgroup_states 848 # 849 # Check to see if effective cpu list changes 850 # 851 NEWLIST=$(cat cpuset.cpus.effective) 852 RETRY=0 853 while [[ $NEWLIST != $CPULIST && $RETRY -lt 8 ]] 854 do 855 # Wait a bit longer & recheck a few times 856 pause 0.02 857 ((RETRY++)) 858 NEWLIST=$(cat cpuset.cpus.effective) 859 done 860 [[ $NEWLIST != $CPULIST ]] && { 861 echo "Effective cpus changed to $NEWLIST after test $I!" 862 exit 1 863 } 864 null_isolcpus_check 865 [[ $VERBOSE -gt 0 ]] && echo "Test $I done." 866 ((I++)) 867 done 868 echo "All $I tests of $TEST PASSED." 869} 870 871# 872# Testing the new "isolated" partition root type 873# 874test_isolated() 875{ 876 cd $CGROUP2/test 877 echo 2-3 > cpuset.cpus 878 TYPE=$(cat cpuset.cpus.partition) 879 [[ $TYPE = member ]] || echo member > cpuset.cpus.partition 880 881 console_msg "Change from member to root" 882 test_partition root 883 884 console_msg "Change from root to isolated" 885 test_partition isolated 886 887 console_msg "Change from isolated to member" 888 test_partition member 889 890 console_msg "Change from member to isolated" 891 test_partition isolated 892 893 console_msg "Change from isolated to root" 894 test_partition root 895 896 console_msg "Change from root to member" 897 test_partition member 898 899 # 900 # Testing partition root with no cpu 901 # 902 console_msg "Distribute all cpus to child partition" 903 echo +cpuset > cgroup.subtree_control 904 test_partition root 905 906 mkdir A1 907 cd A1 908 echo 2-3 > cpuset.cpus 909 test_partition root 910 test_effective_cpus 2-3 911 cd .. 912 test_effective_cpus "" 913 914 console_msg "Moving task to partition test" 915 test_add_proc "No space left" 916 cd A1 917 test_add_proc "" 918 cd .. 919 920 console_msg "Shrink and expand child partition" 921 cd A1 922 echo 2 > cpuset.cpus 923 cd .. 924 test_effective_cpus 3 925 cd A1 926 echo 2-3 > cpuset.cpus 927 cd .. 928 test_effective_cpus "" 929 930 # Cleaning up 931 console_msg "Cleaning up" 932 echo $$ > $CGROUP2/cgroup.procs 933 [[ -d A1 ]] && rmdir A1 934 null_isolcpus_check 935} 936 937# 938# Wait for inotify event for the given file and read it 939# $1: cgroup file to wait for 940# $2: file to store the read result 941# 942wait_inotify() 943{ 944 CGROUP_FILE=$1 945 OUTPUT_FILE=$2 946 947 $WAIT_INOTIFY $CGROUP_FILE 948 cat $CGROUP_FILE > $OUTPUT_FILE 949} 950 951# 952# Test if inotify events are properly generated when going into and out of 953# invalid partition state. 954# 955test_inotify() 956{ 957 ERR=0 958 PRS=/tmp/.prs_$$ 959 cd $CGROUP2/test 960 [[ -f $WAIT_INOTIFY ]] || { 961 echo "wait_inotify not found, inotify test SKIPPED." 962 return 963 } 964 965 pause 0.01 966 echo 1 > cpuset.cpus 967 echo 0 > cgroup.procs 968 echo root > cpuset.cpus.partition 969 pause 0.01 970 rm -f $PRS 971 wait_inotify $PWD/cpuset.cpus.partition $PRS & 972 pause 0.01 973 set_ctrl_state . "O1=0" 974 pause 0.01 975 check_cgroup_states ".:P-1" 976 if [[ $? -ne 0 ]] 977 then 978 echo "FAILED: Inotify test - partition not invalid" 979 ERR=1 980 elif [[ ! -f $PRS ]] 981 then 982 echo "FAILED: Inotify test - event not generated" 983 ERR=1 984 kill %1 985 elif [[ $(cat $PRS) != "root invalid"* ]] 986 then 987 echo "FAILED: Inotify test - incorrect state" 988 cat $PRS 989 ERR=1 990 fi 991 online_cpus 992 echo member > cpuset.cpus.partition 993 echo 0 > ../cgroup.procs 994 if [[ $ERR -ne 0 ]] 995 then 996 exit 1 997 else 998 echo "Inotify test PASSED" 999 fi 1000} 1001 1002trap cleanup 0 2 3 6 1003run_state_test TEST_MATRIX 1004test_isolated 1005test_inotify 1006echo "All tests PASSED." 1007