2 # SPDX-License-Identifier: GPL-2.0
4 # Test for cpuset v2 partition root state (PRS)
6 # The sched verbose flag is set, if available, so that the console log
7 # can be examined for the correct setting of scheduling domain.
16 [[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!"
19 # Get wait_inotify location
20 WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify
22 # Find cgroup v2 mount point
23 CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}')
24 [[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!"
26 CPUS=$(lscpu | grep "^CPU(s):" | sed -e "s/.*:[[:space:]]*//")
27 [[ $CPUS -lt 8 ]] && skip_test "Test needs at least 8 cpus available!"
29 # Set verbose flag and delay factor
38 # Enable sched/verbose can slow thing down
39 [[ $DELAY_FACTOR -eq 1 ]] &&
47 *) echo "Usage: $PROG [-v] [-d <delay-factor>"
54 # Set sched verbose flag if available when "-v" option is specified
55 if [[ -n "$VERBOSE" && -d /sys/kernel/debug/sched ]]
57 # Used to restore the original setting during cleanup
58 SCHED_DEBUG=$(cat /sys/kernel/debug/sched/verbose)
59 echo Y > /sys/kernel/debug/sched/verbose
63 echo +cpuset > cgroup.subtree_control
64 [[ -d test ]] || mkdir test
70 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1
72 rmdir test > /dev/null 2>&1
73 [[ -n "$SCHED_DEBUG" ]] &&
74 echo "$SCHED_DEBUG" > /sys/kernel/debug/sched/verbose
82 while [[ $LOOP -lt $DELAY_FACTOR ]]
94 echo "" > /dev/console
95 echo "$MSG" > /dev/console
102 echo $EXPECTED_VAL > cpuset.cpus.partition
103 [[ $? -eq 0 ]] || exit 1
104 ACTUAL_VAL=$(cat cpuset.cpus.partition)
105 [[ $ACTUAL_VAL != $EXPECTED_VAL ]] && {
106 echo "cpuset.cpus.partition: expect $EXPECTED_VAL, found $EXPECTED_VAL"
112 test_effective_cpus()
115 ACTUAL_VAL=$(cat cpuset.cpus.effective)
116 [[ "$ACTUAL_VAL" != "$EXPECTED_VAL" ]] && {
117 echo "cpuset.cpus.effective: expect '$EXPECTED_VAL', found '$EXPECTED_VAL'"
123 # Adding current process to cgroup.procs as a test
127 ERRMSG=$((echo $$ > cgroup.procs) |& cat)
128 echo $ERRMSG | grep -q "$OUTSTR"
130 echo "cgroup.procs: expect '$OUTSTR', got '$ERRMSG'"
134 echo $$ > $CGROUP2/cgroup.procs # Move out the task
138 # Testing the new "isolated" partition root type
142 echo 2-3 > cpuset.cpus
143 TYPE=$(cat cpuset.cpus.partition)
144 [[ $TYPE = member ]] || echo member > cpuset.cpus.partition
146 console_msg "Change from member to root"
149 console_msg "Change from root to isolated"
150 test_partition isolated
152 console_msg "Change from isolated to member"
153 test_partition member
155 console_msg "Change from member to isolated"
156 test_partition isolated
158 console_msg "Change from isolated to root"
161 console_msg "Change from root to member"
162 test_partition member
165 # Testing partition root with no cpu
167 console_msg "Distribute all cpus to child partition"
168 echo +cpuset > cgroup.subtree_control
173 echo 2-3 > cpuset.cpus
175 test_effective_cpus 2-3
177 test_effective_cpus ""
179 console_msg "Moving task to partition test"
180 test_add_proc "No space left"
185 console_msg "Shrink and expand child partition"
189 test_effective_cpus 3
191 echo 2-3 > cpuset.cpus
193 test_effective_cpus ""
196 console_msg "Cleaning up"
197 echo $$ > $CGROUP2/cgroup.procs
198 [[ -d A1 ]] && rmdir A1
202 # Cpuset controller state transition test matrix.
204 # Cgroup test hierarchy
206 # test -- A1 -- A2 -- A3
209 # P<v> = set cpus.partition (0:member, 1:root, 2:isolated, -1:root invalid)
210 # C<l> = add cpu-list
211 # S<p> = use prefix in subtree_control
212 # T = put a task into cgroup
213 # O<c>-<v> = Write <v> to CPU online file of <c>
215 SETUP_A123_PARTITIONS="C1-3:P1:S+ C2-3:P1:S+ C3:P1"
217 # test old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate
218 # ---- ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------
219 " S+ C0-1 . . C2-3 S+ C4-5 . . 0 A2:0-1"
220 " S+ C0-1 . . C2-3 P1 . . . 0 "
221 " S+ C0-1 . . C2-3 P1:S+ C0-1:P1 . . 0 "
222 " S+ C0-1 . . C2-3 P1:S+ C1:P1 . . 0 "
223 " S+ C0-1:S+ . . C2-3 . . . P1 0 "
224 " S+ C0-1:P1 . . C2-3 S+ C1 . . 0 "
225 " S+ C0-1:P1 . . C2-3 S+ C1:P1 . . 0 "
226 " S+ C0-1:P1 . . C2-3 S+ C1:P1 . P1 0 "
227 " S+ C0-1:P1 . . C2-3 C4-5 . . . 0 A1:4-5"
228 " S+ C0-1:P1 . . C2-3 S+:C4-5 . . . 0 A1:4-5"
229 " S+ C0-1 . . C2-3:P1 . . . C2 0 "
230 " S+ C0-1 . . C2-3:P1 . . . C4-5 0 B1:4-5"
231 " S+ C0-3:P1:S+ C2-3:P1 . . . . . . 0 A1:0-1,A2:2-3"
232 " S+ C0-3:P1:S+ C2-3:P1 . . C1-3 . . . 0 A1:1,A2:2-3"
233 " S+ C2-3:P1:S+ C3:P1 . . C3 . . . 0 A1:,A2:3 A1:P1,A2:P1"
234 " S+ C2-3:P1:S+ C3:P1 . . C3 P0 . . 0 A1:3,A2:3 A1:P1,A2:P0"
235 " S+ C2-3:P1:S+ C2:P1 . . C2-4 . . . 0 A1:3-4,A2:2"
236 " S+ C2-3:P1:S+ C3:P1 . . C3 . . C0-2 0 A1:,B1:0-2 A1:P1,A2:P1"
237 " S+ $SETUP_A123_PARTITIONS . C2-3 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
239 # CPU offlining cases:
240 " S+ C0-1 . . C2-3 S+ C4-5 . O2-0 0 A1:0-1,B1:3"
241 " S+ C0-3:P1:S+ C2-3:P1 . . O2-0 . . . 0 A1:0-1,A2:3"
242 " S+ C0-3:P1:S+ C2-3:P1 . . O2-0 O2-1 . . 0 A1:0-1,A2:2-3"
243 " S+ C0-3:P1:S+ C2-3:P1 . . O1-0 . . . 0 A1:0,A2:2-3"
244 " S+ C0-3:P1:S+ C2-3:P1 . . O1-0 O1-1 . . 0 A1:0-1,A2:2-3"
245 " S+ C2-3:P1:S+ C3:P1 . . O3-0 O3-1 . . 0 A1:2,A2:3 A1:P1,A2:P1"
246 " S+ C2-3:P1:S+ C3:P2 . . O3-0 O3-1 . . 0 A1:2,A2:3 A1:P1,A2:P2"
247 " S+ C2-3:P1:S+ C3:P1 . . O2-0 O2-1 . . 0 A1:2,A2:3 A1:P1,A2:P1"
248 " S+ C2-3:P1:S+ C3:P2 . . O2-0 O2-1 . . 0 A1:2,A2:3 A1:P1,A2:P2"
249 " S+ C2-3:P1:S+ C3:P1 . . O2-0 . . . 0 A1:,A2:3 A1:P1,A2:P1"
250 " S+ C2-3:P1:S+ C3:P1 . . O3-0 . . . 0 A1:2,A2: A1:P1,A2:P1"
251 " S+ C2-3:P1:S+ C3:P1 . . T:O2-0 . . . 0 A1:3,A2:3 A1:P1,A2:P-1"
252 " S+ C2-3:P1:S+ C3:P1 . . . T:O3-0 . . 0 A1:2,A2:2 A1:P1,A2:P-1"
253 " S+ $SETUP_A123_PARTITIONS . O1-0 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
254 " S+ $SETUP_A123_PARTITIONS . O2-0 . . . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1"
255 " S+ $SETUP_A123_PARTITIONS . O3-0 . . . 0 A1:1,A2:2,A3: A1:P1,A2:P1,A3:P1"
256 " S+ $SETUP_A123_PARTITIONS . T:O1-0 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
257 " S+ $SETUP_A123_PARTITIONS . . T:O2-0 . . 0 A1:1,A2:3,A3:3 A1:P1,A2:P1,A3:P-1"
258 " S+ $SETUP_A123_PARTITIONS . . . T:O3-0 . 0 A1:1,A2:2,A3:2 A1:P1,A2:P1,A3:P-1"
259 " S+ $SETUP_A123_PARTITIONS . T:O1-0 O1-1 . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
260 " S+ $SETUP_A123_PARTITIONS . . T:O2-0 O2-1 . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
261 " S+ $SETUP_A123_PARTITIONS . . . T:O3-0 O3-1 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
262 " S+ $SETUP_A123_PARTITIONS . T:O1-0 O2-0 O1-1 . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1"
263 " S+ $SETUP_A123_PARTITIONS . T:O1-0 O2-0 O2-1 . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
265 # test old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate
266 # ---- ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------
268 # Incorrect change to cpuset.cpus invalidates partition root
270 # Adding CPUs to partition root that are not in parent's
271 # cpuset.cpus is allowed, but those extra CPUs are ignored.
272 " S+ C2-3:P1:S+ C3:P1 . . . C2-4 . . 0 A1:,A2:2-3 A1:P1,A2:P1"
274 # Taking away all CPUs from parent or itself if there are tasks
275 # will make the partition invalid.
276 " S+ C2-3:P1:S+ C3:P1 . . T C2-3 . . 0 A1:2-3,A2:2-3 A1:P1,A2:P-1"
277 " S+ C3:P1:S+ C3 . . T P1 . . 0 A1:3,A2:3 A1:P1,A2:P-1"
278 " S+ $SETUP_A123_PARTITIONS . T:C2-3 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
279 " S+ $SETUP_A123_PARTITIONS . T:C2-3:C1-3 . . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
281 # Changing a partition root to member makes child partitions invalid
282 " S+ C2-3:P1:S+ C3:P1 . . P0 . . . 0 A1:2-3,A2:3 A1:P0,A2:P-1"
283 " S+ $SETUP_A123_PARTITIONS . C2-3 P0 . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P0,A3:P-1"
285 # cpuset.cpus can contains cpus not in parent's cpuset.cpus as long
287 " S+ C2-3:P1:S+ . . . . C3-4:P1 . . 0 A1:2,A2:3 A1:P1,A2:P1"
289 # Deletion of CPUs distributed to child cgroup is allowed.
290 " S+ C0-1:P1:S+ C1 . C2-3 C4-5 . . . 0 A1:4-5,A2:4-5"
292 # To become a valid partition root, cpuset.cpus must overlap parent's
294 " S+ C0-1:P1 . . C2-3 S+ C4-5:P1 . . 0 A1:0-1,A2:0-1 A1:P1,A2:P-1"
296 # Enabling partition with child cpusets is allowed
297 " S+ C0-1:S+ C1 . C2-3 P1 . . . 0 A1:0-1,A2:1 A1:P1"
299 # A partition root with non-partition root parent is invalid, but it
300 # can be made valid if its parent becomes a partition root too.
301 " S+ C0-1:S+ C1 . C2-3 . P2 . . 0 A1:0-1,A2:1 A1:P0,A2:P-2"
302 " S+ C0-1:S+ C1:P2 . C2-3 P1 . . . 0 A1:0,A2:1 A1:P1,A2:P2"
304 # A non-exclusive cpuset.cpus change will invalidate partition and its siblings
305 " S+ C0-1:P1 . . C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P0"
306 " S+ C0-1:P1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P-1"
307 " S+ C0-1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P0,B1:P-1"
309 # test old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate
310 # ---- ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------
313 # A task cannot be added to a partition with no cpu
314 " S+ C2-3:P1:S+ C3:P1 . . O2-0:T . . . 1 A1:,A2:3 A1:P1,A2:P1"
318 # Write to the cpu online file
319 # $1 - <c>-<v> where <c> = cpu number, <v> value to be written
325 CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online
328 OFFLINE_CPUS="$OFFLINE_CPUS $CPU"
330 [[ -n "$OFFLINE_CPUS" ]] && {
331 OFFLINE_CPUS=$(echo $CPU $CPU $OFFLINE_CPUS | fmt -1 |\
340 # Set controller state
341 # $1 - cgroup directory
345 # The presence of ":" in state means transition from one to the next.
352 SHOWERR=${3}${VERBOSE}
353 CTRL=${CTRL:=$CONTROLLER}
355 REDIRECT="2> $TMPMSG"
356 [[ -z "$STATE" || "$STATE" = '.' ]] && return 0
359 for CMD in $(echo $STATE | sed -e "s/:/ /g")
361 TFILE=$CGRP/cgroup.procs
362 SFILE=$CGRP/cgroup.subtree_control
363 PFILE=$CGRP/cpuset.cpus.partition
364 CFILE=$CGRP/cpuset.cpus
365 S=$(expr substr $CMD 1 1)
369 COMM="echo ${PREFIX}${CTRL} > $SFILE"
374 COMM="echo $CPUS > $CFILE"
387 echo "Invalid partition state - $VAL"
391 COMM="echo $VAL > $PFILE"
396 write_cpu_online $VAL
399 COMM="echo 0 > $TFILE"
403 [[ $RET -ne 0 ]] && {
404 [[ -n "$SHOWERR" ]] && {
416 set_ctrl_state_noerr()
420 [[ -d $CGRP ]] || mkdir $CGRP
421 set_ctrl_state $CGRP $STATE 1
423 echo "ERROR: Failed to set $2 to cgroup $1!"
430 [[ -n "OFFLINE_CPUS" ]] && {
431 for C in $OFFLINE_CPUS
433 write_cpu_online ${C}-1
439 # Return 1 if the list of effective cpus isn't the same as the initial list.
441 reset_cgroup_states()
443 echo 0 > $CGROUP2/cgroup.procs
445 rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1
452 for DIR in A1 A1/A2 A1/A2/A3 B1
454 ECPUS=$DIR/cpuset.cpus.effective
455 PRS=$DIR/cpuset.cpus.partition
456 [[ -e $ECPUS ]] && echo "$ECPUS: $(cat $ECPUS)"
457 [[ -e $PRS ]] && echo "$PRS: $(cat $PRS)"
462 # Check effective cpus
463 # $1 - check string, format: <cgroup>:<cpu-list>[,<cgroup>:<cpu-list>]*
465 check_effective_cpus()
468 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g")
470 set -- $(echo $CHK | sed -e "s/:/ /g")
473 [[ $CGRP = A2 ]] && CGRP=A1/A2
474 [[ $CGRP = A3 ]] && CGRP=A1/A2/A3
475 FILE=$CGRP/cpuset.cpus.effective
476 [[ -e $FILE ]] || return 1
477 [[ $CPUS = $(cat $FILE) ]] || return 1
482 # Check cgroup states
483 # $1 - check string, format: <cgroup>:<state>[,<cgroup>:<state>]*
485 check_cgroup_states()
488 for CHK in $(echo $CHK_STR | sed -e "s/,/ /g")
490 set -- $(echo $CHK | sed -e "s/:/ /g")
494 EVAL=$(expr substr $STATE 2 2)
495 [[ $CGRP = A2 ]] && CGRP=A1/A2
496 [[ $CGRP = A3 ]] && CGRP=A1/A2/A3
499 P*) FILE=$CGRP/cpuset.cpus.partition
501 *) echo "Unknown state: $STATE!"
522 [[ $EVAL != $VAL ]] && return 1
528 # Run cpuset state transition test
529 # $1 - test matrix name
531 # This test is somewhat fragile as delays (sleep x) are added in various
532 # places to make sure state changes are fully propagated before the next
533 # action. These delays may need to be adjusted if running in a slower machine.
541 eval CNT="\${#$TEST[@]}"
544 echo $CPULIST > cpuset.cpus
545 echo root > cpuset.cpus.partition
546 console_msg "Running state transition test ..."
548 while [[ $I -lt $CNT ]]
550 echo "Running test $I ..." > /dev/console
551 eval set -- "\${$TEST[$I]}"
565 set_ctrl_state_noerr . $ROOT
566 set_ctrl_state_noerr A1 $OLD_A1
567 set_ctrl_state_noerr A1/A2 $OLD_A2
568 set_ctrl_state_noerr A1/A2/A3 $OLD_A3
569 set_ctrl_state_noerr B1 $OLD_B1
571 set_ctrl_state A1 $NEW_A1; ((RETVAL += $?))
572 set_ctrl_state A1/A2 $NEW_A2; ((RETVAL += $?))
573 set_ctrl_state A1/A2/A3 $NEW_A3; ((RETVAL += $?))
574 set_ctrl_state B1 $NEW_B1; ((RETVAL += $?))
576 [[ $RETVAL -ne $RESULT ]] && {
577 echo "Test $TEST[$I] failed result check!"
578 eval echo \"\${$TEST[$I]}\"
583 [[ -n "$ECPUS" && "$ECPUS" != . ]] && {
584 check_effective_cpus $ECPUS
586 echo "Test $TEST[$I] failed effective CPU check!"
587 eval echo \"\${$TEST[$I]}\"
594 [[ -n "$STATES" ]] && {
595 check_cgroup_states $STATES
597 echo "FAILED: Test $TEST[$I] failed states check!"
598 eval echo \"\${$TEST[$I]}\"
607 # Check to see if effective cpu list changes
610 NEWLIST=$(cat cpuset.cpus.effective)
611 [[ $NEWLIST != $CPULIST ]] && {
612 echo "Effective cpus changed to $NEWLIST after test $I!"
615 [[ -n "$VERBOSE" ]] && echo "Test $I done."
618 echo "All $I tests of $TEST PASSED."
620 echo member > cpuset.cpus.partition
624 # Wait for inotify event for the given file and read it
625 # $1: cgroup file to wait for
626 # $2: file to store the read result
633 $WAIT_INOTIFY $CGROUP_FILE
634 cat $CGROUP_FILE > $OUTPUT_FILE
638 # Test if inotify events are properly generated when going into and out of
639 # invalid partition state.
645 [[ -f $WAIT_INOTIFY ]] || {
646 echo "wait_inotify not found, inotify test SKIPPED."
652 echo 0 > cgroup.procs
653 echo root > cpuset.cpus.partition
656 wait_inotify $PWD/cpuset.cpus.partition $PRS &
658 set_ctrl_state . "O1-0"
660 check_cgroup_states ".:P-1"
663 echo "FAILED: Inotify test - partition not invalid"
667 echo "FAILED: Inotify test - event not generated"
670 elif [[ $(cat $PRS) != "root invalid"* ]]
672 echo "FAILED: Inotify test - incorrect state"
677 echo member > cpuset.cpus.partition
678 echo 0 > ../cgroup.procs
683 echo "Inotify test PASSED"
688 run_state_test TEST_MATRIX
691 echo "All tests PASSED."