2 # SPDX-License-Identifier: GPL-2.0
4 # Copyright (c) 2019 Facebook
6 # This program is free software; you can redistribute it and/or
7 # modify it under the terms of version 2 of the GNU General Public
8 # License as published by the Free Software Foundation.
11 echo "Script for testing HBM (Host Bandwidth Manager) framework."
12 echo "It creates a cgroup to use for testing and load a BPF program to limit"
13 echo "egress or ingress bandwidht. It then uses iperf3 or netperf to create"
14 echo "loads. The output is the goodput in Mbps (unless -D was used)."
16 echo "USAGE: $name [out] [-b=<prog>|--bpf=<prog>] [-c=<cc>|--cc=<cc>] [-D]"
17 echo " [-d=<delay>|--delay=<delay>] [--debug] [-E]"
18 echo " [-f=<#flows>|--flows=<#flows>] [-h] [-i=<id>|--id=<id >]"
19 echo " [-l] [-N] [-p=<port>|--port=<port>] [-P]"
20 echo " [-q=<qdisc>] [-R] [-s=<server>|--server=<server]"
21 echo " [-S|--stats] -t=<time>|--time=<time>] [-w] [cubic|dctcp]"
23 echo " out egress (default)"
24 echo " -b or --bpf BPF program filename to load and attach."
25 echo " Default is hbm_out_kern.o for egress,"
26 echo " -c or -cc TCP congestion control (cubic or dctcp)"
27 echo " --debug print BPF trace buffer"
28 echo " -d or --delay add a delay in ms using netem"
29 echo " -D In addition to the goodput in Mbps, it also outputs"
30 echo " other detailed information. This information is"
31 echo " test dependent (i.e. iperf3 or netperf)."
32 echo " -E enable ECN (not required for dctcp)"
33 echo " -f or --flows number of concurrent flows (default=1)"
34 echo " -i or --id cgroup id (an integer, default is 1)"
35 echo " -N use netperf instead of iperf3"
36 echo " -l do not limit flows using loopback"
38 echo " -p or --port iperf3 port (default is 5201)"
39 echo " -P use an iperf3 instance for each flow"
40 echo " -q use the specified qdisc"
41 echo " -r or --rate rate in Mbps (default 1s 1Gbps)"
42 echo " -R Use TCP_RR for netperf. 1st flow has req"
43 echo " size of 10KB, rest of 1MB. Reply in all"
44 echo " cases is 1 byte."
45 echo " More detailed output for each flow can be found"
46 echo " in the files netperf.<cg>.<flow>, where <cg> is the"
47 echo " cgroup id as specified with the -i flag, and <flow>"
48 echo " is the flow id starting at 1 and increasing by 1 for"
49 echo " flow (as specified by -f)."
50 echo " -s or --server hostname of netperf server. Used to create netperf"
51 echo " test traffic between to hosts (default is within host)"
52 echo " netserver must be running on the host."
53 echo " -S or --stats whether to update hbm stats (default is yes)."
54 echo " -t or --time duration of iperf3 in seconds (default=5)"
55 echo " -w Work conserving flag. cgroup can increase its"
56 echo " bandwidth beyond the rate limit specified"
57 echo " while there is available bandwidth. Current"
58 echo " implementation assumes there is only one NIC"
59 echo " (eth0), but can be extended to support multiple"
61 echo " cubic or dctcp specify which TCP CC to use"
92 function start_hbm () {
94 echo "./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog" > hbm.out
96 ./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog >> hbm.out 2>&1 &
103 # Support for upcomming ingress rate limiting
104 #in) # support for upcoming ingress rate limiting
131 # Support for upcomming fq Early Departure Time egress rate limiting
133 # prog="hbm_out_edt_kern.o"
183 echo "Unknown arg:$i"
192 if [ $debug_flag -eq 1 ] ; then
200 cg_base_dir=/sys/fs/cgroup
201 cg_dir="$cg_base_dir/cgroup-test-work-dir/hbm$id"
203 echo $$ >> $cg_dir/cgroup.procs
208 rm -f hbm.[0-9]*.$dir_name
209 if [ $ecn -ne 0 ] ; then
210 sysctl -w -q -n net.ipv4.tcp_ecn=1
213 if [ $use_netperf -eq 0 ] ; then
214 cur_cc=`sysctl -n net.ipv4.tcp_congestion_control`
215 if [ "$cc" != "x" ] ; then
216 sysctl -w -q -n net.ipv4.tcp_congestion_control=$cc
220 if [ "$netem" -ne "0" ] ; then
221 if [ "$qdisc" != "" ] ; then
222 echo "WARNING: Ignoring -q options because -d option used"
224 tc qdisc del dev lo root > /dev/null 2>&1
225 tc qdisc add dev lo root netem delay $netem\ms > /dev/null 2>&1
226 elif [ "$qdisc" != "" ] ; then
227 tc qdisc del dev lo root > /dev/null 2>&1
228 tc qdisc add dev lo root $qdisc > /dev/null 2>&1
234 if [ $use_netperf -ne 0 ] ; then
235 if [ "$server" != "" ] ; then
240 ( ping6 -i 0.2 -c $m $hn > ping.out 2>&1 ) &
242 if [ $use_netperf -ne 0 ] ; then
243 begNetserverPid=`ps ax | grep netserver | grep --invert-match "grep" | \
245 if [ "$begNetserverPid" == "" ] ; then
246 if [ "$server" == "" ] ; then
247 ( ./netserver > /dev/null 2>&1) &
252 if [ "$server" == "" ] ; then
257 if [ "$cc" == "x" ] ; then
263 while [ $flow_cnt -le $flows ] ; do
264 if [ $rr -ne 0 ] ; then
266 if [ $flow_cnt -eq 1 ] ; then
269 if [ "$dir" == "-i" ] ; then
273 ( ./netperf -H $np_server -l $dur -f m -j -t TCP_RR -- -r $reqSize,$replySize $np_cc -k P50_lATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,REMOTE_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,LOCAL_RECV_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
275 if [ "$dir" == "-i" ] ; then
276 ( ./netperf -H $np_server -l $dur -f m -j -t TCP_RR -- -r 1,10M $np_cc -k P50_LATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,REMOTE_TRANSPORT_RETRANS,REMOTE_SEND_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
278 ( ./netperf -H $np_server -l $dur -f m -j -t TCP_STREAM -- $np_cc -k P50_lATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
281 flow_cnt=$[flow_cnt+1]
284 # sleep for duration of test (plus some buffer)
288 # force graceful termination of netperf
296 if [ $details -ne 0 ] ; then
298 echo "Details for HBM in cgroup $id"
299 if [ $do_stats -eq 1 ] ; then
300 if [ -e hbm.$id.$dir_name ] ; then
301 cat hbm.$id.$dir_name
305 while [ $flow_cnt -le $flows ] ; do
306 if [ "$dir" == "-i" ] ; then
307 r=`cat netperf.$id.$flow_cnt | grep -o "REMOTE_SEND_THROUGHPUT=[0-9]*" | grep -o "[0-9]*"`
309 r=`cat netperf.$id.$flow_cnt | grep -o "LOCAL_SEND_THROUGHPUT=[0-9]*" | grep -o "[0-9]*"`
311 echo "rate for flow $flow_cnt: $r"
313 if [ $details -ne 0 ] ; then
315 echo "Details for cgroup $id, flow $flow_cnt"
316 cat netperf.$id.$flow_cnt
318 flow_cnt=$[flow_cnt+1]
320 if [ $details -ne 0 ] ; then
322 delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`
323 echo "PING AVG DELAY:$delay"
324 echo "AGGREGATE_GOODPUT:$rate"
328 elif [ $multi_iperf -eq 0 ] ; then
329 (iperf3 -s -p $port -1 > /dev/null 2>&1) &
331 iperf3 -c $host -p $port -i 0 -P $flows -f m -t $dur > iperf.$id
332 rates=`grep receiver iperf.$id | grep -o "[0-9.]* Mbits" | grep -o "^[0-9]*"`
333 rate=`echo $rates | grep -o "[0-9]*$"`
335 if [ $details -ne 0 ] ; then
337 echo "Details for HBM in cgroup $id"
338 if [ $do_stats -eq 1 ] ; then
339 if [ -e hbm.$id.$dir_name ] ; then
340 cat hbm.$id.$dir_name
343 delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`
344 echo "PING AVG DELAY:$delay"
345 echo "AGGREGATE_GOODPUT:$rate"
351 while [ $flow_cnt -le $flows ] ; do
352 (iperf3 -s -p $port -1 > /dev/null 2>&1) &
353 ( iperf3 -c $host -p $port -i 0 -P 1 -f m -t $dur | grep receiver | grep -o "[0-9.]* Mbits" | grep -o "^[0-9]*" | grep -o "[0-9]*$" > iperf3.$id.$flow_cnt ) &
355 flow_cnt=$[flow_cnt+1]
361 if [ $details -ne 0 ] ; then
363 echo "Details for HBM in cgroup $id"
364 if [ $do_stats -eq 1 ] ; then
365 if [ -e hbm.$id.$dir_name ] ; then
366 cat hbm.$id.$dir_name
371 while [ $flow_cnt -le $flows ] ; do
372 r=`cat iperf3.$id.$flow_cnt`
373 # echo "rate for flow $flow_cnt: $r"
374 if [ $details -ne 0 ] ; then
375 echo "Rate for cgroup $id, flow $flow_cnt LOCAL_SEND_THROUGHPUT=$r"
378 flow_cnt=$[flow_cnt+1]
380 if [ $details -ne 0 ] ; then
381 delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`
382 echo "PING AVG DELAY:$delay"
383 echo "AGGREGATE_GOODPUT:$rate"
389 if [ $use_netperf -eq 0 ] ; then
390 sysctl -w -q -n net.ipv4.tcp_congestion_control=$cur_cc
392 if [ $ecn -ne 0 ] ; then
393 sysctl -w -q -n net.ipv4.tcp_ecn=0
395 if [ "$netem" -ne "0" ] ; then
396 tc qdisc del dev lo root > /dev/null 2>&1
401 hbmPid=`ps ax | grep "hbm " | grep --invert-match "grep" | awk '{ print $1 }'`
402 if [ "$hbmPid" == "$hbm_pid" ] ; then
408 # Detach any BPF programs that may have lingered
409 ttx=`bpftool cgroup tree | grep hbm`
412 if [ "${x:0:36}" == "/sys/fs/cgroup/cgroup-test-work-dir/" ] ; then
415 if [ $v -eq 0 ] ; then
418 if [ $v -eq 1 ] ; then
419 type=$x ; bpftool cgroup detach $cg $type id $id
426 if [ $use_netperf -ne 0 ] ; then
427 if [ "$server" == "" ] ; then
428 if [ "$begNetserverPid" == "" ] ; then
429 netserverPid=`ps ax | grep netserver | grep --invert-match "grep" | awk '{ print $1 }'`
430 if [ "$netserverPid" != "" ] ; then