Figure 14
Scenario 1
Vanilla, not optimized, kernel-space
RT image, optimized, kernel-space
Vanilla, not optimized, user-space
RT image, optimized, user-space
Scenario 2
Vanilla, not optimized, kernel-space
RT image, optimized, kernel-space
Vanilla, not optimized, user-space
RT image, optimized, user-space
Steps to reproduce the taken measurements
OS images
These Images were used for the experiments
DuT (all three Kernel Variants)
Vanilla
Linux machine 5.10.0-10-amd64 #1 SMP Debian 5.10.84-1 (2021-12-08) x86_64 GNU/LinuxNoHz
Linux machine 5.10.84-nohz #1 SMP Tue Dec 21 05:07:04 CET 2021 x86_64 GNU/LinuxRT
Linux machine 5.10.0-10-rt-amd64 #1 SMP PREEMPT_RT Debian 5.10.84-1 (2021-12-08) x86_64 GNU/LinuxDebian Bullseye amd64 default privileged from 2022-08-09T19-36-56Linux machine 4.19.0-17-amd64 #1 SMP Debian 4.19.194-3 (2021-07-18) x86_64 GNU/LinuxLinux machine 5.10.0-8-amd64 #1 SMP Debian 5.10.46-4 (2021-08-03) x86_64 GNU/LinuxLinux machine 5.10.0-8-amd64 #1 SMP Debian 5.10.46-4 (2021-08-03) x86_64 GNU/LinuxSetup
LOADGEN=moongen
apt-get update
apt-get install -y libssl-dev
git clone --branch dpdk-19.05 --recurse-submodules --jobs 4 https://github.com/emmericp/MoonGen "$LOADGEN"
cd $LOADGEN/
/root/$LOADGEN/build.sh
/root/$LOADGEN/bind-interfaces.sh
/root/$LOADGEN/setup-hugetlbfs.sh
LOADGEN=moongen
apt-get update
apt-get install -y libssl-dev
git clone --branch dpdk-19.05 --recurse-submodules --jobs 4 https://github.com/emmericp/MoonGen "$LOADGEN"
cd $LOADGEN/
/root/$LOADGEN/build.sh
/root/$LOADGEN/bind-interfaces.sh
/root/$LOADGEN/setup-hugetlbfs.sh
Bootparameter
Optimized
mce=ignore_ce tsc=reliable idle=poll nohz=on audit=0 nosmt console=ttyS0,115200
apparmor=0 amd_iommu=off nohz_full=24,25,26,8,9 rcu_nocbs=24,25,26,8,9
skew_tick=1 irqaffinity=0 intel_pstate=disable nmi_watchdog=0 nosoftlockup
rcu_nocb_poll random.trust_cpu=on intel_idle.max_cstate=0
systemd.unified_cgroup_hierarchy=1
Not Optimized
console=ttyS0,115200 apparmor=0 amd_iommu=off systemd.unified_cgroup_hierarchy=1
Install general dependencies
for i in $(pgrep rcu[^c]) ; do taskset -pc 0 "$i" ; done
set -x
# Improving performance for the waiting time as we have no hard-drive to wait for
sysctl vm.dirty_ratio=5
sysctl vm.dirty_background_ratio=1
PACKAGES="pip lxc debootstrap python3-lxc ethtool"
DEBIAN_FRONTEND=noninteractive apt-get -y update --allow-releaseinfo-change
DEBIAN_FRONTEND=noninteractive apt-get -y install $PACKAGES
# disable apparmor. Apparmor otherwise blocks mounting the rootfs in the containers
aa-teardown
cd /root || exit
git clone https://github.com/tumi8/VirtualLXCBMC
cd virtuallxcbmc || exit
# install dependencies
python3 -m pip install -r requirements.txt
# compile project and move the executables to the right locations
python3 setup.py install
# starts the vbmc daemon in the background
vbmcd
Install libmoon Dependencies on Host (only for user-space experiments)
GIT_REPO='https://github.com/libmoon/libmoon.git'
GIT_BRANCH='dpdk-19.05'
FORWARDER=libmoon
cd /root
git clone --branch "$GIT_BRANCH" --recurse-submodules --jobs 4 "$GIT_REPO" "$FORWARDER"
cd $FORWARDER/
# build and load igb_uio
cd deps/dpdk-kmods/linux/igb_uio
make
modprobe uio
insmod igb_uio.ko
cd /root/$FORWARDER/deps/dpdk/usertools/
EXTERNAL_INTERFACES="enp33s0f1 enp100s0f1"
IFS=" "
for device_long in $(echo "$EXTERNAL_INTERFACES"); do
device=$(echo "$device_long" | awk -F '_' '{ print $2 }')
python dpdk-devbind.py --bind=igb_uio "$device"
done
unset IFS
# create hugetable on the host. Those hugetables will be passed through to the container
bash /root/$FORWARDER/setup-hugetlbfs.sh
Configure Container and Management Interface
The following requirements apply before the next step:
- A DHCP server for setting the IP address of bridge and container
- a SSH public key under /root/.ssh/authorized_keys
- prepare_container.sh script on the DuT under /root
Optimized (user-space)
ip link del lxcbr0;
ip link add name br0 type bridge;
sleep 1;
ip link set dev br0 address 52:54:00:00:01;
sleep 1;
ip link set br0 up;
ip link set ens2 master br0;
cat > "/etc/systemd/network/00-management.network" <<-EOF
[Match]
MACAddress=52:54:00:00:01
Name=br0
[Network]
DHCP=yes
[DHCP]
UseDomains=yes
EOF
systemctl restart systemd-networkd;
ip addr del "$(ip addr show dev ens2| awk -F ' *|:' '/inet /')" dev ens2;
apt update;
lxc-create -n container1 -t debian -- --arch amd64 --release bullseye
python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
--bind=igb_uio
$(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
--status | grep enp33s0f1 | awk '')
python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
--bind=igb_uio
$(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
--status | grep enp100s0f1 | awk '')
MAJOR_ID=${ls -la /dev/uio* | grep uio0 | awk '{print $5}' | sed 's/,//g'}
tee -a /var/lib/lxc/container1/config << END
lxc.net.0.type = veth
lxc.net.0.link = br0
lxc.net.0.name = eth0
lxc.net.0.hwaddr = 52:53:00:00:02
lxc.net.0.flags = up
lxc.mount.auto =
lxc.mount.auto = 'proc:rw sys:rw' 'proc:rw sys:rw'
lxc.cgroup2.devices.allow = 'c $MAJOR_ID:0 rwm' 'c $MAJOR_ID:1 rwm'
lxc.mount.entry = '/dev/uio0 dev/uio0 none bind,create=file' '/dev/uio1 dev/uio1 none bind,create=file'
lxc.mount.entry = '/dev/hugepages dev/hugepages none bind,create=dir 0 0'
'/dev/hugepages dev/hugepages none bind,create=dir 0 0'
lxc.cgroup2.cpuset.cpus = 24-26
lxc.cgroup2.memory.max = 14000000000
lxc.cgroup2.cpuset.cpus.partition = 'root'
END
lxc-start -n container1
# Requires the prepare_container.sh script on the path of this script for execution
cat "prepare_container.sh" | lxc-attach -n container1 -- sh
lxc-attach -n container1 -- /bin/bash -c 'mkdir /root/.ssh'
# For later access the SSH files need to be on the selected location
cat "/root/.ssh/authorized_keys" | lxc-attach -n container1 -- /bin/sh -c
'/bin/cat > /root/.ssh/authorized_keys'
lxc-attach -n container1 -- /bin/bash -c 'systemctl set-property system.slice AllowedCPUs=24;
systemctl set-property init.scope AllowedCPUs=24;'
vbmc add --username ADMIN --password password --port 6001 container1;vbmc start container1
lxc-stop -n container1
lxc-start -n container1
Not Optimized (user-space)
ip link del lxcbr0;
ip link add name br0 type bridge;
sleep 1;
ip link set dev br0 address 52:54:00:00:01;
sleep 1;
ip link set br0 up;
ip link set ens2 master br0;
cat > "/etc/systemd/network/00-management.network" <<-EOF
[Match]
MACAddress=52:54:00:00:01
Name=br0
[Network]
DHCP=yes
[DHCP]
UseDomains=yes
EOF
systemctl restart systemd-networkd;
ip addr del "$(ip addr show dev ens2| awk -F ' *|:' '/inet /')" dev ens2;
apt update;
lxc-create -n container1 -t debian -- --arch amd64 --release bullseye
python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
--bind=igb_uio
$(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
--status | grep enp33s0f1 | awk '')
python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
--bind=igb_uio
$(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
--status | grep enp100s0f1 | awk '')
MAJOR_ID=${ls -la /dev/uio* | grep uio0 | awk '{print $5}' | sed 's/,//g'}
tee -a /var/lib/lxc/container1/config << END
lxc.net.0.type = veth
lxc.net.0.link = br0
lxc.net.0.name = eth0
lxc.net.0.hwaddr = 52:53:00:00:02
lxc.net.0.flags = up
lxc.mount.auto =
lxc.mount.auto = 'proc:rw sys:rw' 'proc:rw sys:rw'
lxc.cgroup2.devices.allow = 'c $MAJOR_ID:0 rwm' 'c $MAJOR_ID:1 rwm'
lxc.mount.entry = '/dev/uio0 dev/uio0 none bind,create=file'
'/dev/uio1 dev/uio1 none bind,create=file'
lxc.mount.entry = '/dev/hugepages dev/hugepages none bind,create=dir 0 0' '
/dev/hugepages dev/hugepages none bind,create=dir 0 0'
END
lxc-start -n container1
# Requires the prepare_container.sh script on the path of this script for execution
cat "prepare_container.sh" | lxc-attach -n container1 -- sh
lxc-attach -n container1 -- /bin/bash -c 'mkdir /root/.ssh'
# For later access the SSH files need to be on the selected location
cat "/root/.ssh/authorized_keys" | lxc-attach -n container1 -- /bin/sh -c
'/bin/cat > /root/.ssh/authorized_keys'
vbmc add --username ADMIN --password password --port 6001 container1; vbmc start container1
lxc-stop -n container1
lxc-start -n container1
Optimized (kernel-space)
ip link del lxcbr0;
ip link add name br0 type bridge;
sleep 1;
ip link set dev br0 address 52:54:00:00:01;
sleep 1;
ip link set br0 up;
ip link set ens2 master br0;
cat > "/etc/systemd/network/00-management.network" <<-EOF
[Match]
MACAddress=52:54:00:00:01
Name=br0
[Network]
DHCP=yes
[DHCP]
UseDomains=yes
EOF
systemctl restart systemd-networkd;
ip addr del "$(ip addr show dev ens2| awk -F ' *|:' '/inet /')" dev ens2;
apt update;
lxc-create -n container1 -t debian -- --arch amd64 --release bullseye
tee -a /var/lib/lxc/container1/config << END
lxc.net.0.type = veth
lxc.net.0.link = br0
lxc.net.0.name = eth0
lxc.net.0.hwaddr = 52:53:00:00:02
lxc.net.0.flags = up
lxc.mount.auto =
lxc.mount.auto = 'proc:rw sys:rw' 'proc:rw sys:rw'
lxc.net.1.type = 'phys'
lxc.net.2.type = 'phys'
lxc.net.1.link = enp33s0f1
lxc.net.2.link = enp100s0f1
lxc.net.1.name = ens3
lxc.net.2.name = ens4
lxc.net.1.hwaddr = 52:53:00:00:03
lxc.net.1.flags = up
lxc.net.2.hwaddr = 52:53:00:00:04
lxc.net.2.flags = up
lxc.cgroup2.cpuset.cpus = 24-26
lxc.cgroup2.memory.max = 14000000000
lxc.cgroup2.cpuset.cpus.partition = 'root'
END
lxc-start -n container1
# Requires the prepare_container.sh script on the path of this script for execution
cat "prepare_container.sh" | lxc-attach -n container1 -- sh
lxc-attach -n container1 -- /bin/bash -c 'mkdir /root/.ssh'
# For later access the SSH files need to be on the selected location
cat "/root/.ssh/authorized_keys" | lxc-attach -n container1 -- /bin/sh -c
'/bin/cat > /root/.ssh/authorized_keys'
lxc-attach -n container1 -- /bin/bash -c 'systemctl set-property system.slice AllowedCPUs=24;
systemctl set-property init.scope AllowedCPUs=24;'
vbmc add --username ADMIN --password password --port 6001 container1;vbmc start container1
lxc-stop -n container1
lxc-start -n container1
Not Optimized (kernel-space)
ip link del lxcbr0;
ip link add name br0 type bridge;
sleep 1;
ip link set dev br0 address 52:54:00:00:01;
sleep 1;
ip link set br0 up;
ip link set ens2 master br0;
cat > "/etc/systemd/network/00-management.network" <<-EOF
[Match]
MACAddress=52:54:00:00:01
Name=br0
[Network]
DHCP=yes
[DHCP]
UseDomains=yes
EOF
systemctl restart systemd-networkd;
ip addr del "$(ip addr show dev ens2| awk -F ' *|:' '/inet /')" dev ens2;
apt update;
lxc-create -n container1 -t debian -- --arch amd64 --release bullseye
tee -a /var/lib/lxc/container1/config << END
lxc.net.0.type = veth
lxc.net.0.link = br0
lxc.net.0.name = eth0
lxc.net.0.hwaddr = 52:53:00:00:02
lxc.net.0.flags = up
lxc.mount.auto =
lxc.mount.auto = 'proc:rw sys:rw' 'proc:rw sys:rw'
lxc.net.1.type = 'phys'
lxc.net.2.type = 'phys'
lxc.net.1.link = enp33s0f1
lxc.net.2.link = enp100s0f1
lxc.net.1.name = ens3
lxc.net.2.name = ens4
lxc.net.1.hwaddr = 52:53:00:00:03
lxc.net.1.flags = up
lxc.net.2.hwaddr = 52:53:00:00:04
lxc.net.2.flags = up
END
lxc-start -n container1
# Requires the prepare_container.sh script on the path of this script for execution
cat "prepare_container.sh" | lxc-attach -n container1 -- sh
lxc-attach -n container1 -- /bin/bash -c 'mkdir /root/.ssh'
# For later access the SSH files need to be on the selected location
cat "/root/.ssh/authorized_keys" | lxc-attach -n container1 -- /bin/sh -c
'/bin/cat > /root/.ssh/authorized_keys'
vbmc add --username ADMIN --password password --port 6001 container1;vbmc start container1
lxc-stop -n container1
lxc-start -n container1
Optimized (user-space)
FORWARDER=libmoon
prefix=$(hostname)
# modify dpdk-lua.conf so that it grabs the right cores
cat <<EOF > /root/$FORWARDER/dpdk-conf.lua
DPDKConfig {
cores = 24,25,26,
forceNumaNode = 1,
cli = {
"--file-prefix", "$prefix",
"--socket-mem", "512,512,512,512",
}
}
EOF
Not Optimized (user-space)
Nothing
Kernel-space (Optimized/Non-Optimized)
# setup interfaces hardcoded, need to be changed when using another host
INF1="ens3"
INF2="ens4"
ip l set dev $INF1 up
ip l set dev $INF1 promisc on
ip l set dev $INF2 up
ip l set dev $INF2 promisc on
# Enable l2 forwarding on linux
tc qdisc add dev $INF1 ingress
tc filter add dev $INF1 parent ffff: protocol all prio 2 u32 match u32 0 0 flowid 1:1 action mirred egress mirror dev $INF2
After setup, the container can now be started using IPMI from remote or local when the ipmitools are installed:
ipmitool -I lanplus -H [IP-Adresse] -U ADMIN -P password power onExperiment
1. Start Forwarder in the container on the DuT (user-space experiments), kernel-space experiments do not need somehting here, continue with step 2 directly:
FORWARDER=libmoon
/root/$FORWARDER/build/libmoon /root/$FORWARDER/examples/l2-forward.lua [PORT_TX] [PORT_RX]
2. Wait for the forwarder to be finished with the start up, then start MoonGen on the LoadGen:
LOADGEN=moongen
/root/$LOADGEN/build/MoonGen /root/$LOADGEN/examples/moonsniff/traffic-gen.lua -x 64 --fix-packetrate [PACKET_RATE]
--packets [PACKET_RATE*1500] --warm-up 30 --flows 10 --burst 1 [PORT_TX] [PORT_RX]
3. After MoonGen on the LoadGen has been started, a few packets are send for warm-up. After those packets, we have a break in the execution of 30 seconds, which should be used to start the packet sniffer on the timestamper to record the measurements:
Timer: Capture Packets using MoonGen
TIMER=moongen
/root/$TIMER/build/MoonGen /root/$TIMER/examples/moonsniff/sniffer.lua [PORT_PRE] [PORT_POST] --capture --time 150 --snaplen 84
4. The timestamper stops automatically after 150 seconds and creates two PCAPs, a latencies-pre.pcap and latencies-post.pcap to the respective side of the evaluation
5. Repeat Steps 2 to 4 for each rate to be analyzed after saving the PCAPs at another place, because they will be overwritten otherwise.
Reproduce Figures and Data
Raw PCAP Data
All sampled RAW PCAP-Data for scenario1 are available under https://doi.org/10.14459/2023mp1718824.
All sampled RAW PCAP-Data for scenario2 are available under https://doi.org/10.14459/2024mp1736868.
Evaluation
apt update
DEBIAN_FRONTEND=noninteractive apt install -y postgresql
DEBIAN_FRONTEND=noninteractive apt install -y postgresql-client
DEBIAN_FRONTEND=noninteractive apt install -y parallel
DEBIAN_FRONTEND=noninteractive apt install -y python3-pip
DEBIAN_FRONTEND=noninteractive apt install -y texlive-full
DEBIAN_FRONTEND=noninteractive apt install -y lbzip2
DEBIAN_FRONTEND=noninteractive apt install -y rename
DEBIAN_FRONTEND=noninteractive apt install -y zstd
python3 -m pip install pypacker
python3 -m pip install netifaces
python3 -m pip install pylatex
python3 -m pip install matplotlib
python3 -m pip install pandas
python3 -m pip install pyyaml
mkdir /root/results
# Do not use too much as otherwise the evaluation will fail, require a significant amount of disk and memory space
NUM_CORES=4
# Used for the evaluator scripts
git clone https://github.com/AnonymContainer/containierized-low-latency/ /root/containierized-low-latency
# Download PCAPs to /root/results
cd /root/results
Will be available when Double Blind is not necessary anymore due to deanonymization on shared link.
env --chdir /var/lib/postgresql setpriv --init-groups --reuid postgres -- createuser -s root || true
# import and analyze to database
mkdir /root/results/data
cd /root/results/data
parallel -j $NUM_CORES "dropdb --if-exists root{ % }; createdb root{ % };
export PGDATABASE=root{ % };
~/containierized-low-latency/scripts/evaluator/dbscripts/import.sh {};
~/containierized-low-latency/scripts/evaluator/analysis.sh {}"
::: ../latencies-pre.pcap*.zst
# After this under the folder /root/results/data all required CSVs are available
# When using the precompiled CSV data, decompress them first and then put them into /root/results/data for generation of Figures
# Copy required files for plotting
cp -r ~/containierized-low-latency/scripts/evaluator/plotter/* ~/results
cd ~/results
mkdir figures
python3 plotcreator.py figures data .
make -i
# All compiled figures are now available under ~/results/figures
Precompiled CSV Data and Figures
Scenario 1: The precompiled Figures for scenario1 are available for this measurement in the Repository and the precompiled CSVs are available under the following Repository.
Scenario 2: The precompiled Figures for scenario2 are available for this measurement in the Repository and the precompiled CSVs are available under the following Repository.