Figure 8

figures/irq-vanilla-noopt.png
5000 worst case events over time and corresponding interrupt events normalized over time in scenario 2

Steps to reproduce the taken measurements

OS images

These Images were used for the experiments

Vanilla
Linux machine 5.10.0-10-amd64 #1 SMP Debian 5.10.84-1 (2021-12-08) x86_64 GNU/Linux
NoHz
Linux machine 5.10.84-nohz #1 SMP Tue Dec 21 05:07:04 CET 2021 x86_64 GNU/Linux
RT
Linux machine 5.10.0-10-rt-amd64 #1 SMP PREEMPT_RT Debian 5.10.84-1 (2021-12-08) x86_64 GNU/Linux

Debian Bullseye amd64 default privileged from 2022-08-09T19-36-56
For more information see LXC image Server 

Linux machine 4.19.0-17-amd64 #1 SMP Debian 4.19.194-3 (2021-07-18) x86_64 GNU/Linux

Linux machine 5.10.0-8-amd64 #1 SMP Debian 5.10.46-4 (2021-08-03) x86_64 GNU/Linux

Linux machine 5.10.0-8-amd64 #1 SMP Debian 5.10.46-4 (2021-08-03) x86_64 GNU/Linux

Setup


LOADGEN=moongen


apt-get update
apt-get install -y libssl-dev
git clone --branch dpdk-19.05 --recurse-submodules --jobs 4 https://github.com/emmericp/MoonGen "$LOADGEN"
cd $LOADGEN/
/root/$LOADGEN/build.sh
/root/$LOADGEN/bind-interfaces.sh
/root/$LOADGEN/setup-hugetlbfs.sh



LOADGEN=moongen


apt-get update
apt-get install -y libssl-dev
git clone --branch dpdk-19.05 --recurse-submodules --jobs 4 https://github.com/emmericp/MoonGen "$LOADGEN"
cd $LOADGEN/
/root/$LOADGEN/build.sh
/root/$LOADGEN/bind-interfaces.sh
/root/$LOADGEN/setup-hugetlbfs.sh

Bootparameter

Optimized

mce=ignore_ce tsc=reliable idle=poll nohz=on audit=0 nosmt console=ttyS0,115200
apparmor=0 amd_iommu=off nohz_full=24,25,26,8,9 rcu_nocbs=24,25,26,8,9
skew_tick=1 irqaffinity=0 intel_pstate=disable nmi_watchdog=0 nosoftlockup
rcu_nocb_poll random.trust_cpu=on intel_idle.max_cstate=0
systemd.unified_cgroup_hierarchy=1
Not Optimized

console=ttyS0,115200 apparmor=0 amd_iommu=off systemd.unified_cgroup_hierarchy=1

Install general dependencies


for i in $(pgrep rcu[^c]) ; do taskset -pc 0 "$i" ; done

set -x

# Improving performance for the waiting time as we have no hard-drive to wait for
sysctl  vm.dirty_ratio=5
sysctl  vm.dirty_background_ratio=1

PACKAGES="pip lxc debootstrap python3-lxc ethtool"
DEBIAN_FRONTEND=noninteractive apt-get -y update --allow-releaseinfo-change
DEBIAN_FRONTEND=noninteractive apt-get -y install $PACKAGES

# disable apparmor. Apparmor otherwise blocks mounting the rootfs in the containers
aa-teardown

cd /root || exit
git clone https://github.com/tumi8/VirtualLXCBMC
cd virtuallxcbmc || exit

# install dependencies
python3 -m pip install -r requirements.txt
# compile project and move the executables to the right locations
python3 setup.py install
# starts the vbmc daemon in the background
vbmcd

Install libmoon Dependencies on Host


GIT_REPO='https://github.com/libmoon/libmoon.git'
GIT_BRANCH='dpdk-19.05'
FORWARDER=libmoon
cd /root

git clone --branch "$GIT_BRANCH" --recurse-submodules --jobs 4 "$GIT_REPO" "$FORWARDER"
cd $FORWARDER/

# build and load igb_uio
cd deps/dpdk-kmods/linux/igb_uio
make
modprobe uio
insmod igb_uio.ko

cd /root/$FORWARDER/deps/dpdk/usertools/
EXTERNAL_INTERFACES="enp33s0f1 enp100s0f1"
IFS=" "
for device_long in $(echo "$EXTERNAL_INTERFACES"); do
  device=$(echo "$device_long" | awk -F '_' '{ print $2 }')
  python dpdk-devbind.py --bind=igb_uio "$device"
done
unset IFS

# create hugetable on the host. Those hugetables will be passed through to the container
bash /root/$FORWARDER/setup-hugetlbfs.sh

Configure Container and Management Interface

The following requirements apply before the next step:

  • A DHCP server for setting the IP address of bridge and container
  • a SSH public key under /root/.ssh/authorized_keys
  • prepare_container.sh script on the DuT under /root
Optimized

ip link del lxcbr0;

ip link add name br0 type bridge;
sleep 1;

ip link set dev br0 address 52:54:00:00:01;
sleep 1;
ip link set br0 up;

ip link set ens2 master br0;


cat > "/etc/systemd/network/00-management.network" <<-EOF
[Match]
MACAddress=52:54:00:00:01
Name=br0

[Network]
DHCP=yes

[DHCP]
UseDomains=yes
EOF

systemctl restart systemd-networkd;

ip addr del "$(ip addr show dev ens2| awk -F ' *|:' '/inet /')" dev ens2;

apt update;

lxc-create -n container1 -t debian -- --arch amd64 --release bullseye

python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py 
 --bind=igb_uio  
 $(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --status | grep enp33s0f1 | awk '')
python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --bind=igb_uio
   $(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --status | grep enp100s0f1 | awk '')


MAJOR_ID=${ls -la /dev/uio* | grep uio0 | awk '{print $5}' | sed 's/,//g'}


tee -a /var/lib/lxc/container1/config << END
lxc.net.0.type = veth
lxc.net.0.link = br0
lxc.net.0.name = eth0
lxc.net.0.hwaddr = 52:53:00:00:02
lxc.net.0.flags = up

lxc.mount.auto =
lxc.mount.auto = 'proc:rw sys:rw' 'proc:rw sys:rw'

lxc.cgroup2.devices.allow = 'c $MAJOR_ID:0 rwm'  'c $MAJOR_ID:1 rwm'
lxc.mount.entry = '/dev/uio0 dev/uio0 none bind,create=file' '/dev/uio1 dev/uio1 none bind,create=file'
lxc.mount.entry = '/dev/hugepages dev/hugepages none bind,create=dir 0 0' 
 '/dev/hugepages dev/hugepages none bind,create=dir 0 0'
lxc.cgroup2.cpuset.cpus = 24-26
lxc.cgroup2.memory.max = 14000000000
lxc.cgroup2.cpuset.cpus.partition = 'root' 

END

lxc-start -n container1

# Requires the prepare_container.sh script on the path of this script for execution
cat "prepare_container.sh" | lxc-attach -n container1 -- sh

lxc-attach -n container1 --  /bin/bash -c 'mkdir /root/.ssh'

# For later access the SSH files need to be on the selected location
cat "/root/.ssh/authorized_keys" |  lxc-attach -n container1 -- /bin/sh -c 
 '/bin/cat > /root/.ssh/authorized_keys'

lxc-attach -n container1 --  /bin/bash -c 'systemctl set-property system.slice AllowedCPUs=24; 
 systemctl set-property init.scope AllowedCPUs=24;'

vbmc add --username ADMIN --password password --port 6001 container1;vbmc start container1

lxc-stop -n container1

lxc-start -n container1
Not Optimized

ip link del lxcbr0;

ip link add name br0 type bridge;
sleep 1;

ip link set dev br0 address 52:54:00:00:01;
sleep 1;
ip link set br0 up;

ip link set ens2 master br0;


cat > "/etc/systemd/network/00-management.network" <<-EOF
[Match]
MACAddress=52:54:00:00:01
Name=br0

[Network]
DHCP=yes

[DHCP]
UseDomains=yes
EOF

systemctl restart systemd-networkd;

ip addr del "$(ip addr show dev ens2| awk -F ' *|:' '/inet /')" dev ens2;

apt update;

lxc-create -n container1 -t debian -- --arch amd64 --release bullseye

python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py 
 --bind=igb_uio  
 $(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --status | grep enp33s0f1 | awk '')
python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --bind=igb_uio
   $(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --status | grep enp100s0f1 | awk '')

MAJOR_ID=${ls -la /dev/uio* | grep uio0 | awk '{print $5}' | sed 's/,//g'}


tee -a /var/lib/lxc/container1/config << END
lxc.net.0.type = veth
lxc.net.0.link = br0
lxc.net.0.name = eth0
lxc.net.0.hwaddr = 52:53:00:00:02
lxc.net.0.flags = up

lxc.mount.auto =
lxc.mount.auto = 'proc:rw sys:rw' 'proc:rw sys:rw'

lxc.cgroup2.devices.allow = 'c $MAJOR_ID:0 rwm'  'c $MAJOR_ID:1 rwm'
lxc.mount.entry = '/dev/uio0 dev/uio0 none bind,create=file' 
 '/dev/uio1 dev/uio1 none bind,create=file'
lxc.mount.entry = '/dev/hugepages dev/hugepages none bind,create=dir 0 0' '
 /dev/hugepages dev/hugepages none bind,create=dir 0 0'

END

lxc-start -n container1

# Requires the prepare_container.sh script on the path of this script for execution
cat "prepare_container.sh" | lxc-attach -n container1 -- sh

lxc-attach -n container1 --  /bin/bash -c 'mkdir /root/.ssh'

# For later access the SSH files need to be on the selected location
cat "/root/.ssh/authorized_keys" |  lxc-attach -n container1 -- /bin/sh -c 
 '/bin/cat > /root/.ssh/authorized_keys'


vbmc add --username ADMIN --password password --port 6001 container1; vbmc start container1

lxc-stop -n container1

lxc-start -n container1
Optimized

FORWARDER=libmoon

prefix=$(hostname)
# modify dpdk-lua.conf so that it grabs the right cores
cat <<EOF > /root/$FORWARDER/dpdk-conf.lua
DPDKConfig {
  cores = 24,25,26,

  forceNumaNode = 1,

  cli = {
    "--file-prefix", "$prefix",
    "--socket-mem", "512,512,512,512",
  }
}
EOF
Not Optimized

Nothing

After setup, the container can now be started using IPMI from remote or local when the ipmitools are installed: 
 ipmitool -I lanplus -H [IP-Adresse] -U ADMIN -P password power on
For using the Interrupt-recorder functionality, the following file need to be available on the container: irqrecorder.py

Experiment

1. Start Forwarder in the container on the DuT:


FORWARDER=libmoon

/root/$FORWARDER/build/libmoon /root/$FORWARDER/examples/l2-forward.lua [PORT_TX] [PORT_RX]

2. Start the interrupt recorder on the container:

taskset -c 2 /root/irqrecorder.py

3. Wait for the forwarder to be finished with the start up, then start MoonGen on the LoadGen:


LOADGEN=moongen


/root/$LOADGEN/build/MoonGen /root/$LOADGEN/examples/moonsniff/traffic-gen.lua -x 64 --fix-packetrate [PACKET_RATE]
 --packets [PACKET_RATE*1500] --warm-up 30 --flows 10 --burst 1 [PORT_TX] [PORT_RX]

4. After MoonGen on the LoadGen has been started, a few packets are send for warm-up. After those packets, we have a break in the execution of 30 seconds, which should be used to start the packet sniffer on the timestamper to record the measurements:


TIMER=moongen


/root/$TIMER/build/MoonGen /root/$TIMER/examples/moonsniff/sniffer.lua [PORT_PRE] [PORT_POST] --capture --time 150 --snaplen 84

5. The timestamper stops automatically after 150 seconds and creates two PCAPs, a latencies-pre.pcap and latencies-post.pcap to the respective side of the evaluation

6. Stop the IRQ recorder and rename the file for later recognition.

7. Repeat Steps 2 to 6 for each rate to be analyzed after saving the PCAPs and CSVs at another place, because they will be overwritten otherwise.

Reproduce Figures and Data

Evaluation


apt update
DEBIAN_FRONTEND=noninteractive apt install -y postgresql
DEBIAN_FRONTEND=noninteractive apt install -y postgresql-client
DEBIAN_FRONTEND=noninteractive apt install -y parallel
DEBIAN_FRONTEND=noninteractive apt install -y python3-pip
DEBIAN_FRONTEND=noninteractive apt install -y texlive-full
DEBIAN_FRONTEND=noninteractive apt install -y lbzip2
DEBIAN_FRONTEND=noninteractive apt install -y rename
DEBIAN_FRONTEND=noninteractive apt install -y zstd

python3 -m pip install pypacker
python3 -m pip install netifaces
python3 -m pip install pylatex
python3 -m pip install matplotlib
python3 -m pip install pandas
python3 -m pip install pyyaml

mkdir /root/results



# Do not use too much as otherwise the evaluation will fail, require a significant amount of disk and memory space
NUM_CORES=4

# Used for the evaluator scripts
git clone https://github.com/AnonymContainer/containierized-low-latency/ /root/containierized-low-latency

# Download PCAPs to /root/results
cd /root/results
Will be available when Double Blind is not necessary anymore due to deanonymization on shared link.

env --chdir /var/lib/postgresql setpriv --init-groups --reuid postgres -- createuser -s root || true

# import and analyze to database
mkdir /root/results/data
cd /root/results/data

parallel -j $NUM_CORES "dropdb --if-exists root{ % }; createdb root{ % };
 export PGDATABASE=root{ % };
 ~/containierized-low-latency/scripts/evaluator/dbscripts/import.sh {};
 ~/containierized-low-latency/scripts/evaluator/analysis.sh {}"
 ::: ../latencies-pre.pcap*.zst

# After this under the folder /root/results/data all required CSVs are available



# When using the precompiled CSV data, decompress them first and then put them into /root/results/data for generation of Figures

# Copy required files for plotting
cp -r ~/containierized-low-latency/scripts/evaluator/plotter/* ~/results

cd ~/results
mkdir figures

python3 plotcreator.py figures data .
make -i

# All compiled figures are now available under ~/results/figures

Precompiled CSV Data and Figures

The CSVs and Worst-case data for this Figure are available for this measurement in the Repository.