Figure 8

Debian Bullseye amd64 default privileged from 2022-08-09T19-36-56

Linux machine 4.19.0-17-amd64 #1 SMP Debian 4.19.194-3 (2021-07-18) x86_64 GNU/Linux

Linux machine 5.10.0-8-amd64 #1 SMP Debian 5.10.46-4 (2021-08-03) x86_64 GNU/Linux

Linux machine 5.10.0-8-amd64 #1 SMP Debian 5.10.46-4 (2021-08-03) x86_64 GNU/Linux

LOADGEN=moongen


apt-get update
apt-get install -y libssl-dev
git clone --branch dpdk-19.05 --recurse-submodules --jobs 4 https://github.com/emmericp/MoonGen "$LOADGEN"
cd $LOADGEN/
/root/$LOADGEN/build.sh
/root/$LOADGEN/bind-interfaces.sh
/root/$LOADGEN/setup-hugetlbfs.sh

LOADGEN=moongen


apt-get update
apt-get install -y libssl-dev
git clone --branch dpdk-19.05 --recurse-submodules --jobs 4 https://github.com/emmericp/MoonGen "$LOADGEN"
cd $LOADGEN/
/root/$LOADGEN/build.sh
/root/$LOADGEN/bind-interfaces.sh
/root/$LOADGEN/setup-hugetlbfs.sh

Bootparameter

mce=ignore_ce tsc=reliable idle=poll nohz=on audit=0 nosmt console=ttyS0,115200
apparmor=0 amd_iommu=off nohz_full=24,25,26,8,9 rcu_nocbs=24,25,26,8,9
skew_tick=1 irqaffinity=0 intel_pstate=disable nmi_watchdog=0 nosoftlockup
rcu_nocb_poll random.trust_cpu=on intel_idle.max_cstate=0
systemd.unified_cgroup_hierarchy=0

Install general dependencies

for i in $(pgrep rcu[^c]) ; do taskset -pc 0 "$i" ; done

set -x

# Improving performance for the waiting time as we have no hard-drive to wait for
sysctl  vm.dirty_ratio=5
sysctl  vm.dirty_background_ratio=1

PACKAGES="pip lxc debootstrap python3-lxc ethtool cpuset"
DEBIAN_FRONTEND=noninteractive apt-get -y update --allow-releaseinfo-change
DEBIAN_FRONTEND=noninteractive apt-get -y install $PACKAGES

# disable apparmor. Apparmor otherwise blocks mounting the rootfs in the containers
aa-teardown

cd /root || exit
git clone https://github.com/AnonymContainer/virtuallxcbmc
cd virtuallxcbmc || exit

# install dependencies
python3 -m pip install -r requirements.txt
# compile project and move the executables to the right locations
python3 setup.py install
# starts the vbmc daemon in the background
vbmcd

Install libmoon Dependencies on Host

GIT_REPO='https://github.com/libmoon/libmoon.git'
GIT_BRANCH='dpdk-19.05'
FORWARDER=libmoon
cd /root

git clone --branch "$GIT_BRANCH" --recurse-submodules --jobs 4 "$GIT_REPO" "$FORWARDER"
cd $FORWARDER/

# build and load igb_uio
cd deps/dpdk-kmods/linux/igb_uio
make
modprobe uio
insmod igb_uio.ko

cd /root/$FORWARDER/deps/dpdk/usertools/
EXTERNAL_INTERFACES="enp33s0f1 enp100s0f1"
IFS=" "
for device_long in $(echo "$EXTERNAL_INTERFACES"); do
  device=$(echo "$device_long" | awk -F '_' '{ print $2 }')
  python dpdk-devbind.py --bind=igb_uio "$device"
done
unset IFS

# create hugetable on the host. Those hugetables will be passed through to the container
bash /root/$FORWARDER/setup-hugetlbfs.sh

Configure Container and Management Interface

The following requirements apply before the next step:

• A DHCP server for setting the IP address of bridge and container
• a SSH public key under /root/.ssh/authorized_keys
• prepare_container.sh script on the DuT under /root

ip link del lxcbr0;

ip link add name br0 type bridge;
sleep 1;

ip link set dev br0 address 52:54:00:00:01;
sleep 1;
ip link set br0 up;

ip link set ens2 master br0;


cat > "/etc/systemd/network/00-management.network" <<-EOF
[Match]
MACAddress=52:54:00:00:01
Name=br0

[Network]
DHCP=yes

[DHCP]
UseDomains=yes
EOF

systemctl restart systemd-networkd;

ip addr del "$(ip addr show dev ens2| awk -F ' *|:' '/inet /')" dev ens2;

apt update;

lxc-create -n container1 -t debian -- --arch amd64 --release bullseye

python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py 
 --bind=igb_uio  
 $(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --status | grep enp33s0f1 | awk '')
python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --bind=igb_uio
   $(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --status | grep enp100s0f1 | awk '')


MAJOR_ID=${ls -la /dev/uio* | grep uio0 | awk '{print $5}' | sed 's/,//g'}


tee -a /var/lib/lxc/container1/config << END
lxc.net.0.type = veth
lxc.net.0.link = br0
lxc.net.0.name = eth0
lxc.net.0.hwaddr = 52:53:00:00:02
lxc.net.0.flags = up

lxc.mount.auto =
lxc.mount.auto = 'proc:rw sys:rw' 'proc:rw sys:rw'

lxc.cgroup2.devices.allow = 'c $MAJOR_ID:0 rwm'  'c $MAJOR_ID:1 rwm'
lxc.mount.entry = '/dev/uio0 dev/uio0 none bind,create=file' '/dev/uio1 dev/uio1 none bind,create=file'
lxc.mount.entry = '/dev/hugepages dev/hugepages none bind,create=dir 0 0' 
 '/dev/hugepages dev/hugepages none bind,create=dir 0 0'
lxc.cgroup2.cpuset.cpus = 24-26
lxc.cgroup2.memory.max = 14000000000
lxc.cgroup2.cpuset.cpus.partition = 'root' 

END

cd /sys/fs/cgroup/cpuset
mkdir housekeeping
/bin/echo 0 > housekeeping/cpuset.mems
/bin/echo 0-23,27-31 > housekeeping/cpuset.cpus

# move all processes into housekeeping
cset proc --move --fromset=root --toset=housekeeping

# move kernel tasks into housekeeping
cset proc --kthread --fromset=root --toset=housekeeping --force

# init-process must be in root cg 
cset proc --toset=root --move --pid 1

lxc-start -n container1

# Requires the prepare_container.sh script on the path of this script for execution
cat "prepare_container.sh" | lxc-attach -n container1 -- sh

lxc-attach -n container1 --  /bin/bash -c 'mkdir /root/.ssh'

# For later access the SSH files need to be on the selected location
cat "/root/.ssh/authorized_keys" |  lxc-attach -n container1 -- /bin/sh -c 
 '/bin/cat > /root/.ssh/authorized_keys'

lxc-attach -n container1 --  /bin/bash -c 'systemctl set-property system.slice AllowedCPUs=24; 
 systemctl set-property init.scope AllowedCPUs=24;'

vbmc add --username ADMIN --password password --port 6001 container1;vbmc start container1

lxc-stop -n container1

lxc-start -n container1

Bootparameter

mce=ignore_ce tsc=reliable idle=poll nohz=on audit=0 nosmt console=ttyS0,115200
apparmor=0 amd_iommu=off nohz_full=24,25,26,8,9 rcu_nocbs=24,25,26,8,9
skew_tick=1 irqaffinity=0 intel_pstate=disable nmi_watchdog=0 nosoftlockup
rcu_nocb_poll random.trust_cpu=on intel_idle.max_cstate=0
systemd.unified_cgroup_hierarchy=1

Install general dependencies

for i in $(pgrep rcu[^c]) ; do taskset -pc 0 "$i" ; done

set -x

# Improving performance for the waiting time as we have no hard-drive to wait for
sysctl  vm.dirty_ratio=5
sysctl  vm.dirty_background_ratio=1

PACKAGES="pip lxc debootstrap python3-lxc ethtool cpuset"
DEBIAN_FRONTEND=noninteractive apt-get -y update --allow-releaseinfo-change
DEBIAN_FRONTEND=noninteractive apt-get -y install $PACKAGES

# disable apparmor. Apparmor otherwise blocks mounting the rootfs in the containers
aa-teardown

cd /root || exit
git clone https://github.com/AnonymContainer/virtuallxcbmc
cd virtuallxcbmc || exit

# install dependencies
python3 -m pip install -r requirements.txt
# compile project and move the executables to the right locations
python3 setup.py install
# starts the vbmc daemon in the background
vbmcd

Install libmoon Dependencies on Host

GIT_REPO='https://github.com/libmoon/libmoon.git'
GIT_BRANCH='dpdk-19.05'
FORWARDER=libmoon
cd /root

git clone --branch "$GIT_BRANCH" --recurse-submodules --jobs 4 "$GIT_REPO" "$FORWARDER"
cd $FORWARDER/

# build and load igb_uio
cd deps/dpdk-kmods/linux/igb_uio
make
modprobe uio
insmod igb_uio.ko

cd /root/$FORWARDER/deps/dpdk/usertools/
EXTERNAL_INTERFACES="enp33s0f1 enp100s0f1"
IFS=" "
for device_long in $(echo "$EXTERNAL_INTERFACES"); do
  device=$(echo "$device_long" | awk -F '_' '{ print $2 }')
  python dpdk-devbind.py --bind=igb_uio "$device"
done
unset IFS

# create hugetable on the host. Those hugetables will be passed through to the container
bash /root/$FORWARDER/setup-hugetlbfs.sh

Configure Container and Management Interface

The following requirements apply before the next step:

• A DHCP server for setting the IP address of bridge and container
• a SSH public key under /root/.ssh/authorized_keys
• prepare_container.sh script on the DuT under /root

ip link del lxcbr0;

ip link add name br0 type bridge;
sleep 1;

ip link set dev br0 address 52:54:00:00:01;
sleep 1;
ip link set br0 up;

ip link set ens2 master br0;


cat > "/etc/systemd/network/00-management.network" <<-EOF
[Match]
MACAddress=52:54:00:00:01
Name=br0

[Network]
DHCP=yes

[DHCP]
UseDomains=yes
EOF

systemctl restart systemd-networkd;

ip addr del "$(ip addr show dev ens2| awk -F ' *|:' '/inet /')" dev ens2;

apt update;

lxc-create -n container1 -t debian -- --arch amd64 --release bullseye

python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py 
 --bind=igb_uio  
 $(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --status | grep enp33s0f1 | awk '')
python3 /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --bind=igb_uio
   $(python /root/libmoon/deps/dpdk/usertools/dpdk-devbind.py
  --status | grep enp100s0f1 | awk '')


MAJOR_ID=${ls -la /dev/uio* | grep uio0 | awk '{print $5}' | sed 's/,//g'}


tee -a /var/lib/lxc/container1/config << END
lxc.net.0.type = veth
lxc.net.0.link = br0
lxc.net.0.name = eth0
lxc.net.0.hwaddr = 52:53:00:00:02
lxc.net.0.flags = up

lxc.mount.auto =
lxc.mount.auto = 'proc:rw sys:rw' 'proc:rw sys:rw'

lxc.cgroup2.devices.allow = 'c $MAJOR_ID:0 rwm'  'c $MAJOR_ID:1 rwm'
lxc.mount.entry = '/dev/uio0 dev/uio0 none bind,create=file' '/dev/uio1 dev/uio1 none bind,create=file'
lxc.mount.entry = '/dev/hugepages dev/hugepages none bind,create=dir 0 0' 
 '/dev/hugepages dev/hugepages none bind,create=dir 0 0'
lxc.cgroup2.cpuset.cpus = 24-26
lxc.cgroup2.memory.max = 14000000000
lxc.cgroup2.cpuset.cpus.partition = 'root' 

END

lxc-start -n container1

# Requires the prepare_container.sh script on the path of this script for execution
cat "prepare_container.sh" | lxc-attach -n container1 -- sh

lxc-attach -n container1 --  /bin/bash -c 'mkdir /root/.ssh'

# For later access the SSH files need to be on the selected location
cat "/root/.ssh/authorized_keys" |  lxc-attach -n container1 -- /bin/sh -c 
 '/bin/cat > /root/.ssh/authorized_keys'

lxc-attach -n container1 --  /bin/bash -c 'systemctl set-property system.slice AllowedCPUs=24; 
 systemctl set-property init.scope AllowedCPUs=24;'

vbmc add --username ADMIN --password password --port 6001 container1;vbmc start container1

lxc-stop -n container1

lxc-start -n container1

FORWARDER=libmoon

prefix=$(hostname)
# modify dpdk-lua.conf so that it grabs the right cores
cat <<EOF > /root/$FORWARDER/dpdk-conf.lua
DPDKConfig {
  cores = 24,25,26,

  forceNumaNode = 1,

  cli = {
    "--file-prefix", "$prefix",
    "--socket-mem", "512,512,512,512",
  }
}
EOF

FORWARDER=libmoon

/root/$FORWARDER/build/libmoon /root/$FORWARDER/examples/l2-forward.lua [PORT_TX] [PORT_RX]

LOADGEN=moongen


/root/$LOADGEN/build/MoonGen /root/$LOADGEN/examples/moonsniff/traffic-gen.lua -x 64 --fix-packetrate [PACKET_RATE]
 --packets [PACKET_RATE*1500] --warm-up 30 --flows 10 --burst 1 [PORT_TX] [PORT_RX]

TIMER=moongen


/root/$TIMER/build/MoonGen /root/$TIMER/examples/moonsniff/sniffer.lua [PORT_PRE] [PORT_POST] --capture --time 150 --snaplen 84

apt update
DEBIAN_FRONTEND=noninteractive apt install -y postgresql
DEBIAN_FRONTEND=noninteractive apt install -y postgresql-client
DEBIAN_FRONTEND=noninteractive apt install -y parallel
DEBIAN_FRONTEND=noninteractive apt install -y python3-pip
DEBIAN_FRONTEND=noninteractive apt install -y texlive-full
DEBIAN_FRONTEND=noninteractive apt install -y lbzip2
DEBIAN_FRONTEND=noninteractive apt install -y rename
DEBIAN_FRONTEND=noninteractive apt install -y zstd

python3 -m pip install pypacker
python3 -m pip install netifaces
python3 -m pip install pylatex
python3 -m pip install matplotlib
python3 -m pip install pandas
python3 -m pip install pyyaml

mkdir /root/results

# Do not use too much as otherwise the evaluation will fail, require a significant amount of disk and memory space
NUM_CORES=4

# Used for the evaluator scripts
git clone https://github.com/wiednerf/cgroups-nfv/ /root/cgroups-nfv/ 

# Download PCAPs to /root/results
cd /root/results
Will be available when Double Blind is not necessary anymore due to deanonymization on shared link.

env --chdir /var/lib/postgresql setpriv --init-groups --reuid postgres -- createuser -s root || true

# import and analyze to database
mkdir /root/results/data
cd /root/results/data

parallel -j $NUM_CORES "dropdb --if-exists root{ % }; createdb root{ % };
 export PGDATABASE=root{ % };
 ~/cgroups-nfv/scripts/evaluator/dbscripts/import.sh {};
 ~/cgroups-nfv/scripts/evaluator/analysis.sh {}"
 ::: ../latencies-pre.pcap*.zst

# After this under the folder /root/results/data all required CSVs are available

# When using the precompiled CSV data, decompress them first and then put them into /root/results/data for generation of Figures

# Copy required files for plotting
cp -r ~/cgroups-nfv/scripts/evaluator/plotter/* ~/results

cd ~/results
mkdir figures

python3 plotcreator.py figures data .
make -i

# All compiled figures are now available under ~/results/figures