From 53a0a5473f45d37027af9663638adbf56667e4ca Mon Sep 17 00:00:00 2001
From: Andrew Opalach <andrew@akon.city>
Date: Tue, 12 Nov 2024 22:26:02 -0500
Subject: Hopefully no leaks

---
 hosts/iroha/libvirt/hooks/arcade.conf              |  4 ++
 hosts/iroha/libvirt/hooks/cpuset.conf              | 10 +++++
 hosts/iroha/libvirt/hooks/cpuset_prepare_common.sh | 51 ++++++++++++++++++++++
 hosts/iroha/libvirt/hooks/cpuset_release_common.sh | 33 ++++++++++++++
 hosts/iroha/libvirt/hooks/qemu                     | 29 ++++++++++++
 .../hooks/qemu.d/arcade/prepare/begin/bind_vfio.sh | 20 +++++++++
 .../arcade/prepare/begin/cpuset_prepare_common.sh  |  1 +
 .../qemu.d/arcade/prepare/begin/port_forward.sh    | 14 ++++++
 .../arcade/release/end/cpuset_release_common.sh    |  1 +
 .../hooks/qemu.d/arcade/release/end/port_close.sh  | 14 ++++++
 .../hooks/qemu.d/arcade/release/end/unbind_vfio.sh | 17 ++++++++
 11 files changed, 194 insertions(+)
 create mode 100644 hosts/iroha/libvirt/hooks/arcade.conf
 create mode 100644 hosts/iroha/libvirt/hooks/cpuset.conf
 create mode 100755 hosts/iroha/libvirt/hooks/cpuset_prepare_common.sh
 create mode 100755 hosts/iroha/libvirt/hooks/cpuset_release_common.sh
 create mode 100755 hosts/iroha/libvirt/hooks/qemu
 create mode 100755 hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/bind_vfio.sh
 create mode 120000 hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/cpuset_prepare_common.sh
 create mode 100755 hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/port_forward.sh
 create mode 120000 hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/cpuset_release_common.sh
 create mode 100755 hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/port_close.sh
 create mode 100755 hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/unbind_vfio.sh

(limited to 'hosts/iroha/libvirt/hooks')

diff --git a/hosts/iroha/libvirt/hooks/arcade.conf b/hosts/iroha/libvirt/hooks/arcade.conf
new file mode 100644
index 0000000..4282629
--- /dev/null
+++ b/hosts/iroha/libvirt/hooks/arcade.conf
@@ -0,0 +1,4 @@
+VIRSH_GPU_VIDEO=pci_0000_0f_00_0
+VIRSH_GPU_AUDIO=pci_0000_0f_00_1
+VIRSH_USB=pci_0000_03_00_0
+VIRSH_NVME=pci_0000_0e_00_0
diff --git a/hosts/iroha/libvirt/hooks/cpuset.conf b/hosts/iroha/libvirt/hooks/cpuset.conf
new file mode 100644
index 0000000..d208294
--- /dev/null
+++ b/hosts/iroha/libvirt/hooks/cpuset.conf
@@ -0,0 +1,10 @@
+TOTAL_CORES='0-23'       # 3900X: 12-core, 24-thread cpu
+HOST_CORES='0-4,12-16'
+VIRT_CORES0='5-11,17-23'
+MEMNODES='0'
+
+CPUSET_REFCOUNT=/tmp/cpuset-ref
+CGROUP=/sys/fs/cgroup
+CPUSET=${CGROUP}/cpuset
+SETNAME=host
+PARTNAME0=arcade
diff --git a/hosts/iroha/libvirt/hooks/cpuset_prepare_common.sh b/hosts/iroha/libvirt/hooks/cpuset_prepare_common.sh
new file mode 100755
index 0000000..e4bb35f
--- /dev/null
+++ b/hosts/iroha/libvirt/hooks/cpuset_prepare_common.sh
@@ -0,0 +1,51 @@
+#! /usr/bin/env bash
+
+source "/etc/libvirt/hooks/cpuset.conf"
+
+if test -f $CPUSET_REFCOUNT;
+then
+  ref=$(tail -1 $CPUSET_REFCOUNT)
+  printf "%06d" $(expr $ref + 1) > $CPUSET_REFCOUNT
+  exit 0
+else
+  printf "%06d" 1 > $CPUSET_REFCOUNT
+fi
+
+/bin/echo ${TOTAL_CORES} > ${CPUSET}/cpuset.cpus
+/bin/echo ${MEMNODES} > ${CPUSET}/cpuset.mems
+
+if test -d ${CPUSET}/machine;
+then
+  :
+else
+  mkdir -p ${CPUSET}/machine
+fi
+
+/bin/echo ${TOTAL_CORES} > ${CPUSET}/machine/cpuset.cpus
+/bin/echo ${MEMNODES} > ${CPUSET}/machine/cpuset.mems
+
+if test -d ${CPUSET}/${SETNAME};
+then
+  :
+else
+  mkdir ${CPUSET}/${SETNAME}
+fi
+
+/bin/echo ${HOST_CORES} > ${CPUSET}/${SETNAME}/cpuset.cpus
+/bin/echo ${MEMNODES} > ${CPUSET}/${SETNAME}/cpuset.mems
+
+for i in blkio cpu cpuacct cpuset devices freezer memory net_cls perf_event;
+do
+  mkdir -p ${CGROUP}/$i/machine/${PARTNAME0}.partition
+done
+
+/bin/echo ${VIRT_CORES0} > ${CPUSET}/machine/${PARTNAME0}.partition/cpuset.cpus
+/bin/echo ${MEMNODES} > ${CPUSET}/machine/${PARTNAME0}.partition/cpuset.mems
+
+for i in `cat ${CPUSET}/tasks`;
+do
+  /bin/echo ${i} > ${CPUSET}/${SETNAME}/tasks || :
+done
+
+# (pgrep kthreadd) = 2
+taskset -pc ${HOST_CORES} 2
diff --git a/hosts/iroha/libvirt/hooks/cpuset_release_common.sh b/hosts/iroha/libvirt/hooks/cpuset_release_common.sh
new file mode 100755
index 0000000..624fa5b
--- /dev/null
+++ b/hosts/iroha/libvirt/hooks/cpuset_release_common.sh
@@ -0,0 +1,33 @@
+#! /usr/bin/env bash
+
+source "/etc/libvirt/hooks/cpuset.conf"
+
+if test -f $CPUSET_REFCOUNT;
+then
+  ref=$(tail -1 $CPUSET_REFCOUNT)
+  dec=$(expr $ref - 1)
+  printf "%06d" $(expr $dec) > $CPUSET_REFCOUNT
+  if [ $dec -gt 0 ];
+  then
+    exit 0
+  fi
+  rm $CPUSET_REFCOUNT
+else
+  exit 1
+fi
+
+if test -d ${CPUSET}/${SETNAME};
+then
+  sed -un p < ${CPUSET}/${SETNAME}/tasks > ${CPUSET}/tasks
+  rmdir ${CPUSET}/${SETNAME}
+fi
+
+for i in blkio cpu cpuacct cpuset devices freezer memory net_cls perf_event;
+do
+  rmdir ${CGROUP}/$i/machine/${PARTNAME0}.partition
+done
+
+/bin/echo ${TOTAL_CORES} > ${CPUSET}/machine/cpuset.cpus
+
+# (pgrep kthreadd) = 2
+taskset -pc ${TOTAL_CORES} 2
diff --git a/hosts/iroha/libvirt/hooks/qemu b/hosts/iroha/libvirt/hooks/qemu
new file mode 100755
index 0000000..ccdf9ae
--- /dev/null
+++ b/hosts/iroha/libvirt/hooks/qemu
@@ -0,0 +1,29 @@
+#! /usr/bin/env bash
+#
+# Author: Sebastiaan Meijer (sebastiaan@passthroughpo.st)
+#
+# Copy this file to /etc/libvirt/hooks, make sure it's called "qemu".
+# After this file is installed, restart libvirt.
+# From now on, you can easily add per-guest qemu hooks.
+# Add your hooks in /etc/libvirt/hooks/qemu.d/vm_name/hook_name/state_name.
+# For a list of available hooks, please refer to https://www.libvirt.org/hooks.html
+#
+
+GUEST_NAME="$1"
+HOOK_NAME="$2"
+STATE_NAME="$3"
+MISC="${@:4}"
+
+BASEDIR="$(dirname $0)"
+
+HOOKPATH="$BASEDIR/qemu.d/$GUEST_NAME/$HOOK_NAME/$STATE_NAME"
+
+set -e # If a script exits with an error, we should as well.
+
+if [ -f "$HOOKPATH" ]; then
+    eval \""$HOOKPATH"\" "$@"
+elif [ -d "$HOOKPATH" ]; then
+    while read file; do
+        eval \""$file"\" "$@"
+    done <<< "$(find -L "$HOOKPATH" -maxdepth 1 -type f -executable -print;)"
+fi
diff --git a/hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/bind_vfio.sh b/hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/bind_vfio.sh
new file mode 100755
index 0000000..f89b61a
--- /dev/null
+++ b/hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/bind_vfio.sh
@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+source "/etc/libvirt/hooks/arcade.conf"
+
+echo 0 > /sys/class/vtconsole/vtcon0/bind
+echo 0 > /sys/class/vtconsole/vtcon1/bind
+
+modprobe -r nvidia-drm
+modprobe -r nvidia-modeset
+modprobe -r nvidia-uvm
+modprobe -r nvidia
+
+modprobe vfio
+modprobe vfio_iommu_type1
+modprobe vfio_pci
+
+virsh nodedev-detach $VIRSH_GPU_VIDEO
+virsh nodedev-detach $VIRSH_GPU_AUDIO
+virsh nodedev-detach $VIRSH_NVME
+virsh nodedev-detach $VIRSH_USB
diff --git a/hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/cpuset_prepare_common.sh b/hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/cpuset_prepare_common.sh
new file mode 120000
index 0000000..fb5a2ab
--- /dev/null
+++ b/hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/cpuset_prepare_common.sh
@@ -0,0 +1 @@
+../../../../cpuset_prepare_common.sh
\ No newline at end of file
diff --git a/hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/port_forward.sh b/hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/port_forward.sh
new file mode 100755
index 0000000..7d5d484
--- /dev/null
+++ b/hosts/iroha/libvirt/hooks/qemu.d/arcade/prepare/begin/port_forward.sh
@@ -0,0 +1,14 @@
+#! /usr/bin/env bash
+
+# kdeconnect
+iptables -t nat -C PREROUTING -p tcp -s 192.168.1.0/24 --dport 1714:1764 \
+  -j DNAT --to-destination 192.168.122.40:1714-1764 || \
+iptables -t nat -A PREROUTING -p tcp -s 192.168.1.0/24 --dport 1714:1764 \
+  -j DNAT --to-destination 192.168.122.40:1714-1764
+iptables -t nat -C PREROUTING -p udp -s 192.168.1.0/24 --dport 1714:1764 \
+  -j DNAT --to-destination 192.168.122.40:1714-1764 || \
+iptables -t nat -A PREROUTING -p udp -s 192.168.1.0/24 --dport 1714:1764 \
+  -j DNAT --to-destination 192.168.122.40:1714-1764
+
+iptables -C FORWARD -m state -d 192.168.122.0/24 --state NEW,RELATED,ESTABLISHED -j ACCEPT || \
+iptables -I FORWARD -m state -d 192.168.122.0/24 --state NEW,RELATED,ESTABLISHED -j ACCEPT
diff --git a/hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/cpuset_release_common.sh b/hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/cpuset_release_common.sh
new file mode 120000
index 0000000..d823a60
--- /dev/null
+++ b/hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/cpuset_release_common.sh
@@ -0,0 +1 @@
+../../../../cpuset_release_common.sh
\ No newline at end of file
diff --git a/hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/port_close.sh b/hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/port_close.sh
new file mode 100755
index 0000000..0d58fa7
--- /dev/null
+++ b/hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/port_close.sh
@@ -0,0 +1,14 @@
+#! /usr/bin/env bash
+
+# kdeconnect (remove)
+iptables -t nat -C PREROUTING -p tcp -s 192.168.1.0/24 --dport 1714:1764 \
+  -j DNAT --to-destination 192.168.122.40:1714-1764 || \
+iptables -t nat -D PREROUTING -p tcp -s 192.168.1.0/24 --dport 1714:1764 \
+  -j DNAT --to-destination 192.168.122.40:1714-1764
+iptables -t nat -C PREROUTING -p udp -s 192.168.1.0/24 --dport 1714:1764 \
+  -j DNAT --to-destination 192.168.122.40:1714-1764 || \
+iptables -t nat -D PREROUTING -p udp -s 192.168.1.0/24 --dport 1714:1764 \
+  -j DNAT --to-destination 192.168.122.40:1714-1764
+
+iptables -C FORWARD -m state -d 192.168.122.0/24 --state NEW,RELATED,ESTABLISHED -j ACCEPT || \
+iptables -D FORWARD -m state -d 192.168.122.0/24 --state NEW,RELATED,ESTABLISHED -j ACCEPT
diff --git a/hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/unbind_vfio.sh b/hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/unbind_vfio.sh
new file mode 100755
index 0000000..c256d2a
--- /dev/null
+++ b/hosts/iroha/libvirt/hooks/qemu.d/arcade/release/end/unbind_vfio.sh
@@ -0,0 +1,17 @@
+#! /usr/bin/env bash
+
+source "/etc/libvirt/hooks/arcade.conf"
+
+virsh nodedev-reattach $VIRSH_GPU_VIDEO
+virsh nodedev-reattach $VIRSH_GPU_AUDIO
+virsh nodedev-reattach $VIRSH_NVME
+virsh nodedev-reattach $VIRSH_USB
+
+modprobe -r vfio_pci
+modprobe -r vfio_iommu_type1
+modprobe -r vfio
+
+modprobe nvidia-drm
+modprobe nvidia-modeset
+modprobe nvidia-uvm
+modprobe nvidia
-- 
cgit v1.2.3-101-g0448