diff --git a/features/vm-guest.nix b/features/vm-guest.nix index 475a221..ae4528e 100644 --- a/features/vm-guest.nix +++ b/features/vm-guest.nix @@ -43,19 +43,15 @@ config = lib.mkIf cfg.enable ( lib.mkMerge [ { - services.qemuGuest.enable = true; services.spice-vdagentd.enable = lib.mkIf (!cfg.headless) true; boot.kernelParams = lib.mkIf cfg.headless [ "console=ttyS0,115200" ]; + # 9p autoloads on first mount boot.initrd.availableKernelModules = [ "9p" "9pnet_virtio" ]; - boot.kernelModules = [ - "9p" - "9pnet_virtio" - ]; networking = { useDHCP = true; @@ -68,7 +64,6 @@ curl wget htop - sshfs ]; } diff --git a/hosts/ephvm/configuration.nix b/hosts/ephvm/configuration.nix index ec5de00..516d661 100644 --- a/hosts/ephvm/configuration.nix +++ b/hosts/ephvm/configuration.nix @@ -13,19 +13,38 @@ documentation.enable = false; environment.defaultPackages = [ ]; - # compressed qcow2, no channel copy + # qcow2, no channel copy; post-processed with parallel zstd on qcow2 v3 + # (~half the size of zlib v2, faster decompress) image.modules.qemu = { config, modulesPath, ... }: { system.build.image = lib.mkForce ( - import (modulesPath + "/../lib/make-disk-image.nix") { - inherit lib config pkgs; - inherit (config.virtualisation) diskSize; + let + rawImage = import (modulesPath + "/../lib/make-disk-image.nix") { + inherit lib config pkgs; + inherit (config.virtualisation) diskSize; + inherit (config.image) baseName; + format = "qcow2"; + copyChannel = false; + partitionTableType = "legacy"; + }; inherit (config.image) baseName; - format = "qcow2-compressed"; - copyChannel = false; - partitionTableType = "legacy"; - } + in + pkgs.runCommand baseName { nativeBuildInputs = [ pkgs.qemu-utils ]; } '' + mkdir -p $out + # qemu-img caps -m at 16 + cores="''${NIX_BUILD_CORES:-4}" + [ "$cores" -gt 0 ] || cores=4 + [ "$cores" -gt 16 ] && cores=16 + qemu-img convert \ + -f qcow2 \ + -O qcow2 \ + -c \ + -o compression_type=zstd \ + -m "$cores" \ + ${rawImage}/${baseName}.qcow2 \ + $out/${baseName}.qcow2 + '' ); }; @@ -70,7 +89,7 @@ features.neovim.dotfiles = inputs.nvim; # ensure .config exists with correct ownership before automount - systemd.tmpfiles.rules = [ "d /home/matej/.config 0755 matej users -" ]; + systemd.tmpfiles.rules = [ "d /home/matej/.config 0700 matej users -" ]; # TODO:(@janezicmatej) replace ssh with virtio-console (hvc0) when qemu 11.0 lands # https://www.mail-archive.com/qemu-devel@nongnu.org/msg1162844.html diff --git a/scripts/ephvm-run.sh b/scripts/ephvm-run.sh index 2048d05..533e7d4 100755 --- a/scripts/ephvm-run.sh +++ b/scripts/ephvm-run.sh @@ -27,14 +27,32 @@ info() { # globals for cleanup trap CLEANUP_OVERLAY="" +CLEANUP_TMPDIR="" QEMU_PID="" +VM_READY=false cleanup() { [ -n "$QEMU_PID" ] && kill "$QEMU_PID" 2>/dev/null && wait "$QEMU_PID" 2>/dev/null [ -n "$CLEANUP_OVERLAY" ] && rm -rf "$CLEANUP_OVERLAY" + # preserve tmpdir on abnormal exit so the qemu log survives for inspection + if [ -n "$CLEANUP_TMPDIR" ]; then + if [ "$VM_READY" = true ]; then + rm -rf "$CLEANUP_TMPDIR" + else + echo "qemu log preserved: $CLEANUP_TMPDIR/qemu.log" >&2 + fi + fi return 0 } trap cleanup EXIT +# returns 0 once the guest's sshd is speaking (first bytes are "SSH-") +awaiting_ssh_banner() { + local port="$1" + local banner + banner=$(timeout 2 bash -c "exec 3<>/dev/tcp/localhost/$port; head -c 4 <&3" 2>/dev/null) || return 1 + [ "$banner" = "SSH-" ] +} + usage() { cat </dev/null || die "qemu-system-x86_64 not found" - - local accel="tcg" - [ -r /dev/kvm ] && accel="kvm" + [ -r /dev/kvm ] || die "/dev/kvm not readable; kvm is required" # auto-allocate ssh port unless serial mode if [ "$serial" = false ] && [ -z "$ssh_port" ]; then @@ -128,28 +146,33 @@ main() { done fi - local nic_arg="user" + local nic_arg="user,model=virtio-net-pci" if [ -n "$ssh_port" ]; then - nic_arg="user,hostfwd=tcp::${ssh_port}-:22" + nic_arg="user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" fi local -a qemu_args=( qemu-system-x86_64 - -accel "$accel" + -accel kvm + -cpu host -m "$memory" -smp "$cpus" -drive "$drive_arg" + -device "virtio-blk-pci,drive=hd0" + -device virtio-rng-pci -nic "$nic_arg" -nographic + -sandbox "on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny" ) - if [ "$accel" != "tcg" ]; then - qemu_args+=(-cpu host) - fi - local fs_id=0 mount_path name tag for mount_path in "${mounts[@]}"; do + [ -e "$mount_path" ] || die "--mount path does not exist: $mount_path" mount_path=$(realpath "$mount_path") + # qemu parses -virtfs as csv, a comma in the path would inject options + case "$mount_path" in + *,*) die "--mount path may not contain commas: $mount_path" ;; + esac name=$(basename "$mount_path") tag="m_${name:0:29}" qemu_args+=( @@ -163,6 +186,9 @@ main() { mkdir -p "$CLAUDE_CONFIG_DIR" local claude_dir claude_dir=$(realpath "$CLAUDE_CONFIG_DIR") + case "$claude_dir" in + *,*) die "claude config dir may not contain commas: $claude_dir" ;; + esac qemu_args+=( -virtfs "local,path=$claude_dir,mount_tag=claude,security_model=none,id=fs${fs_id}" @@ -171,27 +197,38 @@ main() { fi info "---" - info "Accel: $accel" + [ -n "$ssh_port" ] && info "SSH: ssh -p $ssh_port matej@localhost" info "---" if [ "$serial" = true ]; then exec "${qemu_args[@]}" fi + CLEANUP_TMPDIR=$(mktemp -d) + local qemu_log="$CLEANUP_TMPDIR/qemu.log" + # start qemu in background and auto-ssh - "${qemu_args[@]}" &>/dev/null & + "${qemu_args[@]}" &>"$qemu_log" & QEMU_PID=$! + # throwaway ssh key (vm accepts any key via AuthorizedKeysCommand) + local ssh_key="$CLEANUP_TMPDIR/id_ed25519" + ssh-keygen -t ed25519 -f "$ssh_key" -N "" -q + info "waiting for vm (port $ssh_port)..." local attempts=0 - while ! (echo > /dev/tcp/localhost/"$ssh_port") 2>/dev/null; do + # poll for the real SSH banner, not TCP accept: qemu's user-mode nic + # accepts host-side the moment qemu starts, well before guest sshd is up + while ! awaiting_ssh_banner "$ssh_port"; do attempts=$((attempts + 1)) - [ $attempts -gt 60 ] && die "vm did not become ready in 60s" + [ $attempts -gt 120 ] && die "vm did not become ready in 60s" kill -0 "$QEMU_PID" 2>/dev/null || die "qemu exited unexpectedly" - sleep 1 + sleep 0.5 done + VM_READY=true ssh -p "$ssh_port" -t \ + -i "$ssh_key" \ -o StrictHostKeyChecking=no \ -o UserKnownHostsFile=/dev/null \ -o LogLevel=ERROR \