TSI: impersonated INET sockets always get SO_REUSEPORT, so a second VM's `listen()` silently shares the host port

// Guest-side probe for the libkrun TSI loopback-sharing question.
//
// Each VM runs one of these with a distinct role letter. It listens on
// the guest's own 127.0.0.1:6379, then connects to 127.0.0.1:6379 and
// prints who answered. In an isolated guest the answer is always its
// own role. Any other letter means the connection left this VM.
use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};
use std::thread;
use std::time::Duration;

fn main() {
    let role = std::env::args().nth(1).unwrap_or_else(|| "X".to_string());
    let listener = match TcpListener::bind("127.0.0.1:6379") {
        Ok(l) => l,
        Err(e) => {
            println!("PROBE {role}: bind 127.0.0.1:6379 failed: {e}");
            return;
        }
    };
    println!("PROBE {role}: listening on 127.0.0.1:6379");
    let served = role.clone();
    thread::spawn(move || {
        for stream in listener.incoming().flatten() {
            let mut stream = stream;
            let mut buf = [0u8; 64];
            let _ = stream.read(&mut buf);
            let _ = stream.write_all(format!("SERVED-BY-{served}\n").as_bytes());
        }
    });

    thread::sleep(Duration::from_secs(3));
    for attempt in 0..8 {
        match TcpStream::connect("127.0.0.1:6379") {
            Ok(mut s) => {
                let _ = s.write_all(b"ping\n");
                let _ = s.set_read_timeout(Some(Duration::from_secs(2)));
                let mut out = String::new();
                let _ = s.read_to_string(&mut out);
                println!("PROBE {role}: attempt {attempt} answered by {}", out.trim());
            }
            Err(e) => println!("PROBE {role}: attempt {attempt} connect failed: {e}"),
        }
        thread::sleep(Duration::from_millis(400));
    }
    println!("PROBE {role}: done");
}

/*
 * Minimal reproducer: does a guest's loopback listen/connect under
 * libkrun TSI reach the *host's* loopback, so that two concurrently
 * running VMs share one host port?
 *
 * Usage: tsi_loopback_repro <rootfs-dir> <role-letter>
 *
 * Run two of these at once with different role letters. Each guest
 * listens on its own 127.0.0.1:6379 and then connects to it. If a
 * guest is ever answered by the other guest's role letter, guest
 * loopback is not guest-private.
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include <unistd.h>
#include <libkrun.h>
#include <libkrun_init.h>

static bool push_to_stderr(void *userdata, KrunStr s)
{
    (void)userdata;
    fwrite(s.data, 1, s.len, stderr);
    return true;
}

static KrunVtableHandle init_stderr_writer = KRUN_VTABLE_HANDLE(
    KRUN_INIT_PUSH_STR_TYPE_TAG,
    ((KrunInitPushStrVtable){ .drop = NULL, .push = push_to_stderr }),
    NULL);

static KrunVtableHandle krun_stderr_writer = KRUN_VTABLE_HANDLE(
    KRUN_PUSH_STR_TYPE_TAG,
    ((KrunPushStrVtable){ .drop = NULL, .push = push_to_stderr }),
    NULL);

#define CHECK(call)                                                            \
    err = NULL;                                                                \
    call;                                                                      \
    if (err) {                                                                 \
        fprintf(stderr, "%s failed: ", #call);                                 \
        krun_error_message(err, &krun_stderr_writer);                          \
        fputc('\n', stderr);                                                   \
        krun_error_destroy(err);                                               \
        return 1;                                                              \
    }

int main(int argc, char **argv)
{
    KrunError err;
    if (argc < 3) {
        fprintf(stderr, "usage: %s <rootfs-dir> <role-letter>\n", argv[0]);
        return 2;
    }
    const char *rootfs = argv[1];
    const char *role = argv[2];

    CHECK(krun_init_log(-1, KRUN_LOG_LEVEL_WARN, KRUN_LOG_STYLE_AUTO, 0, &err));

    KrunFsOverlay overlay = krun_fs_overlay_new();
    KrunPayload payload = krun_payload_load_krunfw(&err);
    if (!payload) { fprintf(stderr, "loading krunfw payload failed\n"); return 1; }

    {
        KrunInitError ierr = NULL;
        KrunInitBuilder b = krun_init_config_builder();
        krun_init_builder_arg(&b, KRUN_STR("/probe"));
        krun_init_builder_arg(&b, KRUN_STR((char *)role));
        krun_init_builder_env_var(&b, KRUN_STR("PATH=/"));
        krun_init_builder_workdir(&b, KRUN_STR("/"));
        KrunInitConfig cfg = krun_init_builder_build(&b);
        krun_init_config_apply(cfg, overlay, payload, &ierr);
        if (ierr) {
            fprintf(stderr, "applying init config failed: ");
            krun_init_error_message(ierr, &init_stderr_writer);
            fputc('\n', stderr);
            krun_init_error_destroy(ierr);
            return 1;
        }
    }

    KrunMmioDeviceManager devices = krun_mmio_device_manager_new();
    {
        KrunConsoleBuilder cb = krun_console_device_builder();
        CHECK(krun_console_builder_add_default_console(cb, STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO, &err));
        KrunConsoleDevice console = krun_console_builder_build(cb, &err);
        krun_mmio_device_manager_add(devices, console);
    }
    {
        KrunFsDevice fs;
        CHECK(fs = krun_fs_device_new(KRUN_STR("/dev/root"), KRUN_STR((char *)rootfs), &err));
        krun_fs_device_set_overlay(fs, overlay);
        krun_mmio_device_manager_add(devices, fs);
    }
    {
        /* No net device; TSI hijacks the guest's INET sockets - the
         * exact configuration the krun sccache test boots on Linux. */
        KrunVsockDevice vsock;
        uint32_t tsi_flags = (argc > 3) ? (uint32_t)strtoul(argv[3], NULL, 0)
                                        : (KRUN_TSI_FLAGS_HIJACK_INET | KRUN_TSI_FLAGS_HIJACK_UNIX);
        fprintf(stderr, "tsi_flags=0x%x\n", tsi_flags);
        CHECK(vsock = krun_vsock_device_new(3, tsi_flags, &err));
        krun_mmio_device_manager_add(devices, vsock);
    }
    {
        KrunRngDevice rng;
        CHECK(rng = krun_rng_device_new(&err));
        krun_mmio_device_manager_add(devices, rng);
    }

    KrunVmmBuilder builder = krun_vmm_builder_new();
    CHECK(krun_vmm_builder_vcpus(&builder, 1, &err));
    CHECK(krun_vmm_builder_ram_mib(&builder, 512, &err));
    krun_vmm_builder_payload(&builder, payload);
    krun_vmm_builder_devices(&builder, devices);
    CHECK(krun_vmm_builder_split_irqchip(&builder, false, &err));

    KrunVmm vmm;
    CHECK(vmm = krun_vmm_builder_build(&builder, &err));
    krun_vmm_run(vmm);
    return 0;
}

/*
 * The same experiment as ../tsi_loopback_repro.c, written against the
 * libkrun 1.19.x API: the one Homebrew and the distributions ship. It
 * runs on a released library, macOS included.
 *
 * Usage: tsi_loopback_repro_legacy <rootfs-dir> <role-letter> [tsi_flags]
 *
 * tsi_flags defaults to KRUN_TSI_HIJACK_INET (HIJACK_UNIX is refused on
 * macOS). Pass 0 for the control: no impersonation at all.
 */
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <libkrun.h>

#define CHECK(call)                                                            \
    do {                                                                       \
        int32_t rc_ = (call);                                                  \
        if (rc_ < 0) {                                                         \
            fprintf(stderr, "%s failed: %d\n", #call, rc_);                    \
            return 1;                                                          \
        }                                                                      \
    } while (0)

int main(int argc, char **argv)
{
    if (argc < 3) {
        fprintf(stderr, "usage: %s <rootfs-dir> <role-letter> [tsi_flags]\n", argv[0]);
        return 2;
    }
    const char *rootfs = argv[1];
    const char *role = argv[2];
    uint32_t tsi_flags = (argc > 3) ? (uint32_t)strtoul(argv[3], NULL, 0)
                                    : KRUN_TSI_HIJACK_INET;
    fprintf(stderr, "role=%s tsi_flags=0x%x\n", role, tsi_flags);

    CHECK(krun_set_log_level(KRUN_LOG_LEVEL_WARN));
    int32_t ctx = krun_create_ctx();
    if (ctx < 0) {
        fprintf(stderr, "krun_create_ctx failed: %d\n", ctx);
        return 1;
    }

    CHECK(krun_set_vm_config(ctx, 1, 512));
    CHECK(krun_set_root(ctx, rootfs));
    /* No network device - the configuration in which libkrun's implicit
     * vsock would turn HIJACK_INET on by itself. Configure the device
     * explicitly instead, so the flags are the only thing that differs
     * between the experiment and the control. */
    CHECK(krun_disable_implicit_vsock(ctx));
    CHECK(krun_add_vsock(ctx, tsi_flags));
    CHECK(krun_set_workdir(ctx, "/"));

    /* argv excludes argv[0]; libkrun fills that in with the exec path. */
    const char *const exec_argv[] = { role, NULL };
    const char *const exec_envp[] = { "PATH=/", NULL };
    CHECK(krun_set_exec(ctx, "/probe", exec_argv, exec_envp));

    /* Only returns if the VM could not be started. */
    CHECK(krun_start_enter(ctx));
    return 0;
}

// Guest-side probe for the libkrun TSI loopback-sharing question.
//
// Same as ../../probe.rs, except that the connect has a timeout. Under
// TSI on macOS a guest connect to 127.0.0.1 is known to be able to wedge
// (libkrun/libkrun#344), and a probe stuck in connect() reports nothing.
//
// Each VM runs one of these with a distinct role letter. It listens on
// the guest's own 127.0.0.1:6379, then connects to 127.0.0.1:6379 and
// prints who answered. In an isolated guest the answer is always its
// own role. Any other letter means the connection left this VM.
use std::io::{Read, Write};
use std::net::{SocketAddr, TcpListener, TcpStream};
use std::thread;
use std::time::Duration;

fn main() {
    let role = std::env::args().nth(1).unwrap_or_else(|| "X".to_string());
    let listener = match TcpListener::bind("127.0.0.1:6379") {
        Ok(l) => l,
        Err(e) => {
            println!("PROBE {role}: bind 127.0.0.1:6379 failed: {e}");
            return;
        }
    };
    println!("PROBE {role}: listening on 127.0.0.1:6379");
    let served = role.clone();
    thread::spawn(move || {
        for stream in listener.incoming().flatten() {
            let mut stream = stream;
            let mut buf = [0u8; 64];
            let _ = stream.read(&mut buf);
            let _ = stream.write_all(format!("SERVED-BY-{served}\n").as_bytes());
        }
    });

    thread::sleep(Duration::from_secs(3));
    let addr: SocketAddr = "127.0.0.1:6379".parse().expect("a literal socket address");
    for attempt in 0..8 {
        match TcpStream::connect_timeout(&addr, Duration::from_secs(3)) {
            Ok(mut s) => {
                let _ = s.write_all(b"ping\n");
                let _ = s.set_read_timeout(Some(Duration::from_secs(2)));
                let mut out = String::new();
                let _ = s.read_to_string(&mut out);
                let who = match out.trim() {
                    "" => "nobody (connected, empty reply)",
                    other => other,
                };
                println!("PROBE {role}: attempt {attempt} answered by {who}");
            }
            Err(e) => println!("PROBE {role}: attempt {attempt} connect failed: {e}"),
        }
        thread::sleep(Duration::from_millis(400));
    }
    println!("PROBE {role}: done");
}

[package]
name = "probe"
version = "0.1.0"
edition = "2021"

[[bin]]
name = "probe"
path = "src/main.rs"

[profile.release]
opt-level = "s"

# Standalone: not a member of any enclosing workspace.
[workspace]

#!/usr/bin/env bash
# Build and run the TSI loopback experiment on macOS against Homebrew's
# libkrun (brew tap libkrun/krun && brew install libkrun).
#
#   ./run.sh           build, then run all three experiments
#
# 1. two VMs at once with HIJACK_INET - does a guest's connect to its own
#    127.0.0.1:6379 reach the other VM?
# 2. the same two VMs with flags 0 - the control
# 3. one VM alone with HIJACK_INET
#
# Each VM gets a hard deadline, because a TSI loopback connect on macOS
# is known to be able to wedge (libkrun/libkrun#344).
set -euo pipefail

HERE="$(cd "$(dirname "$0")" && pwd)"
cd "$HERE"
export PATH="$(brew --prefix)/bin:$HOME/.cargo/bin:$PATH"
LK="$(brew --prefix libkrun)"
LKFW="$(brew --prefix libkrunfw)"
DEADLINE=60
OUT="$HERE/out"

build() {
    # The guest probe cross-compiles, so the Rust toolchain in use needs the
    # musl target. Set RUSTUP_TOOLCHAIN to pick a toolchain that has it.
    if ! rustup target list --installed 2>/dev/null | grep -qx aarch64-unknown-linux-musl; then
        echo "the active Rust toolchain ($(rustup show active-toolchain 2>/dev/null | cut -d' ' -f1)) lacks" \
            "aarch64-unknown-linux-musl: run 'rustup target add aarch64-unknown-linux-musl'," \
            "or set RUSTUP_TOOLCHAIN to a toolchain that has it" >&2
        exit 1
    fi
    echo "== building the guest probe (aarch64-unknown-linux-musl)"
    (cd probe && cargo zigbuild --quiet --release --target aarch64-unknown-linux-musl)
    rm -rf rootfs
    mkdir -p rootfs/dev rootfs/proc rootfs/sys rootfs/tmp rootfs/etc
    cp probe/target/aarch64-unknown-linux-musl/release/probe rootfs/probe

    echo "== building and signing the host harness against $LK"
    cc -O2 -Wall -o tsi_repro_legacy tsi_loopback_repro_legacy.c \
        -I"$LK/include" -L"$LK/lib" -lkrun
    codesign --force --sign - --entitlements hv.entitlements tsi_repro_legacy
}

# run_vms <label> <flags> <role>...: start one VM per role at once, show
# the host's listeners on 6379 while they run, then collect the output.
run_vms() {
    local label="$1" flags="$2"
    shift 2
    local pids=() role
    mkdir -p "$OUT"
    echo
    echo "================ $label (tsi_flags=$flags, VMs: $*) ================"
    for role in "$@"; do
        DYLD_LIBRARY_PATH="$LK/lib:$LKFW/lib" ./tsi_repro_legacy "$HERE/rootfs" "$role" "$flags" \
            > "$OUT/$label-$role.log" 2>&1 &
        pids+=("$!")
    done
    sleep 2
    echo "-- host listeners on 6379, 2s in:"
    lsof -nP -iTCP:6379 -sTCP:LISTEN 2>/dev/null | sed 's/^/   /' || true
    local waited=0
    while [ "$waited" -lt "$DEADLINE" ]; do
        local alive=0 pid
        for pid in "${pids[@]}"; do kill -0 "$pid" 2>/dev/null && alive=1; done
        [ "$alive" = 0 ] && break
        sleep 1
        waited=$((waited + 1))
    done
    local pid
    for pid in "${pids[@]}"; do
        if kill -0 "$pid" 2>/dev/null; then
            echo "-- pid $pid still running after ${DEADLINE}s: killing it"
            kill -9 "$pid" 2>/dev/null || true
        fi
    done
    for role in "$@"; do
        echo "-- VM $role:"
        grep -E "PROBE|failed|error|tsi_flags" "$OUT/$label-$role.log" | sed 's/^/   /' || true
    done
}

build
run_vms pair-hijack 1 A B
run_vms pair-control 0 A B
run_vms single-hijack 1 A

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
    <key>com.apple.security.hypervisor</key>
    <true/>
</dict>
</plist>