#!/usr/bin/env python3
"""
NRP FABRIC SENSE — Cross-testbed experiment lifecycle script.

This weave establishes a Layer-2 network path between a FABRIC slice and a
Kubernetes pod on the NRP Nautilus cluster.

Two modes (controlled by SENSE_MODE env var):

  create   (default) — provision a new SENSE L2 path through ESnet, create NADs
                       via the SENSE operator, deploy pod via SENSE operator.

  existing           — skip SENSE path creation entirely. A SENSE path (or any
                       other L2 path) is assumed to already be up and has already
                       configured interface vlan.<VLAN> on the target NRP node.
                       This mode creates a NAD directly targeting that interface,
                       deploys a plain Kubernetes pod against it, and runs the
                       connectivity test. On stop, only the NAD/pod and the
                       FABRIC slice are deleted — the SENSE path is left untouched.

Architecture (create mode):
  FABRIC node (UCSD/STAR)
      └─ NIC → Facility Port (NRP-UCSD)
                    └─ SENSE L2 path (ESnet)
                              └─ Nautilus pod (node-2-6.sdsc.optiputer.net)

Prerequisites (configured in Settings → NRP Cluster):
  NRP_SA_TOKEN           — Kubernetes service account bearer token
  NRP_NAMESPACE          — Nautilus namespace (default: nrp-fabric-integration)
  NRP_DEFAULT_NODE       — Nautilus worker node hostname
  NRP_SENSE_OPERATOR_URL — SENSE operator REST endpoint (create mode only)
  NRP_KUBE_API_SERVER    — Nautilus Kubernetes API server URL

Weave args (set in Run dialog):
  SENSE_MODE      — "create" or "existing" (default: create)
  NRP_FABRIC_SITE — FABRIC site (default: STAR)
  NRP_FACILITY_PORT — Facility port name (default: NRP-UCSD)
  NRP_VLAN        — VLAN tag (required in existing mode; used for facility port
                    in create mode; default: any)
  FABRIC_NODE_IP  — IP for the FABRIC node interface (default: 192.168.1.3)
  NRP_POD_IP      — IP for the Nautilus pod interface (default: 192.168.1.2)

Usage:
  python3 nrp_sense.py start   SLICE_NAME
  python3 nrp_sense.py stop    SLICE_NAME
  python3 nrp_sense.py monitor SLICE_NAME
"""
import json
import os
import sys
import time
import warnings

import requests
import urllib3

# Suppress InsecureRequestWarning — NRP uses an internal CA that we skip
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
warnings.filterwarnings("ignore")

# ---------------------------------------------------------------------------
# Environment helpers
# ---------------------------------------------------------------------------

def _env(key: str, default: str = "") -> str:
    return os.environ.get(key, default).strip()

def _require_env(key: str) -> str:
    v = _env(key)
    if not v:
        print(f"ERROR: {key} is not set. Configure it in Settings → NRP Cluster.")
        sys.exit(1)
    return v


# ---------------------------------------------------------------------------
# SENSE operator helpers  (create mode only)
# ---------------------------------------------------------------------------

def _sense_headers(token: str) -> dict:
    return {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}


def _sense_create_path(base_url: str, token: str, path_name: str,
                       namespace: str, endpoint2: str, vlan: str) -> dict:
    """Create a SENSE L2 path between NRP-UCSD and the target Nautilus node."""
    data = {
        "name": path_name,
        "namespace": namespace,
        "endpoint1": "NRP-UCSD",
        "endpoint2": endpoint2,
        "bandwidth": 1000,
        "vlan": vlan,
    }
    r = requests.post(base_url, headers=_sense_headers(token),
                      json=data, verify=False, timeout=30)
    r.raise_for_status()
    return r.json()


def _sense_wait_ready(base_url: str, token: str, path_name: str,
                      namespace: str, timeout: int = 300) -> None:
    """Poll SENSE path status until Ready or timeout. Exits early on terminal failure."""
    deadline = time.time() + timeout
    last_reason = ""
    while time.time() < deadline:
        r = requests.get(
            f"{base_url}/{path_name}/status?namespace={namespace}&format=json",
            headers=_sense_headers(token), verify=False, timeout=15,
        )
        if r.ok:
            status = r.json()
            state = status.get("status") or status.get("state") or ""
            reason = (status.get("reason") or status.get("message")
                      or status.get("error") or status.get("detail") or "")
            if reason and reason != last_reason:
                print(f"### PROGRESS:   SENSE path reason: {str(reason)[:200]}")
                last_reason = reason
            print(f"### PROGRESS:   SENSE path state: {state}")
            if "ready" in state.lower():
                return
            if state.lower() in ("failed", "error", "deleted"):
                raise RuntimeError(
                    f"SENSE path entered terminal state '{state}'"
                    + (f": {str(reason)[:300]}" if reason else "")
                )
        time.sleep(10)
    raise TimeoutError("SENSE path did not become Ready within timeout")


def _sense_create_nads(base_url: str, token: str, path_name: str,
                       namespace: str) -> dict:
    """Create Network Attachment Definitions for the SENSE path."""
    r = requests.post(
        f"{base_url}/{path_name}/create-nad?namespace={namespace}",
        headers=_sense_headers(token), verify=False, timeout=30,
    )
    r.raise_for_status()
    return r.json()


def _sense_deploy_pod(base_url: str, token: str, path_name: str,
                      namespace: str) -> str:
    """Deploy a test pod attached to the SENSE path. Returns pod name."""
    r = requests.post(
        f"{base_url}/{path_name}/deploy-pod?namespace={namespace}",
        headers=_sense_headers(token), verify=False, timeout=30,
    )
    r.raise_for_status()
    result = r.json()
    return result["data"]["pods"][0]["pod_name"]


def _sense_delete_path(base_url: str, token: str, path_name: str,
                       namespace: str) -> None:
    requests.delete(
        f"{base_url}/{path_name}?namespace={namespace}",
        headers=_sense_headers(token), verify=False, timeout=30,
    )
    # 404 is fine — path may already be gone


# ---------------------------------------------------------------------------
# Kubernetes helpers
# ---------------------------------------------------------------------------

def _k8s_client():
    from kubernetes import client as k8s_client
    cfg = k8s_client.Configuration()
    cfg.host = _require_env("NRP_KUBE_API_SERVER")
    cfg.api_key = {"authorization": f"Bearer {_require_env('NRP_SA_TOKEN')}"}
    cfg.verify_ssl = False
    return k8s_client.ApiClient(cfg)


def _k8s_create_nad(namespace: str, nad_name: str, vlan: str) -> None:
    """Create a NetworkAttachmentDefinition targeting vlan.<VLAN> directly.

    Used in 'existing' mode where the VLAN interface is already up on the node
    (either from a pre-existing SENSE path or any other mechanism).
    """
    from kubernetes import client as k8s_client
    api = k8s_client.CustomObjectsApi(_k8s_client())
    nad = {
        "apiVersion": "k8s.cni.cncf.io/v1",
        "kind": "NetworkAttachmentDefinition",
        "metadata": {"name": nad_name, "namespace": namespace},
        "spec": {
            "config": json.dumps({
                "cniVersion": "0.3.1",
                "type": "macvlan",
                "master": f"vlan.{vlan}",
                "mode": "bridge",
                "ipam": {},
            })
        },
    }
    api.create_namespaced_custom_object(
        group="k8s.cni.cncf.io",
        version="v1",
        namespace=namespace,
        plural="network-attachment-definitions",
        body=nad,
    )


def _k8s_deploy_pod(namespace: str, pod_name: str, nad_name: str,
                    node_name: str) -> None:
    """Deploy a plain test pod with the NAD attached, pinned to node_name."""
    from kubernetes import client as k8s_client
    api = k8s_client.CoreV1Api(_k8s_client())
    pod = k8s_client.V1Pod(
        metadata=k8s_client.V1ObjectMeta(
            name=pod_name,
            namespace=namespace,
            annotations={"k8s.v1.cni.cncf.io/networks": nad_name},
        ),
        spec=k8s_client.V1PodSpec(
            node_selector={"kubernetes.io/hostname": node_name},
            restart_policy="Never",
            containers=[
                k8s_client.V1Container(
                    name="net-test",
                    image="nicolaka/netshoot",
                    command=["sleep", "infinity"],
                    # NET_ADMIN: required for ip addr add/flush on the macvlan net1 interface.
                    # NET_RAW: required for ping with raw ICMP sockets.
                    security_context=k8s_client.V1SecurityContext(
                        capabilities=k8s_client.V1Capabilities(
                            add=["NET_ADMIN", "NET_RAW"]
                        )
                    ),
                    resources=k8s_client.V1ResourceRequirements(
                        requests={"cpu": "100m", "memory": "128Mi"},
                        limits={"cpu": "500m", "memory": "256Mi"},
                    ),
                )
            ],
        ),
    )
    api.create_namespaced_pod(namespace=namespace, body=pod)


def _k8s_delete_nad(namespace: str, nad_name: str) -> None:
    from kubernetes import client as k8s_client
    api = k8s_client.CustomObjectsApi(_k8s_client())
    try:
        api.delete_namespaced_custom_object(
            group="k8s.cni.cncf.io", version="v1", namespace=namespace,
            plural="network-attachment-definitions", name=nad_name,
        )
    except Exception:
        pass


def _k8s_delete_pod(namespace: str, pod_name: str) -> None:
    from kubernetes import client as k8s_client
    api = k8s_client.CoreV1Api(_k8s_client())
    try:
        api.delete_namespaced_pod(name=pod_name, namespace=namespace)
    except Exception:
        pass


def _k8s_wait_pod_running(pod_name: str, namespace: str,
                           timeout: int = 180) -> None:
    from kubernetes import client as k8s_client
    api = k8s_client.CoreV1Api(_k8s_client())
    deadline = time.time() + timeout
    while time.time() < deadline:
        pod = api.read_namespaced_pod(name=pod_name, namespace=namespace)
        phase = pod.status.phase or "Unknown"
        print(f"### PROGRESS:   Pod phase: {phase}")
        if phase == "Running":
            return
        time.sleep(8)
    raise TimeoutError(f"Pod {pod_name} did not reach Running within timeout")


def _k8s_exec(pod_name: str, namespace: str, command: list) -> str:
    from kubernetes import client as k8s_client, stream
    api = k8s_client.CoreV1Api(_k8s_client())
    return stream.stream(
        api.connect_get_namespaced_pod_exec,
        pod_name, namespace,
        command=command,
        stderr=True, stdin=False, stdout=True, tty=False,
    )


def _configure_pod_interface(pod_name: str, namespace: str, ip: str) -> None:
    result = _k8s_exec(pod_name, namespace, ["/bin/sh", "-c",
        f"ip addr flush dev net1 2>&1; "
        f"ip addr add {ip}/24 dev net1 2>&1; "
        f"ip link set net1 up 2>&1; "
        f"echo 'CONFIG_DONE'; "
        f"ip addr show net1 2>&1"])
    # Always print the full interface state for visibility
    print(f"### PROGRESS:   net1 addrs:")
    for line in result.strip().splitlines():
        print(f"### PROGRESS:     {line}")
    if "Operation not permitted" in result or "CONFIG_DONE" not in result:
        print(f"### PROGRESS:   ERROR: pod interface config failed (no NET_ADMIN?): {result.strip()[-200:]}")
        sys.exit(1)
    elif ip not in result:
        print(f"### PROGRESS:   WARNING: {ip} not visible on net1 after config")
    else:
        print(f"### PROGRESS:   net1 assigned {ip}/24 — OK")


# ---------------------------------------------------------------------------
# State persistence
# ---------------------------------------------------------------------------

def _state_path(slice_name: str) -> str:
    return os.path.join(os.path.dirname(__file__), f".state-{slice_name}.json")


def _save_state(slice_name: str, state: dict) -> None:
    with open(_state_path(slice_name), "w") as f:
        json.dump(state, f, indent=2)


def _load_state(slice_name: str) -> dict:
    p = _state_path(slice_name)
    if os.path.exists(p):
        with open(p) as f:
            return json.load(f)
    return {}


def _clear_state(slice_name: str) -> None:
    p = _state_path(slice_name)
    if os.path.exists(p):
        os.remove(p)


# ---------------------------------------------------------------------------
# Lifecycle
# ---------------------------------------------------------------------------

def start(slice_name: str) -> None:
    from fabrictestbed_extensions.fablib.fablib import FablibManager

    token = _require_env("NRP_SA_TOKEN")
    namespace = _env("NRP_NAMESPACE", "nrp-fabric-integration")
    nrp_node = _env("NRP_DEFAULT_NODE", "node-2-6.sdsc.optiputer.net")
    sense_url = _env("NRP_SENSE_OPERATOR_URL",
                     "https://sense-operator.nrp-nautilus.io/api/v1/sensepaths")
    fabric_site = _env("NRP_FABRIC_SITE", "STAR")
    facility_port_name = _env("NRP_FACILITY_PORT", "NRP-UCSD")
    vlan = _env("NRP_VLAN", "any")
    fabric_ip = _env("FABRIC_NODE_IP", "192.168.1.3")
    nrp_ip = _env("NRP_POD_IP", "192.168.1.2")
    sense_mode = _env("SENSE_MODE", "create").lower()

    if sense_mode not in ("create", "existing"):
        print(f"### PROGRESS: ERROR — SENSE_MODE must be 'create' or 'existing', got '{sense_mode}'")
        sys.exit(1)

    if sense_mode == "existing" and (not vlan or vlan == "any"):
        print("### PROGRESS: ERROR — SENSE_MODE=existing requires a specific NRP_VLAN (not 'any').")
        sys.exit(1)

    import random, string
    exp_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6))
    path_name = f"fabric-sense-{exp_id}"   # only used in create mode
    nad_name = f"fabric-vlan-{vlan}-{exp_id}"

    # ── Step 1 — Create FABRIC slice ──────────────────────────────────────
    total_steps = 4 if sense_mode == "existing" else 5
    print(f"### PROGRESS: Step 1/{total_steps} — Creating FABRIC slice '{slice_name}' at {fabric_site}...")
    fablib = FablibManager()
    sl = fablib.new_slice(name=slice_name)

    node = sl.add_node(name="Node1", site=fabric_site,
                       cores=2, ram=8, disk=50, image="docker_ubuntu_20")
    node_iface = node.add_component(model="NIC_Basic", name="nic1").get_interfaces()[0]

    fp = sl.add_facility_port(name=facility_port_name, site="UCSD", vlan=vlan)
    fp.get_interfaces()[0]

    sl.add_l2network(name="net_nrp", interfaces=[node_iface, fp.get_interfaces()[0]])

    try:
        sl.submit()
    except Exception as e:
        err_str = str(e)
        if "Interface already used for L2VPN" in err_str or "already used for L2VPN" in err_str:
            print(f"### PROGRESS: ERROR — VLAN {vlan} is already in use on the FABRIC switch.")
            print(f"### PROGRESS:   Another L2VPN is already allocated on Bundle-Ether100.{vlan}.")
            print(f"### PROGRESS:   Use the VLAN Availability checker to find a free VLAN.")
        else:
            print(f"### PROGRESS: ERROR — Slice submission failed: {err_str[:300]}")
        sys.exit(1)

    print("### PROGRESS:   Slice submitted — waiting for SSH access...")
    try:
        sl.wait_ssh(progress=True)
    except Exception as e:
        print(f"### PROGRESS: ERROR — Timed out or failed waiting for slice SSH: {e}")
        sys.exit(1)

    node1 = sl.get_node(name="Node1")
    print(f"### PROGRESS:   Node1 ready: {node1.get_management_ip()}")

    # ── Step 2 — Configure FABRIC node interface ──────────────────────────
    print(f"### PROGRESS: Step 2/{total_steps} — Configuring FABRIC node interface ({fabric_ip})...")

    # get_os_interface() on the pre-submit object is empty — re-fetch from the live node.
    iface_name = node_iface.get_os_interface()
    if not iface_name:
        # Try re-fetching the interface from the running node object
        for iface in node1.get_interfaces():
            candidate = iface.get_os_interface()
            if candidate:
                iface_name = candidate
                break
    if not iface_name:
        # Last resort: detect via SSH — pick the first non-loopback, non-management iface
        detect_out, _ = node1.execute(
            "ip -o link show | awk -F': ' '{print $2}' | cut -d@ -f1 | grep -v '^lo$' | tail -n +2 | head -3",
            quiet=True,
        )
        ifaces = [l.strip() for l in detect_out.strip().splitlines() if l.strip()]
        # Management is usually eth0/ens3; data plane is typically the last or enp* one
        iface_name = next((i for i in ifaces if i.startswith("enp")), None) or (ifaces[-1] if ifaces else "enp7s0")
    print(f"### PROGRESS:   Interface: {iface_name}")

    stdout2, _ = node1.execute(
        f"sudo ip addr flush dev {iface_name} 2>&1; "
        f"sudo ip addr add {fabric_ip}/24 dev {iface_name} 2>&1; "
        f"sudo ip link set {iface_name} up 2>&1; "
        f"ip addr show {iface_name}",
        quiet=True,
    )
    for line in stdout2.strip().splitlines():
        print(f"### PROGRESS:   {line}")
    if fabric_ip not in stdout2:
        print(f"### PROGRESS:   WARNING: {fabric_ip} not visible on {iface_name} after config")

    # ── Step 3 — Create SENSE path  (create mode only) ────────────────────
    if sense_mode == "create":
        print(f"### PROGRESS: Step 3/{total_steps} — Creating SENSE path '{path_name}' to {nrp_node}...")
        try:
            _sense_create_path(sense_url, token, path_name, namespace, nrp_node, vlan)
        except Exception as e:
            print(f"### PROGRESS: ERROR — SENSE path creation failed: {e}")
            sys.exit(1)
        print("### PROGRESS:   Waiting for SENSE path to become Ready...")
        try:
            _sense_wait_ready(sense_url, token, path_name, namespace)
        except RuntimeError as e:
            print(f"### PROGRESS: ERROR — SENSE path failed: {e}")
            sys.exit(1)
        except TimeoutError:
            print(f"### PROGRESS: ERROR — SENSE path timed out after 5 minutes.")
            sys.exit(1)
        print("### PROGRESS:   SENSE path is Ready.")
    else:
        print(f"### PROGRESS: Step 3 skipped — using existing SENSE path on VLAN {vlan}.")

    # ── Step 3 or 4 — Deploy NAD + pod ────────────────────────────────────
    step_n = 4 if sense_mode == "create" else 3
    print(f"### PROGRESS: Step {step_n}/{total_steps} — Creating NAD and deploying Nautilus pod...")

    if sense_mode == "create":
        # SENSE operator manages NADs and pod
        try:
            _sense_create_nads(sense_url, token, path_name, namespace)
        except Exception as e:
            print(f"### PROGRESS: ERROR — SENSE NAD creation failed: {e}")
            sys.exit(1)
        try:
            pod_name = _sense_deploy_pod(sense_url, token, path_name, namespace)
        except Exception as e:
            print(f"### PROGRESS: ERROR — SENSE pod deploy failed: {e}")
            sys.exit(1)
    else:
        # existing mode — create NAD and pod directly via k8s API
        pod_name = f"fabric-test-{exp_id}"
        print(f"### PROGRESS:   Creating NAD '{nad_name}' targeting vlan.{vlan}...")
        try:
            _k8s_create_nad(namespace, nad_name, vlan)
        except Exception as e:
            print(f"### PROGRESS: ERROR — NAD creation failed: {e}")
            sys.exit(1)
        print(f"### PROGRESS:   Deploying pod '{pod_name}' on {nrp_node}...")
        try:
            _k8s_deploy_pod(namespace, pod_name, nad_name, nrp_node)
        except Exception as e:
            print(f"### PROGRESS: ERROR — Pod creation failed: {e}")
            print(f"### PROGRESS:   Cleaning up NAD '{nad_name}'...")
            _k8s_delete_nad(namespace, nad_name)
            sys.exit(1)

    print(f"### PROGRESS:   Pod '{pod_name}' created — waiting for Running state...")
    try:
        _k8s_wait_pod_running(pod_name, namespace)
    except TimeoutError as e:
        print(f"### PROGRESS: ERROR — {e}")
        sys.exit(1)

    # nicolaka/netshoot already ships with iproute2, iputils-ping, iperf3, traceroute —
    # no apt-get needed; just verify the binaries are present.
    print("### PROGRESS:   Verifying tools on NRP pod (netshoot image)...")
    verify_out = _k8s_exec(pod_name, namespace,
                           ["/bin/sh", "-c",
                            "which ip && which ping && which iperf3 && which traceroute && "
                            "echo 'TOOLS_OK'"])
    for line in verify_out.strip().splitlines():
        print(f"### PROGRESS:   [verify] {line}")
    if "TOOLS_OK" not in verify_out:
        print(f"### PROGRESS:   ERROR: expected tools missing from netshoot pod")
        sys.exit(1)
    print("### PROGRESS:   NRP pod tools verified OK")

    _configure_pod_interface(pod_name, namespace, nrp_ip)

    # ── Step 4 or 5 — Connectivity test ───────────────────────────────────
    step_n = 5 if sense_mode == "create" else 4
    print(f"### PROGRESS: Step {step_n}/{total_steps} — Testing connectivity FABRIC ↔ Nautilus...")

    # Install networking tools on FABRIC node
    print("### PROGRESS:   Installing tools on FABRIC node (iputils-ping, iperf3, traceroute)...")
    fab_install_out, _ = node1.execute(
        "export DEBIAN_FRONTEND=noninteractive; "
        "sudo apt-get update -q 2>&1 && "
        "sudo apt-get install -y iputils-ping iperf3 traceroute 2>&1 && "
        "echo BINARIES: && which ping && which iperf3 && which traceroute && "
        "echo INSTALL_OK",
        quiet=True,
    )
    for line in fab_install_out.strip().splitlines():
        print(f"### PROGRESS:   [fab-apt] {line}")
    if "INSTALL_OK" not in fab_install_out:
        print("### PROGRESS:   WARNING: FABRIC node tool install may have failed")
    else:
        print("### PROGRESS:   FABRIC node tools installed OK")

    # ── Ping FABRIC → NRP ───────────────────────────────────────────────
    print(f"### PROGRESS: --- Ping FABRIC → NRP ({fabric_ip} → {nrp_ip}) ---")
    stdout, _ = node1.execute(f"ping -I {fabric_ip} -c 4 {nrp_ip}", quiet=True)
    for line in stdout.strip().splitlines():
        print(f"### PROGRESS:   {line}")
    if "0 received" in stdout or "unreachable" in stdout.lower() or not stdout.strip():
        print("### PROGRESS:   WARNING: Ping FABRIC → NRP got no response (path may still be stabilising)")
    else:
        print("### PROGRESS:   Ping FABRIC → NRP: OK")

    # ── Ping NRP → FABRIC ───────────────────────────────────────────────
    # Use source IP bind (-I {nrp_ip}) rather than interface name to ensure
    # the correct source address is used on the macvlan interface.
    print(f"### PROGRESS: --- Ping NRP → FABRIC ({nrp_ip} → {fabric_ip}) ---")
    result = _k8s_exec(pod_name, namespace,
                       ["/bin/sh", "-c", f"ping -I {nrp_ip} -c 4 {fabric_ip} 2>&1"])
    for line in result.strip().splitlines():
        print(f"### PROGRESS:   {line}")
    if ("0 received" in result or "unreachable" in result.lower()
            or "not found" in result or "cannot assign" in result.lower()
            or not result.strip()):
        print("### PROGRESS:   WARNING: Ping NRP → FABRIC got no response")
    else:
        print("### PROGRESS:   Ping NRP → FABRIC: OK")

    # ── iperf3 bandwidth test ────────────────────────────────────────────
    print(f"### PROGRESS: --- iperf3 bandwidth test (FABRIC → NRP, 10 s) ---")
    try:
        # Start iperf3 server on NRP pod — bind to all interfaces so it works
        # regardless of which interface the IP lands on. Avoid sleep in the exec
        # to prevent websocket timeout; verify with kill -0 instead.
        srv_out = _k8s_exec(pod_name, namespace,
                            ["/bin/sh", "-c",
                             "pkill -f 'iperf3 -s' 2>/dev/null || true; "
                             "iperf3 -s >/tmp/iperf3.log 2>&1 & "
                             "IPID=$!; sleep 1; "
                             "kill -0 $IPID 2>/dev/null && echo SERVER_STARTED || "
                             "(echo SERVER_FAILED; cat /tmp/iperf3.log)"])
        print(f"### PROGRESS:   iperf3 server: {srv_out.strip() or '(no output)'}")
        if "SERVER_STARTED" not in srv_out:
            print("### PROGRESS:   WARNING: iperf3 server may not have started")
        time.sleep(1)
        # Run iperf3 client from FABRIC node, 10-second test with 2-second intervals
        print(f"### PROGRESS:   Running iperf3 client: {fabric_ip} → {nrp_ip} (10 s)...")
        iperf_out, iperf_err = node1.execute(
            f"iperf3 -c {nrp_ip} -B {fabric_ip} -t 10 -i 2 2>&1",
            quiet=True,
        )
        combined = (iperf_out + iperf_err).strip()
        if combined:
            for line in combined.splitlines():
                print(f"### PROGRESS:   {line}")
            if "sender" in combined or "receiver" in combined:
                print("### PROGRESS:   iperf3: PASS")
            else:
                print("### PROGRESS:   iperf3: no sender/receiver line (may have failed)")
        else:
            print("### PROGRESS:   iperf3 produced no output (server may not have started in time)")
        # Also dump iperf3 server log for diagnostics
        srv_log = _k8s_exec(pod_name, namespace,
                            ["/bin/sh", "-c", "cat /tmp/iperf3.log 2>/dev/null"])
        if srv_log.strip():
            print("### PROGRESS:   iperf3 server log:")
            for line in srv_log.strip().splitlines():
                print(f"### PROGRESS:     {line}")
    except Exception as e:
        print(f"### PROGRESS:   iperf3 test failed: {e}")

    # ── Traceroute FABRIC → NRP ──────────────────────────────────────────
    print(f"### PROGRESS: --- Traceroute FABRIC → NRP ---")
    try:
        tr_out, _ = node1.execute(f"traceroute -n -m 10 {nrp_ip}", quiet=True)
        for line in tr_out.strip().splitlines():
            print(f"### PROGRESS:   {line}")
    except Exception as e:
        print(f"### PROGRESS:   Traceroute failed: {e}")

    # ── Save state for monitor/stop ────────────────────────────────────────
    _save_state(slice_name, {
        "slice_name": slice_name,
        "sense_mode": sense_mode,
        "path_name": path_name if sense_mode == "create" else "",
        "nad_name": nad_name if sense_mode == "existing" else "",
        "pod_name": pod_name,
        "namespace": namespace,
        "vlan": vlan,
        "fabric_ip": fabric_ip,
        "nrp_ip": nrp_ip,
        "iface_name": iface_name,
        "node_management_ip": str(node1.get_management_ip()),
        "kube_api_server": _env("NRP_KUBE_API_SERVER"),
    })

    print("")
    print("### PROGRESS: READY! Experiment is running.")
    print(f"  FABRIC node:   {node1.get_management_ip()} ({fabric_ip} on {iface_name})")
    print(f"  Nautilus pod:  {pod_name} ({nrp_ip} on net1)")
    if sense_mode == "create":
        print(f"  SENSE path:    {path_name}")
    else:
        print(f"  NAD:           {nad_name} (vlan.{vlan})")
    print(f"  Namespace:     {namespace}")
    print(f"  Mode:          {sense_mode}")
    print("")
    print("  Open notebooks/tutorial.ipynb in JupyterLab for the interactive walkthrough.")
    print("  Click Stop to tear down the experiment.")


def stop(slice_name: str) -> None:
    """Tear down the experiment.

    In 'create' mode: delete SENSE path (which also cleans up pod/NADs), then FABRIC slice.
    In 'existing' mode: delete only the pod and NAD we created, then FABRIC slice.
                        The SENSE path is left untouched.
    """
    from fabrictestbed_extensions.fablib.fablib import FablibManager

    state = _load_state(slice_name)
    sense_mode = state.get("sense_mode", "create")
    token = _env("NRP_SA_TOKEN")
    sense_url = _env("NRP_SENSE_OPERATOR_URL",
                     "https://sense-operator.nrp-nautilus.io/api/v1/sensepaths")
    namespace = state.get("namespace") or _env("NRP_NAMESPACE", "nrp-fabric-integration")

    if sense_mode == "create":
        path_name = state.get("path_name", "")
        if path_name and token:
            print(f"### PROGRESS: Deleting SENSE path '{path_name}'...")
            try:
                _sense_delete_path(sense_url, token, path_name, namespace)
                print("### PROGRESS:   SENSE path deleted.")
            except Exception as e:
                print(f"### PROGRESS:   SENSE delete: {e} (continuing)")
    else:
        # existing mode — delete only what we created (NAD + pod), leave SENSE path alone
        nad_name = state.get("nad_name", "")
        pod_name = state.get("pod_name", "")
        if pod_name:
            print(f"### PROGRESS: Deleting pod '{pod_name}'...")
            _k8s_delete_pod(namespace, pod_name)
            print("### PROGRESS:   Pod deleted.")
        if nad_name:
            print(f"### PROGRESS: Deleting NAD '{nad_name}'...")
            _k8s_delete_nad(namespace, nad_name)
            print("### PROGRESS:   NAD deleted.")
        print("### PROGRESS:   SENSE path left intact (existing mode).")

    # Always delete the FABRIC slice
    print(f"### PROGRESS: Deleting FABRIC slice '{slice_name}'...")
    try:
        fablib = FablibManager()
        sl = fablib.get_slice(name=slice_name)
        sl.delete()
        print(f"### PROGRESS:   Slice '{slice_name}' deleted.")
    except Exception as e:
        print(f"### PROGRESS:   Slice not found or already deleted: {e}")

    _clear_state(slice_name)
    print("### PROGRESS: Cleanup complete.")


def monitor(slice_name: str) -> None:
    """Check FABRIC slice state and Nautilus pod health.

    Transient network errors (DNS failures, connection timeouts) are retried
    up to 3 times before being treated as a real failure, to avoid tearing
    down a healthy experiment due to a momentary blip.
    """
    from fabrictestbed_extensions.fablib.fablib import FablibManager

    state = _load_state(slice_name)
    if not state:
        print(f"ERROR: No state found for slice '{slice_name}'")
        sys.exit(1)

    # Retry loop for transient connectivity errors
    last_exc = None
    for attempt in range(3):
        try:
            fablib = FablibManager()
            sl = fablib.get_slice(name=slice_name)
            sl_state = str(sl.get_state())
            break
        except (ConnectionError, OSError) as e:
            last_exc = e
            print(f"### PROGRESS: Network blip on monitor attempt {attempt + 1}/3: {e} — retrying...")
            time.sleep(10)
    else:
        print(f"### PROGRESS: WARNING — could not reach FABRIC API after 3 attempts ({last_exc}); assuming healthy.")
        print(f"### PROGRESS: Healthy — FABRIC slice unknown (network unreachable), pod assumed Running.")
        return

    if "StableOK" not in sl_state:
        print(f"ERROR: FABRIC slice state is {sl_state}")
        sys.exit(1)

    token = _env("NRP_SA_TOKEN")
    namespace = state.get("namespace", "nrp-fabric-integration")
    pod_name = state.get("pod_name", "")
    if pod_name and token:
        from kubernetes import client as k8s_client
        api = k8s_client.CoreV1Api(_k8s_client())
        pod = api.read_namespaced_pod(name=pod_name, namespace=namespace)
        phase = pod.status.phase or "Unknown"
        if phase != "Running":
            print(f"ERROR: Nautilus pod phase is {phase}")
            sys.exit(1)

    print(f"### PROGRESS: Healthy — FABRIC slice {sl_state}, pod Running.")


# ---------------------------------------------------------------------------

def _resolve_slice_name(action: str) -> str:
    """Return slice name from argv, then SLICE_NAME env, then auto-detect from saved state."""
    if len(sys.argv) >= 3:
        return sys.argv[2]
    env_name = os.environ.get("SLICE_NAME", "").strip()
    if env_name:
        return env_name
    script_dir = os.path.dirname(os.path.abspath(__file__))
    state_files = sorted(
        f for f in os.listdir(script_dir) if f.startswith(".state-") and f.endswith(".json")
    )
    if len(state_files) == 1:
        name = state_files[0][len(".state-"):-len(".json")]
        print(f"Auto-detected slice: {name}")
        return name
    if len(state_files) > 1:
        names = [f[len(".state-"):-len(".json")] for f in state_files]
        print(f"Multiple slices found: {', '.join(names)}")
        print(f"Usage: nrp_sense.py {action} SLICE_NAME")
        sys.exit(1)
    if action == "start":
        print("Usage: nrp_sense.py start SLICE_NAME")
        sys.exit(1)
    print(f"No saved state found. Usage: nrp_sense.py {action} SLICE_NAME")
    sys.exit(1)


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: nrp_sense.py {start|stop|monitor} [SLICE_NAME]")
        sys.exit(1)
    action = sys.argv[1]
    name = _resolve_slice_name(action)
    if action == "start":
        start(name)
    elif action == "stop":
        stop(name)
    elif action == "monitor":
        monitor(name)
    else:
        print(f"Unknown action: {action}")
        sys.exit(1)
