fixed plots

.gitignore

@@ -15,7 +15,6 @@
 *.log
 *.lot
 *.out
-*.pdf
 *.run.xml
 *.synctex.gz
 *.toc
@@ -23,8 +22,5 @@
 *.nav
 *.snm
 *.mp4
-*.png
-*.jpg
-*.jpeg
 *.npy
 

Deep-SAD-PyTorch/devenv.lock

@@ -0,0 +1,103 @@
+{
+  "nodes": {
+    "devenv": {
+      "locked": {
+        "dir": "src/modules",
+        "lastModified": 1754730435,
+        "owner": "cachix",
+        "repo": "devenv",
+        "rev": "d1388a093a7225c2abe8c244109c5a4490de4077",
+        "type": "github"
+      },
+      "original": {
+        "dir": "src/modules",
+        "owner": "cachix",
+        "repo": "devenv",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1747046372,
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1754416808,
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "9c52372878df6911f9afc1e2a1391f55e4dfc864",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "git-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1754299112,
+        "owner": "cachix",
+        "repo": "devenv-nixpkgs",
+        "rev": "16c21c9f5c6fb978466e91182a248dd8ca1112ac",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "ref": "rolling",
+        "repo": "devenv-nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "devenv": "devenv",
+        "git-hooks": "git-hooks",
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks": [
+          "git-hooks"
+        ]
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

Deep-SAD-PyTorch/devenv.nix

@@ -0,0 +1,26 @@
+{ pkgs, ... }:
+let
+  native_dependencies = with pkgs.python312Packages; [
+    torch-bin
+    torchvision-bin
+    aggdraw # for visualtorch
+    nvidia-ml-py
+  ];
+  tools = with pkgs; [
+    ruff
+    dmidecode
+  ];
+in
+{
+  packages = native_dependencies ++ tools;
+  languages.python = {
+    enable = true;
+    package = pkgs.python312;
+    uv = {
+      enable = true;
+      sync.enable = true;
+    };
+    venv.enable = true;
+  };
+
+}

Deep-SAD-PyTorch/devenv.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://devenv.sh/devenv.schema.json
+inputs:
+  nixpkgs:
+    url: github:cachix/devenv-nixpkgs/rolling
+
+allowUnfree: true
+cudaSupport: true
+# If you're using non-OSS software, you can set allowUnfree to true.
+# allowUnfree: true
+
+# If you're willing to use a package that's vulnerable
+# permittedInsecurePackages:
+#  - "openssl-1.1.1w"
+
+# If you have more than one devenv you can merge them
+#imports:
+# - ./backend

Deep-SAD-PyTorch/hardware_survey/hardware_survey_deepio.tex

@@ -0,0 +1,54 @@
+
+% ---- Add to your LaTeX preamble ----
+% \usepackage{booktabs}
+% \usepackage{array}
+% ------------------------------------
+\begin{table}[p]
+\centering
+\caption{Computational Environment (Hardware \& Software)} \label{tab:system_setup}
+\begin{tabular}{p{0.34\linewidth} p{0.62\linewidth}}
+\toprule
+\textbf{Item} & \textbf{Details} \\
+\midrule
+\multicolumn{2}{l}{\textbf{System}} \\
+Operating System & \ttfamily Ubuntu 22.04.5 LTS \\
+Kernel & \ttfamily 6.5.0-44-generic \\
+Architecture & \ttfamily x86\_64 \\
+CPU Model & \ttfamily AMD Ryzen Threadripper 3970X 32-Core Processor \\
+CPU Cores (physical) & \ttfamily 32 × 1 \\
+CPU Threads (logical) & \ttfamily 64 \\
+CPU Base Frequency & \ttfamily 2200 MHz \\
+CPU Max Frequency & \ttfamily 3700 MHz \\
+Total RAM & \ttfamily 94.14 GiB \\
+\addlinespace
+\multicolumn{2}{l}{\textbf{GPU (Selected Newer Device)}} \\
+Selected GPU Name & \ttfamily NVIDIA GeForce RTX 4090 \\
+Selected GPU Memory & \ttfamily 23.99 GiB \\
+Selected GPU Compute Capability & \ttfamily 8.9 \\
+NVIDIA Driver Version & \ttfamily 535.161.07 \\
+CUDA (Driver) Version & \ttfamily 12.2 \\
+\addlinespace
+\multicolumn{2}{l}{\textbf{Software Environment}} \\
+Python & \ttfamily 3.11.13 \\
+PyTorch & \ttfamily 2.7.1+cu126 \\
+PyTorch Built CUDA & \ttfamily 12.6 \\
+cuDNN (PyTorch build) & \ttfamily 90501 \\
+scikit-learn & \ttfamily 1.7.0 \\
+NumPy & \ttfamily 2.3.0 \\
+SciPy & \ttfamily 1.15.3 \\
+NumPy Build Config & \begin{minipage}[t]{\linewidth}\ttfamily\small "blas": \{
+      "name": "scipy-openblas",
+      "include directory": "/opt/\_internal/cpython-3.11.12/lib/python3.11/site-packages/scipy\_openblas64/include",
+      "lib directory": "/opt/\_internal/cpython-3.11.12/lib/python3.11/site-packages/scipy\_openblas64/lib",
+      "openblas configuration": "OpenBLAS 0.3.29  USE64BITINT DYNAMIC\_ARCH NO\_AFFINITY Haswell MAX\_THREADS=64",
+      "pc file directory": "/project/.openblas"
+    "lapack": \{
+      "name": "scipy-openblas",
+      "include directory": "/opt/\_internal/cpython-3.11.12/lib/python3.11/site-packages/scipy\_openblas64/include",
+      "lib directory": "/opt/\_internal/cpython-3.11.12/lib/python3.11/site-packages/scipy\_openblas64/lib",
+      "openblas configuration": "OpenBLAS 0.3.29  USE64BITINT DYNAMIC\_ARCH NO\_AFFINITY Haswell MAX\_THREADS=64",
+      "pc file directory": "/project/.openblas"\end{minipage} \\
+\addlinespace
+\bottomrule
+\end{tabular}
+\end{table}

Deep-SAD-PyTorch/hardware_survey/main.py

@@ -0,0 +1,501 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+Generate a LaTeX longtable with system + software info for a thesis (Linux + NVIDIA).
+
+Requirements (preflight will check and error if missing):
+- Linux OS
+- lscpu (util-linux)
+- Python packages: nvidia-ml-py3 (pynvml), torch, numpy, scipy, scikit-learn
+- NVIDIA driver present and at least one GPU visible via NVML
+
+What it reports (per user’s list):
+System:
+- OS name + version + distribution (Linux) + kernel version + system arch
+- CPU model name, number of cores and threads, base frequencies (best-effort via lscpu)
+- Total RAM capacity
+- GPU(s): model name (only the newer one; prefer a name matching “4090”, else highest compute capability),
+  memory size, driver version, CUDA (driver) version, cuDNN version (if used via PyTorch)
+
+Software environment:
+- Python version
+- PyTorch version + built CUDA/cuDNN version
+- scikit-learn version
+- NumPy / SciPy version (+ NumPy build config summary: MKL/OpenBLAS/etc.)
+"""
+
+import argparse
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+from typing import Dict, List, Tuple
+
+# -------------------- Helper --------------------
+
+
+def _import_nvml():
+    """
+    Try to import NVML from the supported packages:
+    - 'nvidia-ml-py' (preferred, maintained): provides module 'pynvml'
+    - legacy 'pynvml' (deprecated but still widely installed)
+    Returns the imported module object (with nvml... symbols).
+    """
+    try:
+        import pynvml as _nvml  # provided by 'nvidia-ml-py' or old 'pynvml'
+
+        return _nvml
+    except Exception as e:
+        raise ImportError(
+            "NVML not importable. Please install the maintained package:\n"
+            "  pip install nvidia-ml-py\n"
+            "(and uninstall deprecated ones: pip uninstall nvidia-ml-py3 pynvml)"
+        ) from e
+
+
+def _to_text(x) -> str:
+    """Return a clean str whether NVML gives bytes or str."""
+    if isinstance(x, bytes):
+        try:
+            return x.decode(errors="ignore")
+        except Exception:
+            return x.decode("utf-8", "ignore")
+    return str(x)
+
+
+# -------------------- Utilities --------------------
+
+
+def which(cmd: str) -> str:
+    return shutil.which(cmd) or ""
+
+
+def run(cmd: List[str], timeout: int = 6) -> str:
+    try:
+        out = subprocess.check_output(cmd, stderr=subprocess.STDOUT, timeout=timeout)
+        return out.decode(errors="ignore").strip()
+    except Exception:
+        return ""
+
+
+def human_bytes(nbytes: int) -> str:
+    try:
+        n = float(nbytes)
+    except Exception:
+        return ""
+    units = ["B", "KiB", "MiB", "GiB", "TiB"]
+    i = 0
+    while n >= 1024 and i < len(units) - 1:
+        n /= 1024.0
+        i += 1
+    return f"{n:.2f} {units[i]}"
+
+
+LATEX_SPECIALS = {
+    "\\": r"\textbackslash{}",
+    "&": r"\&",
+    "%": r"\%",
+    "$": r"\$",
+    "#": r"\#",
+    "_": r"\_",
+    "{": r"\{",
+    "}": r"\}",
+    "~": r"\textasciitilde{}",
+    "^": r"\textasciicircum{}",
+}
+
+
+def tex_escape(s: str) -> str:
+    if s is None:
+        return ""
+    return "".join(LATEX_SPECIALS.get(ch, ch) for ch in str(s))
+
+
+def latex_table(sections: List[Tuple[str, Dict[str, str]]], caption: str) -> str:
+    lines = []
+    lines.append(r"\begin{table}[p]")  # float; use [p] or [tbp] as you prefer
+    lines.append(r"\centering")
+    lines.append(r"\caption{" + tex_escape(caption) + r"} \label{tab:system_setup}")
+    lines.append(r"\begin{tabular}{p{0.34\linewidth} p{0.62\linewidth}}")
+    lines.append(r"\toprule")
+    lines.append(r"\textbf{Item} & \textbf{Details} \\")
+    lines.append(r"\midrule")
+
+    for title, kv in sections:
+        if not kv:
+            continue
+        lines.append(r"\multicolumn{2}{l}{\textbf{" + tex_escape(title) + r"}} \\")
+        for k, v in kv.items():
+            val = tex_escape(v)
+            if "\n" in v or len(v) > 120:
+                val = (
+                    r"\begin{minipage}[t]{\linewidth}\ttfamily\small "
+                    + tex_escape(v)
+                    + r"\end{minipage}"
+                )
+            else:
+                val = r"\ttfamily " + val
+            lines.append(tex_escape(k) + " & " + val + r" \\")
+        lines.append(r"\addlinespace")
+
+    lines.append(r"\bottomrule")
+    lines.append(r"\end{tabular}")
+    lines.append(r"\end{table}")
+
+    preamble_hint = r"""
+% ---- Add to your LaTeX preamble ----
+% \usepackage{booktabs}
+% \usepackage{array}
+% ------------------------------------
+"""
+    return preamble_hint + "\n".join(lines)
+
+
+def latex_longtable(sections: List[Tuple[str, Dict[str, str]]], caption: str) -> str:
+    lines = []
+    lines.append(r"\begin{longtable}{p{0.34\linewidth} p{0.62\linewidth}}")
+    lines.append(r"\caption{" + tex_escape(caption) + r"} \label{tab:system_setup}\\")
+    lines.append(r"\toprule")
+    lines.append(r"\textbf{Item} & \textbf{Details} \\")
+    lines.append(r"\midrule")
+    lines.append(r"\endfirsthead")
+    lines.append(r"\toprule \textbf{Item} & \textbf{Details} \\ \midrule")
+    lines.append(r"\endhead")
+    lines.append(r"\bottomrule")
+    lines.append(r"\endfoot")
+    lines.append(r"\bottomrule")
+    lines.append(r"\endlastfoot")
+
+    for title, kv in sections:
+        if not kv:
+            continue
+        lines.append(r"\multicolumn{2}{l}{\textbf{" + tex_escape(title) + r"}} \\")
+        for k, v in kv.items():
+            val = tex_escape(v)
+            if "\n" in v or len(v) > 120:
+                val = (
+                    r"\begin{minipage}[t]{\linewidth}\ttfamily\small "
+                    + tex_escape(v)
+                    + r"\end{minipage}"
+                )
+            else:
+                val = r"\ttfamily " + val
+            lines.append(tex_escape(k) + " & " + val + r" \\")
+        lines.append(r"\addlinespace")
+    lines.append(r"\end{longtable}")
+
+    preamble_hint = r"""
+% ---- Add to your LaTeX preamble ----
+% \usepackage{booktabs}
+% \usepackage{longtable}
+% \usepackage{array}
+% ------------------------------------
+"""
+    return preamble_hint + "\n".join(lines)
+
+
+# -------------------- Preflight --------------------
+
+REQUIRED_CMDS = ["lscpu"]
+REQUIRED_MODULES = [
+    "torch",
+    "numpy",
+    "scipy",
+    "sklearn",
+    "pynvml",
+]  # provided by nvidia-ml-py
+
+
+def preflight() -> List[str]:
+    errors = []
+    if platform.system().lower() != "linux":
+        errors.append(
+            f"This script supports Linux only (detected: {platform.system()})."
+        )
+
+    for c in ["lscpu"]:
+        if not which(c):
+            errors.append(f"Missing required command: {c}")
+
+    for m in REQUIRED_MODULES:
+        try:
+            __import__(m)
+        except Exception:
+            errors.append(f"Missing required Python package: {m}")
+
+    # NVML driver availability
+    if "pynvml" not in errors:
+        try:
+            pynvml = _import_nvml()
+            pynvml.nvmlInit()
+            count = pynvml.nvmlDeviceGetCount()
+            if count < 1:
+                errors.append("No NVIDIA GPUs detected by NVML.")
+            pynvml.nvmlShutdown()
+        except Exception as e:
+            errors.append(f"NVIDIA NVML not available / driver not loaded: {e}")
+
+    return errors
+
+
+# -------------------- Collectors --------------------
+
+
+def collect_system() -> Dict[str, str]:
+    info: Dict[str, str] = {}
+
+    # OS / distro / kernel / arch
+    os_pretty = ""
+    try:
+        with open("/etc/os-release", "r") as f:
+            txt = f.read()
+        m = re.search(r'^PRETTY_NAME="?(.*?)"?$', txt, flags=re.M)
+        if m:
+            os_pretty = m.group(1)
+    except Exception:
+        pass
+    info["Operating System"] = os_pretty or f"{platform.system()} {platform.release()}"
+    info["Kernel"] = platform.release()
+    info["Architecture"] = platform.machine()
+
+    # CPU (via lscpu)
+    lscpu = run(["lscpu"])
+
+    def kvs(text: str) -> Dict[str, str]:
+        out = {}
+        for line in text.splitlines():
+            if ":" in line:
+                k, v = line.split(":", 1)
+                out[k.strip()] = v.strip()
+        return out
+
+    d = kvs(lscpu)
+    info["CPU Model"] = d.get("Model name", d.get("Model Name", ""))
+
+    # cores / threads
+    sockets = d.get("Socket(s)", "")
+    cores_per_socket = d.get("Core(s) per socket", "")
+    threads_total = d.get("CPU(s)", "")
+    if sockets and cores_per_socket:
+        info["CPU Cores (physical)"] = f"{cores_per_socket} × {sockets}"
+    else:
+        info["CPU Cores (physical)"] = cores_per_socket or ""
+    info["CPU Threads (logical)"] = threads_total or str(os.cpu_count() or "")
+
+    # base / max freq
+    # Prefer "CPU max MHz" and "CPU min MHz"; lscpu sometimes exposes "CPU MHz" (current)
+    base = d.get("CPU min MHz", "")
+    maxf = d.get("CPU max MHz", "")
+    if base:
+        info["CPU Base Frequency"] = f"{float(base):.0f} MHz"
+    elif "@" in info["CPU Model"]:
+        # fallback: parse from model string like "Intel(R) ... @ 2.30GHz"
+        m = re.search(r"@\s*([\d.]+)\s*([GM]Hz)", info["CPU Model"])
+        if m:
+            info["CPU Base Frequency"] = f"{m.group(1)} {m.group(2)}"
+    else:
+        cur = d.get("CPU MHz", "")
+        if cur:
+            info["CPU (Current) Frequency"] = f"{float(cur):.0f} MHz"
+    if maxf:
+        info["CPU Max Frequency"] = f"{float(maxf):.0f} MHz"
+
+    # RAM total (/proc/meminfo)
+    try:
+        meminfo = open("/proc/meminfo").read()
+        m = re.search(r"^MemTotal:\s+(\d+)\s+kB", meminfo, flags=re.M)
+        if m:
+            total_bytes = int(m.group(1)) * 1024
+            info["Total RAM"] = human_bytes(total_bytes)
+    except Exception:
+        pass
+
+    return info
+
+
+def collect_gpu() -> Dict[str, str]:
+    """
+    Use NVML to enumerate GPUs and select the 'newer' one:
+    1) Prefer a device whose name matches /4090/i
+    2) Else highest CUDA compute capability (major, minor), tiebreaker by total memory
+    Also reports driver version and CUDA driver version.
+    """
+    pynvml = _import_nvml()
+    pynvml.nvmlInit()
+    try:
+        count = pynvml.nvmlDeviceGetCount()
+        if count < 1:
+            return {"Error": "No NVIDIA GPUs detected by NVML."}
+
+        devices = []
+        for i in range(count):
+            h = pynvml.nvmlDeviceGetHandleByIndex(i)
+
+            # name can be bytes or str depending on wheel; normalize
+            raw_name = pynvml.nvmlDeviceGetName(h)
+            name = _to_text(raw_name)
+
+            mem_info = pynvml.nvmlDeviceGetMemoryInfo(h)
+            total_mem = getattr(mem_info, "total", 0)
+
+            # compute capability may not exist on very old drivers
+            try:
+                maj, minr = pynvml.nvmlDeviceGetCudaComputeCapability(h)
+            except Exception:
+                maj, minr = (0, 0)
+
+            devices.append(
+                {
+                    "index": i,
+                    "handle": h,
+                    "name": name,
+                    "mem": total_mem,
+                    "cc": (maj, minr),
+                }
+            )
+
+        # Prefer explicit "4090"
+        pick = next(
+            (d for d in devices if re.search(r"4090", d["name"], flags=re.I)), None
+        )
+        if pick is None:
+            # Highest compute capability, then largest memory
+            devices.sort(key=lambda x: (x["cc"][0], x["cc"][1], x["mem"]), reverse=True)
+            pick = devices[0]
+
+        # Driver version and CUDA driver version can be bytes or str
+        drv_raw = pynvml.nvmlSystemGetDriverVersion()
+        drv = _to_text(drv_raw)
+
+        # CUDA driver version (integer like 12040 -> 12.4)
+        cuda_drv_ver = ""
+        try:
+            v = pynvml.nvmlSystemGetCudaDriverVersion_v2()
+        except Exception:
+            v = pynvml.nvmlSystemGetCudaDriverVersion()
+        try:
+            major = v // 1000
+            minor = (v % 1000) // 10
+            patch = v % 10
+            cuda_drv_ver = f"{major}.{minor}.{patch}" if patch else f"{major}.{minor}"
+        except Exception:
+            cuda_drv_ver = ""
+
+        gpu_info = {
+            "Selected GPU Name": pick["name"],
+            "Selected GPU Memory": human_bytes(pick["mem"]),
+            "Selected GPU Compute Capability": f"{pick['cc'][0]}.{pick['cc'][1]}",
+            "NVIDIA Driver Version": drv,
+            "CUDA (Driver) Version": cuda_drv_ver,
+        }
+        return gpu_info
+    finally:
+        pynvml.nvmlShutdown()
+
+
+def summarize_numpy_build_config() -> str:
+    """
+    Capture numpy.__config__.show() and try to extract the BLAS/LAPACK backend line(s).
+    """
+    import numpy as np
+    from io import StringIO
+    import sys as _sys
+
+    buf = StringIO()
+    _stdout = _sys.stdout
+    try:
+        _sys.stdout = buf
+        np.__config__.show()
+    finally:
+        _sys.stdout = _stdout
+    txt = buf.getvalue()
+
+    # Heuristic: capture lines mentioning MKL, OpenBLAS, BLIS, LAPACK
+    lines = [
+        l
+        for l in txt.splitlines()
+        if re.search(r"(MKL|OpenBLAS|BLAS|LAPACK|BLIS)", l, re.I)
+    ]
+    if not lines:
+        # fall back to first ~12 lines
+        lines = txt.splitlines()[:12]
+    # Keep it compact
+    return "\n".join(lines[:20]).strip()
+
+
+def collect_software() -> Dict[str, str]:
+    info: Dict[str, str] = {}
+    import sys as _sys
+    import torch
+    import numpy as _np
+    import scipy as _sp
+    import sklearn as _sk
+
+    info["Python"] = _sys.version.split()[0]
+
+    # PyTorch + built CUDA/cuDNN + visible GPUs
+    info["PyTorch"] = torch.__version__
+    info["PyTorch Built CUDA"] = getattr(torch.version, "cuda", "") or ""
+    try:
+        cudnn_build = torch.backends.cudnn.version()  # integer
+        info["cuDNN (PyTorch build)"] = str(cudnn_build) if cudnn_build else ""
+    except Exception:
+        pass
+
+    # scikit-learn
+    info["scikit-learn"] = _sk.__version__
+
+    # NumPy / SciPy + build config
+    info["NumPy"] = _np.__version__
+    info["SciPy"] = _sp.__version__
+    info["NumPy Build Config"] = summarize_numpy_build_config()
+
+    return info
+
+
+# -------------------- Main --------------------
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Generate LaTeX table of system/software environment for thesis (Linux + NVIDIA)."
+    )
+    ap.add_argument(
+        "--output", "-o", type=str, help="Write LaTeX to this file instead of stdout."
+    )
+    ap.add_argument(
+        "--caption", type=str, default="Computational Environment (Hardware & Software)"
+    )
+    args = ap.parse_args()
+
+    errs = preflight()
+    if errs:
+        msg = (
+            "Preflight check failed:\n- "
+            + "\n- ".join(errs)
+            + "\n"
+            + "Please install missing components and re-run."
+        )
+        print(msg, file=sys.stderr)
+        sys.exit(1)
+
+    sections: List[Tuple[str, Dict[str, str]]] = []
+    sections.append(("System", collect_system()))
+    sections.append(("GPU (Selected Newer Device)", collect_gpu()))
+    sections.append(("Software Environment", collect_software()))
+
+    latex = latex_table(sections, caption=args.caption)
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(latex)
+        print(f"Wrote LaTeX to: {args.output}")
+    else:
+        print(latex)
+
+
+if __name__ == "__main__":
+    main()

Deep-SAD-PyTorch/pyproject.toml

@@ -12,6 +12,8 @@ dependencies = [
     "kiwisolver>=1.4.8",
     "matplotlib>=3.10.3",
     "numpy>=2.3.1",
+    "onnx>=1.18.0",
+    "onnxscript>=0.3.2",
     "pandas>=2.3.0",
     "pillow>=11.2.1",
     "pyparsing>=3.2.3",
@@ -21,8 +23,11 @@ dependencies = [
     "scipy>=1.16.0",
     "seaborn>=0.13.2",
     "six>=1.17.0",
+    "tabulate>=0.9.0",
+    "thop>=0.1.1.post2209072238",
     "torch-receptive-field",
     "torchscan>=0.1.1",
+    "visualtorch>=0.2.4",
 ]
 
 [tool.uv.sources]

Deep-SAD-PyTorch/src/baselines/isoforest.py

@@ -261,6 +261,80 @@ class IsoForest(object):
         logger.info("Test Time: {:.3f}s".format(self.results["test_time"]))
         logger.info("Finished testing.")
 
+    def inference(
+        self,
+        dataset: BaseADDataset,
+        device: str = "cpu",
+        n_jobs_dataloader: int = 0,
+        batch_size: int = 32,
+    ):
+        """Perform inference on the dataset using the trained Isolation Forest model."""
+        logger = logging.getLogger()
+
+        # Get inference data loader
+        _, _, inference_loader = dataset.loaders(
+            batch_size=batch_size, num_workers=n_jobs_dataloader
+        )
+
+        # Get data from loader
+        X = ()
+        idxs = []
+        file_ids = []
+        frame_ids = []
+
+        logger.info("Starting inference...")
+        start_time = time.time()
+
+        for data in inference_loader:
+            inputs, idx, (file_id, frame_id) = data
+            inputs = inputs.to(device)
+
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)
+            X_batch = inputs.view(inputs.size(0), -1)
+            X += (X_batch.cpu().data.numpy(),)
+
+            # Store indices and metadata
+            idxs.extend(idx.cpu().data.numpy().tolist())
+            file_ids.extend(file_id.cpu().data.numpy().tolist())
+            frame_ids.extend(frame_id.cpu().data.numpy().tolist())
+
+        X = np.concatenate(X)
+
+        # Get anomaly scores
+        scores = (-1.0) * self.model.decision_function(X)
+        scores = scores.flatten()
+
+        # Store inference results
+        self.inference_time = time.time() - start_time
+        self.inference_indices = np.array(idxs)
+        self.inference_file_ids = np.array(file_ids)
+        self.inference_frame_ids = np.array(frame_ids)
+
+        # Create index mapping similar to DeepSAD trainer
+        self.inference_index_mapping = {
+            "indices": self.inference_indices,
+            "file_ids": self.inference_file_ids,
+            "frame_ids": self.inference_frame_ids,
+        }
+
+        # Log inference statistics
+        logger.info(f"Number of inference samples: {len(self.inference_indices)}")
+        logger.info(
+            f"Number of unique files: {len(np.unique(self.inference_file_ids))}"
+        )
+        logger.info("Inference Time: {:.3f}s".format(self.inference_time))
+        logger.info(
+            "Score statistics: "
+            f"min={scores.min():.3f}, "
+            f"max={scores.max():.3f}, "
+            f"mean={scores.mean():.3f}, "
+            f"std={scores.std():.3f}"
+        )
+        logger.info("Finished inference.")
+
+        return scores
+
     def load_ae(self, dataset_name, model_path):
         """Load pretrained autoencoder from model_path for feature extraction in a hybrid Isolation Forest model."""
 

Deep-SAD-PyTorch/src/baselines/ocsvm.py

@@ -453,6 +453,80 @@ class OCSVM(object):
         logger.info("Test Time: {:.3f}s".format(self.results["test_time"]))
         logger.info("Finished testing.")
 
+    def inference(
+        self,
+        dataset: BaseADDataset,
+        device: str = "cpu",
+        n_jobs_dataloader: int = 0,
+        batch_size: int = 32,
+    ):
+        """Perform inference on the dataset using the trained OC-SVM model."""
+        logger = logging.getLogger()
+
+        # Get inference data loader
+        _, _, inference_loader = dataset.loaders(
+            batch_size=batch_size, num_workers=n_jobs_dataloader
+        )
+
+        # Get data from loader
+        X = ()
+        idxs = []
+        file_ids = []
+        frame_ids = []
+
+        logger.info("Starting inference...")
+        start_time = time.time()
+
+        for data in inference_loader:
+            inputs, idx, (file_id, frame_id) = data
+            inputs = inputs.to(device)
+
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)
+            X_batch = inputs.view(inputs.size(0), -1)
+            X += (X_batch.cpu().data.numpy(),)
+
+            # Store indices and metadata
+            idxs.extend(idx.cpu().data.numpy().tolist())
+            file_ids.extend(file_id.cpu().data.numpy().tolist())
+            frame_ids.extend(frame_id.cpu().data.numpy().tolist())
+
+        X = np.concatenate(X)
+
+        # Get anomaly scores
+        scores = (-1.0) * self.model.decision_function(X)
+        scores = scores.flatten()
+
+        # Store inference results
+        self.inference_time = time.time() - start_time
+        self.inference_indices = np.array(idxs)
+        self.inference_file_ids = np.array(file_ids)
+        self.inference_frame_ids = np.array(frame_ids)
+
+        # Create index mapping similar to DeepSAD trainer
+        self.inference_index_mapping = {
+            "indices": self.inference_indices,
+            "file_ids": self.inference_file_ids,
+            "frame_ids": self.inference_frame_ids,
+        }
+
+        # Log inference statistics
+        logger.info(f"Number of inference samples: {len(self.inference_indices)}")
+        logger.info(
+            f"Number of unique files: {len(np.unique(self.inference_file_ids))}"
+        )
+        logger.info("Inference Time: {:.3f}s".format(self.inference_time))
+        logger.info(
+            "Score statistics: "
+            f"min={scores.min():.3f}, "
+            f"max={scores.max():.3f}, "
+            f"mean={scores.mean():.3f}, "
+            f"std={scores.std():.3f}"
+        )
+        logger.info("Finished inference.")
+
+        return scores
+
     def load_ae(self, model_path, net_name, device="cpu"):
         """Load pretrained autoencoder from model_path for feature extraction in a hybrid OC-SVM model."""
 

Deep-SAD-PyTorch/src/datasets/main.py

@@ -21,6 +21,7 @@ def load_dataset(
     k_fold_num: int = None,
     num_known_normal: int = 0,
     num_known_outlier: int = 0,
+    split: float = 0.7,
 ):
     """Loads the dataset."""
 
@@ -49,6 +50,7 @@ def load_dataset(
             k_fold_num=k_fold_num,
             num_known_normal=num_known_normal,
             num_known_outlier=num_known_outlier,
+            split=split,
         )
 
     if dataset_name == "subtersplit":

Deep-SAD-PyTorch/src/datasets/subter.py

@@ -338,6 +338,8 @@ class SubTerInference(VisionDataset):
         self.frame_ids = np.arange(self.data.shape[0], dtype=np.int32)
         self.file_names = {0: experiment_file.name}
 
+        self.transform = transform if transform else transforms.ToTensor()
+
     def __len__(self):
         return len(self.data)
 

Deep-SAD-PyTorch/src/main.py

@@ -152,6 +152,12 @@ from utils.visualization.plot_images_grid import plot_images_grid
     default=0.001,
     help="Initial learning rate for Deep SAD network training. Default=0.001",
 )
+@click.option(
+    "--train_test_split",
+    type=float,
+    default=0.7,
+    help="Ratio of training data in the train-test split (default: 0.7).",
+)
 @click.option("--n_epochs", type=int, default=50, help="Number of epochs to train.")
 @click.option(
     "--lr_milestone",
@@ -307,6 +313,7 @@ def main(
     seed,
     optimizer_name,
     lr,
+    train_test_split,
     n_epochs,
     lr_milestone,
     batch_size,
@@ -416,6 +423,7 @@ def main(
             k_fold_num=k_fold_num,
             num_known_normal=num_known_normal,
             num_known_outlier=num_known_outlier,
+            split=train_test_split,
         )
         # Log random sample of known anomaly classes if more than 1 class
         if n_known_outlier_classes > 1:
@@ -630,57 +638,185 @@ def main(
             cfg.save_config(export_json=xp_path + "/config.json")
 
     elif action == "infer":
+        # Inference uses a deterministic, non-shuffled loader to preserve temporal order
         dataset = load_dataset(
-            dataset_name,
+            cfg.settings["dataset_name"],
             data_path,
-            normal_class,
-            known_outlier_class,
-            n_known_outlier_classes,
-            ratio_known_normal,
-            ratio_known_outlier,
-            ratio_pollution,
+            cfg.settings["normal_class"],
+            cfg.settings["known_outlier_class"],
+            cfg.settings["n_known_outlier_classes"],
+            cfg.settings["ratio_known_normal"],
+            cfg.settings["ratio_known_outlier"],
+            cfg.settings["ratio_pollution"],
             random_state=np.random.RandomState(cfg.settings["seed"]),
+            k_fold_num=False,
             inference=True,
         )
+
         # Log random sample of known anomaly classes if more than 1 class
         if n_known_outlier_classes > 1:
             logger.info("Known anomaly classes: %s" % (dataset.known_outlier_classes,))
 
-        # Initialize DeepSAD model and set neural network phi
-        deepSAD = DeepSAD(latent_space_dim, cfg.settings["eta"])
-        deepSAD.set_network(net_name)
-
-        # If specified, load Deep SAD model (center c, network weights, and possibly autoencoder weights)
-        if not load_model:
+        # --- Expect a model DIRECTORY (aligned with 'retest') ---
+        if (
+            (not load_model)
+            or (not Path(load_model).exists())
+            or (not Path(load_model).is_dir())
+        ):
             logger.error(
-                "For inference mode a model has to be loaded! Pass the --load_model option with the model path!"
+                "For inference mode a model directory has to be loaded! "
+                "Pass the --load_model option with the model directory path!"
+            )
+            return
+        load_model = Path(load_model)
+
+        # Resolve expected model artifacts (single-model / no k-fold suffixes)
+        deepsad_model_path = load_model / "model_deepsad.tar"
+        ae_model_path = load_model / "model_ae.tar"
+        ocsvm_model_path = load_model / "model_ocsvm.pkl"
+        isoforest_model_path = load_model / "model_isoforest.pkl"
+
+        # Sanity check model files exist
+        model_paths = [
+            deepsad_model_path,
+            ae_model_path,
+            ocsvm_model_path,
+            isoforest_model_path,
+        ]
+        missing = [p.name for p in model_paths if not p.exists() or not p.is_file()]
+        if missing:
+            logger.error(
+                "The following model files do not exist in the provided model directory: "
+                + ", ".join(missing)
             )
             return
 
-        deepSAD.load_model(model_path=load_model, load_ae=True, map_location=device)
-        logger.info("Loading model from %s." % load_model)
+        # Prepare output paths
+        inf_dir = Path(xp_path) / "inference"
+        inf_dir.mkdir(parents=True, exist_ok=True)
+        base_stem = Path(Path(dataset.root).stem)  # keep your previous naming
+        # DeepSAD outputs (keep legacy filenames for backward compatibility)
+        deepsad_scores_path = inf_dir / Path(
+            base_stem.stem + "_deepsad_scores"
+        ).with_suffix(".npy")
+        deepsad_outputs_path = inf_dir / Path(base_stem.stem + "_outputs").with_suffix(
+            ".npy"
+        )
+        # Baselines
+        ocsvm_scores_path = inf_dir / Path(
+            base_stem.stem + "_ocsvm_scores"
+        ).with_suffix(".npy")
+        isoforest_scores_path = inf_dir / Path(
+            base_stem.stem + "_isoforest_scores"
+        ).with_suffix(".npy")
 
-        inference_results, all_outputs = deepSAD.inference(
-            dataset, device=device, n_jobs_dataloader=n_jobs_dataloader
-        )
-        inference_results_path = (
-            Path(xp_path)
-            / "inference"
-            / Path(Path(dataset.root).stem).with_suffix(".npy")
-        )
-        inference_outputs_path = (
-            Path(xp_path)
-            / "inference"
-            / Path(Path(dataset.root).stem + "_outputs").with_suffix(".npy")
+        # Common loader settings
+        _n_jobs = (
+            n_jobs_dataloader
+            if "n_jobs_dataloader" in locals()
+            else cfg.settings.get("n_jobs_dataloader", 0)
         )
 
-        inference_results_path.parent.mkdir(parents=True, exist_ok=True)
-        np.save(inference_results_path, inference_results, fix_imports=False)
-        np.save(inference_outputs_path, all_outputs, fix_imports=False)
+        # ----------------- DeepSAD -----------------
+
+        deepSAD = DeepSAD(cfg.settings["latent_space_dim"], cfg.settings["eta"])
+        deepSAD.set_network(cfg.settings["net_name"])
+        deepSAD.load_model(
+            model_path=deepsad_model_path, load_ae=True, map_location=device
+        )
+        logger.info("Loaded DeepSAD model from %s.", deepsad_model_path)
+
+        deepsad_scores, deepsad_all_outputs = deepSAD.inference(
+            dataset, device=device, n_jobs_dataloader=_n_jobs
+        )
+
+        np.save(deepsad_scores_path, deepsad_scores)
+        # np.save(deepsad_outputs_path, deepsad_all_outputs)
 
         logger.info(
-            f"Inference: median={np.median(inference_results)} mean={np.mean(inference_results)} min={inference_results.min()} max={inference_results.max()}"
+            "DeepSAD inference: median=%.6f mean=%.6f min=%.6f max=%.6f",
+            float(np.median(deepsad_scores)),
+            float(np.mean(deepsad_scores)),
+            float(np.min(deepsad_scores)),
+            float(np.max(deepsad_scores)),
         )
+
+        # ----------------- OCSVM (hybrid) -----------------
+        ocsvm_scores = None
+        ocsvm = OCSVM(
+            kernel=cfg.settings["ocsvm_kernel"],
+            nu=cfg.settings["ocsvm_nu"],
+            hybrid=True,
+            latent_space_dim=cfg.settings["latent_space_dim"],
+        )
+        # load AE to build the feature extractor for hybrid OCSVM
+        ocsvm.load_ae(
+            net_name=cfg.settings["net_name"],
+            model_path=ae_model_path,
+            device=device,
+        )
+        ocsvm.load_model(import_path=ocsvm_model_path)
+
+        ocsvm_scores = ocsvm.inference(
+            dataset, device=device, n_jobs_dataloader=_n_jobs, batch_size=32
+        )
+
+        if ocsvm_scores is not None:
+            np.save(ocsvm_scores_path, ocsvm_scores)
+            logger.info(
+                "OCSVM inference: median=%.6f mean=%.6f min=%.6f max=%.6f",
+                float(np.median(ocsvm_scores)),
+                float(np.mean(ocsvm_scores)),
+                float(np.min(ocsvm_scores)),
+                float(np.max(ocsvm_scores)),
+            )
+        else:
+            logger.warning("OCSVM scores could not be determined; no array saved.")
+
+        # ----------------- Isolation Forest -----------------
+        isoforest_scores = None
+        Isoforest = IsoForest(
+            hybrid=False,
+            n_estimators=cfg.settings["isoforest_n_estimators"],
+            max_samples=cfg.settings["isoforest_max_samples"],
+            contamination=cfg.settings["isoforest_contamination"],
+            n_jobs=cfg.settings["isoforest_n_jobs_model"],
+            seed=cfg.settings["seed"],
+        )
+        Isoforest.load_model(import_path=isoforest_model_path, device=device)
+        isoforest_scores = Isoforest.inference(
+            dataset, device=device, n_jobs_dataloader=_n_jobs
+        )
+        if isoforest_scores is not None:
+            np.save(isoforest_scores_path, isoforest_scores)
+            logger.info(
+                "IsolationForest inference: median=%.6f mean=%.6f min=%.6f max=%.6f",
+                float(np.median(isoforest_scores)),
+                float(np.mean(isoforest_scores)),
+                float(np.min(isoforest_scores)),
+                float(np.max(isoforest_scores)),
+            )
+        else:
+            logger.warning(
+                "Isolation Forest scores could not be determined; no array saved."
+            )
+
+        # Final summary (DeepSAD always runs; baselines are best-effort)
+        logger.info(
+            "Inference complete. Saved arrays to %s:\n"
+            "  DeepSAD scores:   %s\n"
+            "  DeepSAD outputs:  %s\n"
+            "  OCSVM scores:     %s\n"
+            "  IsoForest scores: %s",
+            inf_dir,
+            deepsad_scores_path.name,
+            deepsad_outputs_path.name,
+            ocsvm_scores_path.name if ocsvm_scores is not None else "(not saved)",
+            isoforest_scores_path.name
+            if isoforest_scores is not None
+            else "(not saved)",
+        )
+
     elif action == "ae_elbow_test":
         # Load data once
         dataset = load_dataset(
@@ -694,6 +830,7 @@ def main(
             ratio_pollution,
             random_state=np.random.RandomState(cfg.settings["seed"]),
             k_fold_num=k_fold_num,
+            split=train_test_split,
         )
 
         # Set up k-fold passes
@@ -804,12 +941,14 @@ def main(
             k_fold_num=cfg.settings["k_fold_num"],
             num_known_normal=cfg.settings["num_known_normal"],
             num_known_outlier=cfg.settings["num_known_outlier"],
+            split=train_test_split,
         )
 
         train_passes = (
             range(cfg.settings["k_fold_num"]) if cfg.settings["k_fold"] else [None]
         )
 
+        retest_autoencoder = False
         retest_isoforest = True
         retest_ocsvm = True
         retest_deepsad = True
@@ -865,6 +1004,25 @@ def main(
                     k_fold_idx=fold_idx,
                 )
 
+            if retest_autoencoder:
+                # Initialize DeepSAD model and set neural network phi
+                deepSAD = DeepSAD(cfg.settings["latent_space_dim"], cfg.settings["eta"])
+                deepSAD.set_network(cfg.settings["net_name"])
+                deepSAD.load_model(
+                    model_path=ae_model_path, load_ae=True, map_location=device
+                )
+                logger.info("Loading model from %s." % load_model)
+                # Save pretraining results
+                if fold_idx is None:
+                    deepSAD.save_ae_results(
+                        export_pkl=load_model / "results_ae_retest.pkl"
+                    )
+                else:
+                    deepSAD.save_ae_results(
+                        export_pkl=load_model / f"results_ae_retest_{fold_idx}.pkl"
+                    )
+                del deepSAD
+
             # Initialize DeepSAD model and set neural network phi
             if retest_deepsad:
                 deepSAD = DeepSAD(cfg.settings["latent_space_dim"], cfg.settings["eta"])
@@ -898,7 +1056,7 @@ def main(
                     device=device,
                     n_jobs_dataloader=cfg.settings["n_jobs_dataloader"],
                     k_fold_idx=fold_idx,
-                    batch_size=256,
+                    batch_size=32,
                 )
 
             retest_output_path = load_model / "retest_output"

Deep-SAD-PyTorch/src/network_statistics.py

@@ -0,0 +1,101 @@
+import torch
+from thop import profile
+
+from networks.subter_LeNet import SubTer_LeNet, SubTer_LeNet_Autoencoder
+from networks.subter_LeNet_rf import SubTer_Efficient_AE, SubTer_EfficientEncoder
+
+# Configuration
+LATENT_DIMS = [32, 64, 128, 256, 512, 768, 1024]
+BATCH_SIZE = 1
+INPUT_SHAPE = (BATCH_SIZE, 1, 32, 2048)
+
+
+def count_parameters(model, input_shape):
+    """Count MACs and parameters for a model."""
+    model.eval()
+    with torch.no_grad():
+        input_tensor = torch.randn(input_shape)
+        macs, params = profile(model, inputs=(input_tensor,))
+    return {"MACs": macs, "Parameters": params}
+
+
+def format_number(num: float) -> str:
+    """Format large numbers with K, M, B, T suffixes."""
+    for unit in ["", "K", "M", "B", "T"]:
+        if abs(num) < 1000.0 or unit == "T":
+            return f"{num:3.2f}{unit}"
+        num /= 1000.0
+
+
+def main():
+    # Collect results per latent dimension
+    results = {}  # dim -> dict of 8 values
+    for dim in LATENT_DIMS:
+        # Instantiate models for this latent dim
+        lenet_enc = SubTer_LeNet(rep_dim=dim)
+        eff_enc = SubTer_EfficientEncoder(rep_dim=dim)
+        lenet_ae = SubTer_LeNet_Autoencoder(rep_dim=dim)
+        eff_ae = SubTer_Efficient_AE(rep_dim=dim)
+
+        # Profile each
+        lenet_enc_stats = count_parameters(lenet_enc, INPUT_SHAPE)
+        eff_enc_stats = count_parameters(eff_enc, INPUT_SHAPE)
+        lenet_ae_stats = count_parameters(lenet_ae, INPUT_SHAPE)
+        eff_ae_stats = count_parameters(eff_ae, INPUT_SHAPE)
+
+        results[dim] = {
+            "lenet_enc_params": format_number(lenet_enc_stats["Parameters"]),
+            "lenet_enc_macs": format_number(lenet_enc_stats["MACs"]),
+            "eff_enc_params": format_number(eff_enc_stats["Parameters"]),
+            "eff_enc_macs": format_number(eff_enc_stats["MACs"]),
+            "lenet_ae_params": format_number(lenet_ae_stats["Parameters"]),
+            "lenet_ae_macs": format_number(lenet_ae_stats["MACs"]),
+            "eff_ae_params": format_number(eff_ae_stats["Parameters"]),
+            "eff_ae_macs": format_number(eff_ae_stats["MACs"]),
+        }
+
+    # Build LaTeX table with tabularx
+    header = (
+        "\\begin{table}[!ht]\n"
+        "\\centering\n"
+        "\\renewcommand{\\arraystretch}{1.15}\n"
+        "\\begin{tabularx}{\\linewidth}{lXXXXXXXX}\n"
+        "\\hline\n"
+        " & \\multicolumn{4}{c}{\\textbf{Encoders}} & "
+        "\\multicolumn{4}{c}{\\textbf{Autoencoders}} \\\\\n"
+        "\\cline{2-9}\n"
+        "\\textbf{Latent $z$} & "
+        "\\textbf{LeNet Params} & \\textbf{LeNet MACs} & "
+        "\\textbf{Eff. Params} & \\textbf{Eff. MACs} & "
+        "\\textbf{LeNet Params} & \\textbf{LeNet MACs} & "
+        "\\textbf{Eff. Params} & \\textbf{Eff. MACs} \\\\\n"
+        "\\hline\n"
+    )
+
+    rows = []
+    for dim in LATENT_DIMS:
+        r = results[dim]
+        row = (
+            f"{dim} & "
+            f"{r['lenet_enc_params']} & {r['lenet_enc_macs']} & "
+            f"{r['eff_enc_params']} & {r['eff_enc_macs']} & "
+            f"{r['lenet_ae_params']} & {r['lenet_ae_macs']} & "
+            f"{r['eff_ae_params']} & {r['eff_ae_macs']} \\\\"
+        )
+        rows.append(row)
+
+    footer = (
+        "\\hline\n"
+        "\\end{tabularx}\n"
+        "\\caption{Parameter and MAC counts for SubTer variants across latent dimensionalities.}\n"
+        "\\label{tab:subter_counts}\n"
+        "\\end{table}\n"
+    )
+
+    latex_table = header + "\n".join(rows) + "\n" + footer
+
+    print(latex_table)
+
+
+if __name__ == "__main__":
+    main()

Deep-SAD-PyTorch/src/network_statistics.tex

@@ -0,0 +1,155 @@
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.ConvTranspose2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_upsample() for <class 'torch.nn.modules.upsampling.Upsample'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.ConvTranspose2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_upsample() for <class 'torch.nn.modules.upsampling.Upsample'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.ConvTranspose2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_upsample() for <class 'torch.nn.modules.upsampling.Upsample'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.ConvTranspose2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_upsample() for <class 'torch.nn.modules.upsampling.Upsample'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.ConvTranspose2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_upsample() for <class 'torch.nn.modules.upsampling.Upsample'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.ConvTranspose2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_upsample() for <class 'torch.nn.modules.upsampling.Upsample'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.ConvTranspose2d'>.
+[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.
+[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.
+[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.
+[INFO] Register count_upsample() for <class 'torch.nn.modules.upsampling.Upsample'>.
+[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.
+\begin{table}[!ht]
+\centering
+\renewcommand{\arraystretch}{1.15}
+\begin{tabularx}{\linewidth}{lXXXXXXXX}
+\hline
+ & \multicolumn{4}{c}{\textbf{Encoders}} & \multicolumn{4}{c}{\textbf{Autoencoders}} \\
+\cline{2-9}
+\textbf{Latent $z$} & \textbf{LeNet Params} & \textbf{LeNet MACs} & \textbf{Eff. Params} & \textbf{Eff. MACs} & \textbf{LeNet Params} & \textbf{LeNet MACs} & \textbf{Eff. Params} & \textbf{Eff. MACs} \\
+\hline
+32 & 525.29K & 27.92M & 263.80K & 29.82M & 1.05M & 54.95M & 532.35K & 168.49M \\
+64 & 1.05M & 28.44M & 525.94K & 30.08M & 2.10M & 56.00M & 1.06M & 169.02M \\
+128 & 2.10M & 29.49M & 1.05M & 30.61M & 4.20M & 58.10M & 2.11M & 170.07M \\
+256 & 4.20M & 31.59M & 2.10M & 31.65M & 8.39M & 62.29M & 4.20M & 172.16M \\
+512 & 8.39M & 35.78M & 4.20M & 33.75M & 16.78M & 70.68M & 8.40M & 176.36M \\
+768 & 12.58M & 39.98M & 6.29M & 35.85M & 25.17M & 79.07M & 12.59M & 180.55M \\
+1024 & 16.78M & 44.17M & 8.39M & 37.95M & 33.56M & 87.46M & 16.79M & 184.75M \\
+\hline
+\end{tabularx}
+\caption{Parameter and MAC counts for SubTer variants across latent dimensionalities.}
+\label{tab:subter_counts}
+\end{table}
+

Deep-SAD-PyTorch/src/onnx_export.py

@@ -1,11 +1,15 @@
+from pathlib import Path
+
 import torch
 import torch.onnx
-from networks.mnist_LeNet import MNIST_LeNet_Autoencoder
+
+from networks.subter_LeNet import SubTer_LeNet_Autoencoder
+from networks.subter_LeNet_rf import SubTer_Efficient_AE
 
 
-def export_model_to_onnx(model, filepath, input_shape=(1, 1, 28, 28)):
+def export_model_to_onnx(model, filepath):
     model.eval()  # Set the model to evaluation mode
-    dummy_input = torch.randn(input_shape)  # Create a dummy input tensor
+    dummy_input = torch.randn(model.input_dim)  # Create a dummy input tensor
     torch.onnx.export(
         model,  # model being run
         dummy_input,  # model input (or a tuple for multiple inputs)
@@ -23,13 +27,17 @@ def export_model_to_onnx(model, filepath, input_shape=(1, 1, 28, 28)):
 
 
 if __name__ == "__main__":
-    # Initialize the autoencoder model
-    autoencoder = MNIST_LeNet_Autoencoder(rep_dim=32)
+    output_folder_path = Path("./onnx_models")
+    output_folder_path.mkdir(parents=True, exist_ok=True)
 
-    # Define the file path where the ONNX model will be saved
-    onnx_file_path = "mnist_lenet_autoencoder.onnx"
+    models_to_visualize = [
+        (
+            SubTer_LeNet_Autoencoder(rep_dim=32),
+            output_folder_path / "subter_lenet_ae.onnx",
+        ),
+        (SubTer_Efficient_AE(rep_dim=32), output_folder_path / "subter_ef_ae.onnx"),
+    ]
 
-    # Export the model
-    export_model_to_onnx(autoencoder, onnx_file_path)
-
-    print(f"Model has been exported to {onnx_file_path}")
+    for model, output_path in models_to_visualize:
+        export_model_to_onnx(model, output_path)
+        print(f"Model has been exported to {output_path}")

Deep-SAD-PyTorch/src/optim/DeepSAD_trainer.py

@@ -177,6 +177,8 @@ class DeepSADTrainer(BaseTrainer):
             batch_size=self.batch_size, num_workers=self.n_jobs_dataloader
         )
 
+        latent_dim = net.rep_dim
+
         # Set device for network
         net = net.to(self.device)
 
@@ -184,7 +186,9 @@ class DeepSADTrainer(BaseTrainer):
         logger.info("Starting inference...")
         n_batches = 0
         start_time = time.time()
-        all_outputs = np.zeros((len(inference_loader.dataset), 1024), dtype=np.float32)
+        all_outputs = np.zeros(
+            (len(inference_loader.dataset), latent_dim), dtype=np.float32
+        )
         scores = []
         net.eval()
 
@@ -366,7 +370,9 @@ class DeepSADTrainer(BaseTrainer):
             scores_exp_valid = scores_exp[valid_mask_exp]
 
             self.test_auc_exp_based = roc_auc_score(labels_exp_binary, scores_exp_valid)
-            self.test_roc_exp_based = roc_curve(labels_exp_binary, scores_exp_valid)
+            self.test_roc_exp_based = roc_curve(
+                labels_exp_binary, scores_exp_valid, drop_intermediate=False
+            )
             self.test_prc_exp_based = precision_recall_curve(
                 labels_exp_binary, scores_exp_valid
             )
@@ -403,7 +409,7 @@ class DeepSADTrainer(BaseTrainer):
                 labels_manual_binary, scores_manual_valid
             )
             self.test_roc_manual_based = roc_curve(
-                labels_manual_binary, scores_manual_valid
+                labels_manual_binary, scores_manual_valid, drop_intermediate=False
             )
             self.test_prc_manual_based = precision_recall_curve(
                 labels_manual_binary, scores_manual_valid

thesis/base/declaration_en.tex

@@ -24,15 +24,12 @@
 not used other than the declared sources/resources, and that I have
 explicitly indicated all material which has been quoted either
 literally or by content from the sources used.
-\ifthenelse{\equal{\ThesisTitle}{master's thesis} \or
-	\equal{\ThesisTitle}{diploma thesis} \or
-	\equal{\ThesisTitle}{doctoral thesis}}
-{The text document uploaded to TUGRAZonline is identical to the present \ThesisTitle.}{\reminder{TODO: fix \textbackslash ThesisTitle}}
+The text document uploaded to TUGRAZonline is identical to the present \ThesisTitle.
 
 
 \par\vspace*{4cm}
 \centerline{
-\begin{tabular}{m{1.5cm}cm{1.5cm}m{3cm}m{1.5cm}cm{1.5cm}}
-\cline{1-3} \cline{5-7}
- & date & & & & (signature) &\\
-\end{tabular}}
+	\begin{tabular}{m{1.5cm}cm{1.5cm}m{3cm}m{1.5cm}cm{1.5cm}}
+		\cline{1-3} \cline{5-7}
+		 & date &  &  &  & (signature) & \\
+	\end{tabular}}

thesis/base/layout_thesis.tex

@@ -55,7 +55,7 @@
 \makeatother
 
 % header and footer texts
-\clearscrheadfoot % clear everything
+\clearpairofpagestyles % clear everything
 \KOMAoptions{headlines=1} % header needs two lines here
 % [plain]{actual (scrheadings)}
 \ihead[]{}%
@@ -141,46 +141,46 @@
 \ifthenelse{\equal{\DocumentLanguage}{en}}{\renewcaptionname{USenglish}{\figurename}{Figure}}{}%
 \ifthenelse{\equal{\DocumentLanguage}{de}}{\renewcaptionname{ngerman}{\figurename}{Abbildung}}{}%
 \captionsetup{%
-format=hang,% hanging captions
-labelformat=simple,% just name and number
-labelsep=colon,% colon and space
-justification=justified,%
-singlelinecheck=true,% center single line captions
-font={footnotesize,it},% font style of label and text
-margin=0.025\textwidth,% margin left/right of the caption (to textwidth)
-indention=0pt,% no further indention (just hanging)
-hangindent=0pt,% no further indention (just hanging)}
-aboveskip=8pt,% same spacing above and...
-belowskip=8pt}% ...below the float (this way tables shouln't be a problem, either)
+	format=hang,% hanging captions
+	labelformat=simple,% just name and number
+	labelsep=colon,% colon and space
+	justification=justified,%
+	singlelinecheck=true,% center single line captions
+	font={footnotesize,it},% font style of label and text
+	margin=0.025\textwidth,% margin left/right of the caption (to textwidth)
+	indention=0pt,% no further indention (just hanging)
+	hangindent=0pt,% no further indention (just hanging)}
+	aboveskip=8pt,% same spacing above and...
+	belowskip=8pt}% ...below the float (this way tables shouln't be a problem, either)
 
 % code listings
 \lstloadlanguages{VHDL,Matlab,[ANSI]C,Java,[LaTeX]TeX}
 \lstset{%
-%   general
-breaklines=true,% automatically break long lines
-breakatwhitespace=true,% break only at white spaces
-breakindent=1cm,% additional indentation for broken lines
-%   positioning
-linewidth=\linewidth,% set width of whole thing to \linewidth
-xleftmargin=0.1\linewidth,%
-%   frame and caption
-frame=tlrb,% frame the entire thing
-framexleftmargin=1cm,% to include linenumbering into frame
-captionpos=b,% caption at bottom
-%   format parameters
-basicstyle=\ttfamily\tiny,% small true type font
-keywordstyle=\color{black},%
-identifierstyle=\color{black},%
-commentstyle=\color[rgb]{0.45,0.45,0.45},% gray
-stringstyle=\color{black},%
-showstringspaces=false,%
-showtabs=false,%
-tabsize=2,%
-%   linenumbers
-numberstyle=\tiny,%
-numbers=left,%
-numbersep=3mm,%
-firstnumber=1,%
-stepnumber=1,% number every line (0: off)
-numberblanklines=true%
+	%   general
+	breaklines=true,% automatically break long lines
+	breakatwhitespace=true,% break only at white spaces
+	breakindent=1cm,% additional indentation for broken lines
+	%   positioning
+	linewidth=\linewidth,% set width of whole thing to \linewidth
+	xleftmargin=0.1\linewidth,%
+	%   frame and caption
+	frame=tlrb,% frame the entire thing
+	framexleftmargin=1cm,% to include linenumbering into frame
+	captionpos=b,% caption at bottom
+	%   format parameters
+	basicstyle=\ttfamily\tiny,% small true type font
+	keywordstyle=\color{black},%
+	identifierstyle=\color{black},%
+	commentstyle=\color[rgb]{0.45,0.45,0.45},% gray
+	stringstyle=\color{black},%
+	showstringspaces=false,%
+	showtabs=false,%
+	tabsize=2,%
+	%   linenumbers
+	numberstyle=\tiny,%
+	numbers=left,%
+	numbersep=3mm,%
+	firstnumber=1,%
+	stepnumber=1,% number every line (0: off)
+	numberblanklines=true%
 }

thesis/base/macros.tex

@@ -147,22 +147,22 @@
 %   standard
 \newcommand{\fig}[3]{\begin{figure}\centering\includegraphics[width=\textwidth]{#2}\caption{#3}\label{fig:#1}\end{figure}}%
 %   with controllable parameters
-\newcommand{\figc}[4]{\begin{figure}\centering\includegraphics[#1]{#2}\caption{#3}\label{fig:#4}\end{figure}}%
+\newcommand{\figc}[4]{\begin{figure}\centering\includegraphics[#4]{#2}\caption{#3}\label{fig:#1}\end{figure}}%
 %   two subfigures
 \newcommand{\twofig}[6]{\begin{figure}\centering%
-\subfigure[#2]{\includegraphics[width=0.495\textwidth]{#1}}%
-\subfigure[#4]{\includegraphics[width=0.495\textwidth]{#3}}%
-\caption{#5}\label{fig:#6}\end{figure}}%
+		\subfigure[#2]{\includegraphics[width=0.495\textwidth]{#1}}%
+		\subfigure[#4]{\includegraphics[width=0.495\textwidth]{#3}}%
+		\caption{#5}\label{fig:#6}\end{figure}}%
 %   two subfigures with labels for each subplot
 \newcommand{\twofigs}[8]{\begin{figure}\centering%
-\subfigure[#2]{\includegraphics[width=0.495\textwidth]{#1}\label{fig:#8#3}}%
-\subfigure[#5]{\includegraphics[width=0.495\textwidth]{#4}\label{fig:#8#6}}%
-\caption{#7}\label{fig:#8}\end{figure}}%
+		\subfigure[#2]{\includegraphics[width=0.495\textwidth]{#1}\label{fig:#8#3}}%
+		\subfigure[#5]{\includegraphics[width=0.495\textwidth]{#4}\label{fig:#8#6}}%
+		\caption{#7}\label{fig:#8}\end{figure}}%
 %   two subfigures and controllable parameters
 \newcommand{\twofigc}[8]{\begin{figure}\centering%
-\subfigure[#3]{\includegraphics[#1]{#2}}%
-\subfigure[#6]{\includegraphics[#4]{#5}}%
-\caption{#7}\label{fig:#8}\end{figure}}%
+		\subfigure[#3]{\includegraphics[#1]{#2}}%
+		\subfigure[#6]{\includegraphics[#4]{#5}}%
+		\caption{#7}\label{fig:#8}\end{figure}}%
 
 % framed figures
 %   standard
@@ -171,19 +171,19 @@
 \newcommand{\figcf}[4]{\begin{figure}\centering\fbox{\includegraphics[#1]{#2}}\caption{#3}\label{fig:#4}\end{figure}}%
 %   two subfigures
 \newcommand{\twofigf}[6]{\begin{figure}\centering%
-\fbox{\subfigure[#2]{\includegraphics[width=0.495\textwidth]{#1}}}%
-\fbox{\subfigure[#4]{\includegraphics[width=0.495\textwidth]{#3}}}%
-\caption{#5}\label{fig:#6}\end{figure}}%
+		\fbox{\subfigure[#2]{\includegraphics[width=0.495\textwidth]{#1}}}%
+		\fbox{\subfigure[#4]{\includegraphics[width=0.495\textwidth]{#3}}}%
+		\caption{#5}\label{fig:#6}\end{figure}}%
 %   two subfigures with labels for each subplot
 \newcommand{\twofigsf}[8]{\begin{figure}\centering%
-\fbox{\subfigure[#2]{\includegraphics[width=0.495\textwidth]{#1}\label{fig:#8#3}}}%
-\fbox{\subfigure[#5]{\includegraphics[width=0.495\textwidth]{#4}\label{fig:#8#6}}}%
-\caption{#7}\label{fig:#8}\end{figure}}%
+		\fbox{\subfigure[#2]{\includegraphics[width=0.495\textwidth]{#1}\label{fig:#8#3}}}%
+		\fbox{\subfigure[#5]{\includegraphics[width=0.495\textwidth]{#4}\label{fig:#8#6}}}%
+		\caption{#7}\label{fig:#8}\end{figure}}%
 %   two subfigures and controllable parameters
 \newcommand{\twofigcf}[8]{\begin{figure}\centering%
-\fbox{\subfigure[#3]{\includegraphics[#1]{#2}}}%
-\fbox{\subfigure[#6]{\includegraphics[#4]{#5}}}%
-\caption{#7}\label{fig:#8}\end{figure}}%
+		\fbox{\subfigure[#3]{\includegraphics[#1]{#2}}}%
+		\fbox{\subfigure[#6]{\includegraphics[#4]{#5}}}%
+		\caption{#7}\label{fig:#8}\end{figure}}%
 
 % listings
 \newcommand{\filelisting}[5][]{\lstinputlisting[style=#2,caption={#4},label={lst:#5},#1]{#3}}

thesis/base/packages.tex

@@ -47,33 +47,33 @@
 \usepackage{fixltx2e}% LaTeX 2e bugfixes
 \usepackage{ifthen}% for optional parts
 \ifthenelse{\equal{\PaperSize}{a4paper}}{
-\usepackage[paper=\PaperSize,twoside=\Twosided,%
-textheight=246mm,%
-textwidth=160mm,%
-heightrounded=true,% round textheight to multiple of lines (avoids overfull vboxes)
-ignoreall=true,% do not include header, footer, and margins in calculations
-marginparsep=5pt,% marginpar only used for signs (centered), thus only small sep. needed
-marginparwidth=10mm,% prevent margin notes to be out of page
-hmarginratio=2:1,% set margin ration (inner:outer for twoside) - (2:3 is default)
-]{geometry}}{}%
+	\usepackage[paper=\PaperSize,twoside=\Twosided,%
+		textheight=246mm,%
+		textwidth=160mm,%
+		heightrounded=true,% round textheight to multiple of lines (avoids overfull vboxes)
+		ignoreall=true,% do not include header, footer, and margins in calculations
+		marginparsep=5pt,% marginpar only used for signs (centered), thus only small sep. needed
+		marginparwidth=10mm,% prevent margin notes to be out of page
+		hmarginratio=2:1,% set margin ration (inner:outer for twoside) - (2:3 is default)
+	]{geometry}}{}%
 \ifthenelse{\equal{\PaperSize}{letterpaper}}{
-\usepackage[paper=\PaperSize,twoside=\Twosided,%
-textheight=9in,%
-textwidth=6.5in,%
-heightrounded=true,% round textheight to multiple of lines (avoids overfull vboxes)
-ignoreheadfoot=false,% do not include header and footer in calculations
-marginparsep=5pt,% marginpar only used for signs (centered), thus only small sep. needed
-marginparwidth=10mm,% prevent margin notes to be out of page
-hmarginratio=3:2,% set margin ration (inner:outer for twoside) - (2:3 is default)
-]{geometry}}{}%
+	\usepackage[paper=\PaperSize,twoside=\Twosided,%
+		textheight=9in,%
+		textwidth=6.5in,%
+		heightrounded=true,% round textheight to multiple of lines (avoids overfull vboxes)
+		ignoreheadfoot=false,% do not include header and footer in calculations
+		marginparsep=5pt,% marginpar only used for signs (centered), thus only small sep. needed
+		marginparwidth=10mm,% prevent margin notes to be out of page
+		hmarginratio=3:2,% set margin ration (inner:outer for twoside) - (2:3 is default)
+	]{geometry}}{}%
 \ifthenelse{\equal{\DocumentLanguage}{en}}{\usepackage[T1]{fontenc}\usepackage[utf8]{inputenc}\usepackage[USenglish]{babel}}{}%
 \ifthenelse{\equal{\DocumentLanguage}{de}}{\usepackage[T1]{fontenc}\usepackage[utf8]{inputenc}\usepackage[ngerman]{babel}}{}%
 \usepackage[%
-headtopline,plainheadtopline,% activate all lines (header and footer)
-headsepline,plainheadsepline,%
-footsepline,plainfootsepline,%
-footbotline,plainfootbotline,%
-automark% auto update \..mark
+	headtopline,plainheadtopline,% activate all lines (header and footer)
+	headsepline,plainheadsepline,%
+	footsepline,plainfootsepline,%
+	footbotline,plainfootbotline,%
+	automark% auto update \..mark
 ]{scrlayer-scrpage}% (KOMA)
 \usepackage{imakeidx}
 \usepackage[]{caption}% customize captions
@@ -91,7 +91,7 @@ automark% auto update \..mark
 \usepackage[normalem]{ulem}% cross-out, strike-out, underlines (normalem: keep \emph italic)
 %\usepackage[safe]{textcomp}% loading in safe mode to avoid problems (see LaTeX companion)
 %\usepackage[geometry,misc]{ifsym}% technical symbols
-\usepackage{remreset}%\@removefromreset commands (e.g., for continuous footnote numbering)
+%\usepackage{remreset}%\@removefromreset commands (e.g., for continuous footnote numbering)
 \usepackage{paralist}% extended list environments
 % \usepackage[Sonny]{fncychap}
 \usepackage[avantgarde]{quotchap}
@@ -140,35 +140,35 @@ automark% auto update \..mark
 \usepackage{mdwlist}              %list extensions
 \ifthenelse{\equal{\DocumentLanguage}{de}}
 {
-  \usepackage[german]{fancyref}   %Bessere Querverweise
-  \usepackage[locale=DE]{siunitx}     %Zahlen und SI Einheiten => Binary units aktivieren...
-  \usepackage[autostyle=true,     %Anführungszeichen und Übersetzung der Literaturverweise
-  german=quotes]{csquotes}        %Anführungszeichen und Übersetzung der Literaturverweise
+	\usepackage[german]{fancyref}   %Bessere Querverweise
+	\usepackage[locale=DE]{siunitx}     %Zahlen und SI Einheiten => Binary units aktivieren...
+	\usepackage[autostyle=true,     %Anführungszeichen und Übersetzung der Literaturverweise
+		german=quotes]{csquotes}        %Anführungszeichen und Übersetzung der Literaturverweise
 }
 {
-  \usepackage[english]{fancyref}  %Bessere Querverweise
-  \usepackage[locale=US]{siunitx}     %Zahlen und SI Einheiten => Binary units aktivieren...
-  \usepackage[autostyle=true]     %Anführungszeichen und Übersetzung der Literaturverweise
-  {csquotes}
+	\usepackage[english]{fancyref}  %Bessere Querverweise
+	\usepackage[locale=US]{siunitx}     %Zahlen und SI Einheiten => Binary units aktivieren...
+	\usepackage[autostyle=true]     %Anführungszeichen und Übersetzung der Literaturverweise
+	{csquotes}
 }
 \sisetup{detect-weight=true, detect-family=true} %format like surrounding environment
 %extending fancyref for listings in both languages:
 \newcommand*{\fancyreflstlabelprefix}{lst}
 \fancyrefaddcaptions{english}{%
-  \providecommand*{\freflstname}{listing}%
-  \providecommand*{\Freflstname}{Listing}%
+	\providecommand*{\freflstname}{listing}%
+	\providecommand*{\Freflstname}{Listing}%
 }
 \fancyrefaddcaptions{german}{%
-  \providecommand*{\freflstname}{Listing}%
-  \providecommand*{\Freflstname}{Listing}%
+	\providecommand*{\freflstname}{Listing}%
+	\providecommand*{\Freflstname}{Listing}%
 }
 \frefformat{plain}{\fancyreflstlabelprefix}{\freflstname\fancyrefdefaultspacing#1}
 \Frefformat{plain}{\fancyreflstlabelprefix}{\Freflstname\fancyrefdefaultspacing#1}
 \frefformat{vario}{\fancyreflstlabelprefix}{%
-  \freflstname\fancyrefdefaultspacing#1#3%
+	\freflstname\fancyrefdefaultspacing#1#3%
 }
 \Frefformat{vario}{\fancyreflstlabelprefix}{%
-  \Freflstname\fancyrefdefaultspacing#1#3%
+	\Freflstname\fancyrefdefaultspacing#1#3%
 }
 
 \sisetup{separate-uncertainty}    %enable uncertainity for siunitx
@@ -176,30 +176,30 @@ automark% auto update \..mark
 \DeclareSIUnit\permille{\text{\textperthousand}} %add \permille to siunitx
 \usepackage{xfrac}                %Schönere brüche für SI Einheiten
 \sisetup{per-mode=fraction,       %Bruchstriche bei SI Einheiten aktivieren
-fraction-function=\sfrac}         %xfrac als Bruchstrichfunktion verwenden
+	fraction-function=\sfrac}         %xfrac als Bruchstrichfunktion verwenden
 \usepackage[scaled=0.78]{inconsolata}%Schreibmaschinenschrift für Quellcode
 
 \usepackage[backend=biber,        %Literaturverweiserweiterung Backend auswählen
-bibencoding=utf8,                 %.bib-File ist utf8-codiert...
-maxbibnames=99,                   %Immer alle Authoren in der Bibliographie darstellen...
-style=ieee
+	bibencoding=utf8,                 %.bib-File ist utf8-codiert...
+	maxbibnames=99,                   %Immer alle Authoren in der Bibliographie darstellen...
+	style=ieee
 ]{biblatex}
 \bibliography{bib/bibliography}      %literatur.bib wird geladen und als Literaturverweis Datei verwendet
 
 \ifthenelse{\equal{\FramedLinks}{true}}
 {
-  \usepackage[%
-  breaklinks=true,% allow line break in links
-  colorlinks=false,% if false: framed link
-  linkcolor=black,anchorcolor=black,citecolor=black,filecolor=black,%
-  menucolor=black,urlcolor=black,bookmarksnumbered=true]{hyperref}% hyperlinks for references
+	\usepackage[%
+		breaklinks=true,% allow line break in links
+		colorlinks=false,% if false: framed link
+		linkcolor=black,anchorcolor=black,citecolor=black,filecolor=black,%
+		menucolor=black,urlcolor=black,bookmarksnumbered=true]{hyperref}% hyperlinks for references
 }
 {
-  \usepackage[%
-  breaklinks=true,% allow line break in links
-  colorlinks=true,% if false: framed link
-  linkcolor=black,anchorcolor=black,citecolor=black,filecolor=black,%
-  menucolor=black,urlcolor=black,bookmarksnumbered=true]{hyperref}% hyperlinks for references
+	\usepackage[%
+		breaklinks=true,% allow line break in links
+		colorlinks=true,% if false: framed link
+		linkcolor=black,anchorcolor=black,citecolor=black,filecolor=black,%
+		menucolor=black,urlcolor=black,bookmarksnumbered=true]{hyperref}% hyperlinks for references
 }
 
 \setcounter{biburlnumpenalty}{100}%Urls in Bibliographie Zeilenbrechbar machen
@@ -213,8 +213,8 @@ style=ieee
 
 \ifthenelse{\equal{\DocumentLanguage}{de}}
 {
-  \deftranslation[to=ngerman]       %Dem Paket babel den deutschen Abkürzungsverzeichnis-Kapitelnamen
-  {Acronyms}{Abkürzungsverzeichnis} %beibringen
+	\deftranslation[to=ngerman]       %Dem Paket babel den deutschen Abkürzungsverzeichnis-Kapitelnamen
+	{Acronyms}{Abkürzungsverzeichnis} %beibringen
 }{}
 
 % misc

thesis/diagrams/deepsad_procedure/deepsad_procedure.tex

@@ -0,0 +1,147 @@
+\documentclass[tikz,border=10pt]{standalone}
+\usepackage{tikz}
+\usepackage{amsfonts}
+\usetikzlibrary{positioning, shapes.geometric, fit, arrows, arrows.meta, backgrounds}
+
+% Define box styles
+\tikzset{
+  databox/.style={rectangle, align=center, draw=black, fill=blue!50, thick, rounded corners},%, inner sep=4},
+  procbox/.style={rectangle, align=center, draw=black, fill=orange!30, thick, rounded corners},
+  hyperbox/.style={rectangle, align=center, draw=black, fill=green!30, thick, rounded corners},
+  stepsbox/.style={rectangle, align=left, draw=black,fill=white, rounded corners, minimum width=5.2cm, minimum height=1.5cm, font=\small},
+  outputbox/.style={rectangle, align=center, draw=red!80, fill=red!20, rounded corners, minimum width=5.2cm, minimum height=1.5cm, font=\small},
+  hlabelbox/.style={rectangle, align=center, draw=black,fill=white, rounded corners, minimum width=5.2cm, minimum height=1.5cm, font=\small},
+  vlabelbox/.style={rectangle, align=center, draw=black,fill=white, rounded corners, minimum width=3cm, minimum height=1.8cm, font=\small},
+  arrow/.style={-{Latex[length=3mm]}},
+  arrowlabel/.style={fill=white,inner sep=2pt,midway}
+}
+
+\newcommand{\boxtitle}[1]{\textbf{#1}\\[.4em]}
+
+\pgfdeclarelayer{background}
+\pgfdeclarelayer{foreground}
+\pgfsetlayers{background,main,foreground}
+
+\begin{document}
+\begin{tikzpicture}[node distance=1cm and 2cm]
+
+	\node (data) {Data};
+	\node[right=4.9 of data] (process) {Procedure};
+	\node[right=4.1 of process] (hyper) {Hyperparameters};
+
+	\begin{pgfonlayer}{foreground}
+		\node[hlabelbox, below=1.29 of data] (unlabeled) {\boxtitle{Unlabeled Data} Significantly more normal than \\ anomalous samples required};
+		\node[hlabelbox, below=.1 of unlabeled] (labeled) {\boxtitle{Labeled Data} No requirement regarding ratio \\ +1 = normal, -1 = anomalous};
+	\end{pgfonlayer}
+	\begin{pgfonlayer}{background}
+		\node[databox, fit=(unlabeled) (labeled), label={[label distance = 1, name=traindatalabel]above:{\textbf{Training Data}}}] (traindata) {};
+	\end{pgfonlayer}
+
+	%\draw[arrow] (latent.east) -- node{} (autoenc.west);
+
+	\begin{pgfonlayer}{foreground}
+		\node[stepsbox, below=of process] (pretrainproc) {Train Autoencoder $\mathcal{\phi}_{AE}$ \\ optimize Autoencoding Objective \\ for $E_A$ Epochs \\ with $L_A$ Learning Rate \\ No Labels Used / Required};
+		\node[outputbox, below=.1 of pretrainproc] (pretrainout) {\boxtitle{Outputs} $\mathcal{\phi}$: Encoder / DeepSAD Network \\ $\mathcal{W}_E$: Encoder Network Weights};
+	\end{pgfonlayer}
+	\begin{pgfonlayer}{background}
+		\node[procbox, fit=(pretrainproc) (pretrainout), label={[label distance = 1, name=pretrainlab]above:{\textbf{Pre-Training of Autoencoder}}}] (pretrain) {};
+	\end{pgfonlayer}
+
+	\begin{pgfonlayer}{foreground}
+		\node[hlabelbox, below=1.26 of hyper] (autoencarch) {\boxtitle{Autoencoder Architecture} $\mathcal{\phi}_{AE}$: Autoencoder Network \\ $\mathbb{R}^d$: Latent Space Size };
+		\node[hlabelbox, below=.1 of autoencarch] (pretrainhyper) {\boxtitle{Hyperparameters} $E_A$: Number of Epochs  \\ $L_A$: Learning Rate AE};
+	\end{pgfonlayer}
+	\begin{pgfonlayer}{background}
+		\node[hyperbox, fit=(autoencarch) (pretrainhyper), label={[label distance = 1, name=autoenclabel]above:{\textbf{Pre-Training Hyperparameters}}}] (pretrainhyp) {};
+	\end{pgfonlayer}
+
+	\draw[arrow] (pretrainhyp.west) -- (pretrain.east);
+
+	%\draw[arrow] (node cs:name=traindata,angle=10) -- node[arrowlabel]{data type} (node cs:name=autoenc,angle=177);
+
+	%	\draw[arrow] (node cs:name=autoenc,angle=196) |- (node cs:name=pretrain,angle=5);
+
+	\begin{pgfonlayer}{foreground}
+		\node[stepsbox, below=1.4 of pretrain] (calccproc) {Init Network $\mathcal{\phi}$ with $\mathcal{W}_E$ \\ Forward Pass on all data \\  Hypersphere center $\mathbf{c}$ is mean \\ of all Latent Representation};
+		\node[outputbox, below=.1 of calccproc] (calccout) {\boxtitle{Outputs} $\mathbf{c}$: Hypersphere Center};
+	\end{pgfonlayer}
+	\begin{pgfonlayer}{background}
+		\node[procbox, fit=(calccproc) (calccout), label={[label distance = 1, name=calcclab]above:{\textbf{Calculate Hypersphere Center}}}] (calcc) {};
+	\end{pgfonlayer}
+
+	\draw[arrow] (pretrain.south) -- (calcclab.north);
+	\draw[arrow] (traindata.east) -- (pretrain.west);
+	\draw[arrow] (traindata.south) |- (calcc.west);
+
+	%\draw[arrow] (node cs:name=traindata,angle=45) |- node[arrowlabel]{all training data, labels removed} (node cs:name=pretrain,angle=160);
+	%\draw[arrow] (node cs:name=traindata,angle=-45) |- node[arrowlabel]{all training data, labels removed} (node cs:name=calcc,angle=200);
+
+	\begin{pgfonlayer}{foreground}
+		\node[stepsbox, below=1.4 of calcc] (maintrainproc) {Init Network $\mathcal{\phi}$ with $\mathcal{W}_E$ \\ Train Network $\mathcal{\phi}$  \\ optimize DeepSAD Objective\\ for $E_M$ Epochs \\ with $L_M$ Learning Rate \\ Considers Labels with $\eta$ strength};
+		\node[outputbox, below=.1 of maintrainproc] (maintrainout) {\boxtitle{Outputs} $\mathcal{\phi}$: DeepSAD Network \\ $\mathcal{W}$: DeepSAD Network Weights  \\ $\mathbf{c}$: Hypersphere Center};
+	\end{pgfonlayer}
+	\begin{pgfonlayer}{background}
+		\node[procbox, fit=(maintrainproc) (maintrainout), label={[label distance = 1, name=maintrainlab]above:{\textbf{Main Training}}}] (maintrain) {};
+	\end{pgfonlayer}
+
+	\begin{pgfonlayer}{foreground}
+		\node[hlabelbox, below=12.48 of hyper] (maintrainhyper) {$E_M$: Number of Epochs  \\ $L_M$: Learning Rate  \\ $\eta$: Weight Labeled/Unlabeled};
+	\end{pgfonlayer}
+	\begin{pgfonlayer}{background}
+		\node[hyperbox, fit=(maintrainhyper), label={[label distance = 1, name=autoenclabel]above:{\textbf{Main-Training Hyperparameters}}}] (maintrainhyp) {};
+	\end{pgfonlayer}
+
+	\draw[arrow] (node cs:name=pretrain,angle=-50) |- +(1.5, -0.55) -- +(1.5,-5.4) -| (node cs:name=maintrain,angle=50);
+
+
+	%\draw[arrow] (pretrainoutput.south) -- (node cs:name=maintrain,angle=22);
+	\draw[arrow] (calcc.south) -- (maintrainlab.north);
+	\draw[arrow] (traindata.south) |- (maintrain.west);
+	%\draw[arrow] (node cs:name=traindata,angle=-135) |- node[arrowlabel]{all training data, including labels} (maintrain.west);
+	\draw[arrow] (maintrainhyp.west) -- (maintrain.east);
+
+
+	\begin{pgfonlayer}{foreground}
+		\node[stepsbox, below=1.4 of maintrain] (inferenceproc) {Init Network $\mathcal{\phi}$ with $\mathcal{W}$ \\Forward Pass on sample = $\mathbf{p}$ \\ Calculate Distance $\mathbf{p} \rightarrow \mathbf{c}$ \\ Distance = Anomaly Score};
+		\node[outputbox, below=.1 of inferenceproc] (inferenceout) {\boxtitle{Outputs} Anomaly Score (Analog Value) \\ Higher for Anomalies};
+	\end{pgfonlayer}
+	\begin{pgfonlayer}{background}
+		\node[procbox, fit=(inferenceproc) (inferenceout), label={[label distance = 1, name=inferencelab]above:{\textbf{Inference}}}] (inference) {};
+	\end{pgfonlayer}
+
+	\begin{pgfonlayer}{foreground}
+		\node[hlabelbox, below=13.32 of traindata] (newdatasample) {\boxtitle{New Data Sample} Same data type as training data};
+	\end{pgfonlayer}
+	\begin{pgfonlayer}{background}
+		\node[databox, fit=(newdatasample),  label={[label distance = 1] above:{\textbf{Unseen Data}}}] (newdata) {};
+	\end{pgfonlayer}
+
+	\draw[arrow] (maintrain.south) -- (inferencelab.north);
+	\draw[arrow] (newdata.east) -- (inference.west);
+
+\end{tikzpicture}
+\end{document}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

thesis/filters/drop-images.lua

@@ -0,0 +1,11 @@
+-- drop-images.lua
+-- Replaces all images (figures, graphics) with a short placeholder.
+function Image(el) return pandoc.Str("[image omitted]") end
+
+-- For LaTeX figures that are still raw
+function RawBlock(el)
+    if el.format == "tex" and el.text:match("\\begin%s*{%s*figure%s*}") then
+        return pandoc.Plain({pandoc.Str("[figure omitted]")})
+    end
+end
+

thesis/filters/drop-tables.lua

@@ -0,0 +1,11 @@
+-- drop-tables.lua
+-- Removes LaTeX tabular and tabularx environments (and their contents).
+function RawBlock(el)
+    if el.format == "tex" then
+        -- Check for tabular or tabularx environment
+        if el.text:match("\\begin%s*{%s*tabularx?%s*}") then
+            return pandoc.Plain({pandoc.Str("[table omitted]")})
+        end
+    end
+end
+

thesis/filters/keep-citations.lua

@@ -0,0 +1,43 @@
+-- keep-citations.lua
+-- Replace citations with a placeholder and eat any preceding space.
+local PH = "[citation]"
+
+-- Pandoc-native citations (if the reader produced Cite nodes)
+function Cite(el) return pandoc.Str(PH) end
+
+-- Raw LaTeX \cite-like macros (when not parsed as Cite)
+function RawInline(el)
+    if el.format and el.format:match("tex") and el.text:match("\\%a-*cite%*?") then
+        return pandoc.Str(PH)
+    end
+end
+
+-- Remove a single leading Space before our placeholder
+local function squash_spaces(inlines)
+    local out = {}
+    local i = 1
+    while i <= #inlines do
+        local cur = inlines[i]
+        local nxt = inlines[i + 1]
+        if cur and cur.t == "Space" and nxt and nxt.t == "Str" and nxt.text ==
+            PH then
+            table.insert(out, nxt)
+            i = i + 2
+        else
+            table.insert(out, cur)
+            i = i + 1
+        end
+    end
+    return out
+end
+
+function Para(el)
+    el.content = squash_spaces(el.content)
+    return el
+end
+
+function Plain(el)
+    el.content = squash_spaces(el.content)
+    return el
+end
+

thesis/filters/math-omit.lua

@@ -0,0 +1,48 @@
+-- math-omit.lua
+-- Replace any math with a placeholder and ensure a space before it when appropriate.
+local PH = "[math omitted]"
+
+function Math(el)
+    -- Emit the placeholder as a Str; spacing is fixed in Para/Plain below.
+    return pandoc.Str(PH)
+end
+
+local function ensure_space_before_ph(inlines)
+    local out = {}
+    for i = 1, #inlines do
+        local cur = inlines[i]
+        if cur.t == "Str" and cur.text == PH then
+            local prev = out[#out]
+            local need_space = true
+
+            -- No space if it's the first token in the block
+            if not prev then
+                need_space = false
+            elseif prev.t == "Space" then
+                need_space = false
+            elseif prev.t == "Str" then
+                -- If previous char is an opening bracket/paren/slash/hyphen or whitespace, skip
+                local last = prev.text:sub(-1)
+                if last:match("[%(%[%{%/%-]") or last:match("%s") then
+                    need_space = false
+                end
+            end
+
+            if need_space then table.insert(out, pandoc.Space()) end
+            table.insert(out, cur)
+        else
+            table.insert(out, cur)
+        end
+    end
+    return out
+end
+
+function Para(el)
+    el.content = ensure_space_before_ph(el.content)
+    return el
+end
+
+function Plain(el)
+    el.content = ensure_space_before_ph(el.content)
+    return el
+end

thesis/flake.nix

@@ -15,6 +15,8 @@
       let
         pkgs = import nixpkgs { inherit system; };
 
+        aspellWithDicts = pkgs.aspellWithDicts (d: [ d.en ]);
+
         latex-packages = with pkgs; [
           texlive.combined.scheme-full
           which
@@ -25,16 +27,43 @@
           texlab
           zathura
           wmctrl
+          python312
+          pandoc
+          pandoc-lua-filters
         ];
+        filtersPath = "${pkgs.pandoc-lua-filters}/share/pandoc/filters";
       in
       {
         devShell = pkgs.mkShell {
           buildInputs = [
             latex-packages
             dev-packages
+            aspellWithDicts
           ];
         };
 
+        shellHook = ''
+          set -eu
+          # local folder in your repo to reference in commands
+          link_target="pandoc-filters"
+          # refresh symlink each time you enter the shell
+          ln -sfn ${filtersPath} "$link_target"
+          echo "Linked $link_target -> ${filtersPath}"
+
+          # (optional) write a defaults file that uses the relative symlink
+          if [ ! -f pandoc.defaults.yaml ]; then
+            cat > pandoc.defaults.yaml <<'YAML'
+            from: latex
+            to: plain
+            wrap: none
+            lua-filter:
+              - pandoc-filters/latex-hyphen.lua
+              - pandoc-filters/pandoc-quotes.lua
+            YAML
+            echo "Wrote pandoc.defaults.yaml"
+          fi
+        '';
+
       }
     );
 }

thesis/tex2plaintext.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Usage:
+#   ./tex2plaintext.sh [INPUT_TEX] [OUT_BASENAME]
+#
+# Defaults:
+#   INPUT_TEX     = Main.txt     (your original file name)
+#   OUT_BASENAME  = thesis       (produces thesis.txt, thesis_part1.txt, thesis_part2.txt)
+
+INPUT_TEX="${1:-Main.tex}"
+OUT_BASE="${2:-thesis}"
+
+FLAT_TEX="flat.tex"
+NO_TABLES_TEX="flat_notables.tex"
+PLAIN_TXT="${OUT_BASE}.txt"
+PART1_TXT="${OUT_BASE}_part1.txt"
+PART2_TXT="${OUT_BASE}_part2.txt"
+MARKER="Data and Preprocessing"
+
+echo "[1/5] Flattening with latexpand -> ${FLAT_TEX}"
+latexpand "${INPUT_TEX}" > "${FLAT_TEX}"
+
+echo "[2/5] Removing tabular/tabularx environments -> ${NO_TABLES_TEX}"
+# Replace entire tabular / tabularx environments with a placeholder
+perl -0777 -pe 's/\\begin\{(tabularx?)\}.*?\\end\{\1\}/[table omitted]/gs' \
+  "${FLAT_TEX}" > "${NO_TABLES_TEX}"
+
+echo "[3/5] Converting to plain text with pandoc -> ${PLAIN_TXT}"
+pandoc -f latex -t plain --wrap=none \
+  --lua-filter=filters/keep-citations.lua \
+  --lua-filter=filters/math-omit.lua \
+  "${NO_TABLES_TEX}" -o "${PLAIN_TXT}"
+
+echo "[4/5] Replacing [] placeholders with [figure]"
+sed -i 's/\[\]/[figure]/g' "${PLAIN_TXT}"
+
+echo "[5/5] Splitting ${PLAIN_TXT} before the marker line: \"${MARKER}\""
+
+# Ensure the marker exists exactly on its own line
+if ! grep -xq "${MARKER}" "${PLAIN_TXT}"; then
+  echo "ERROR: Marker line not found exactly as \"${MARKER}\" in ${PLAIN_TXT}."
+  echo "       (It must be the only content on that line.)"
+  exit 1
+fi
+
+# Clean previous outputs if present
+rm -f -- "${PART1_TXT}" "${PART2_TXT}"
+
+# Split so the marker line becomes the FIRST line of part 2
+awk -v marker="${MARKER}" -v out1="${PART1_TXT}" -v out2="${PART2_TXT}" '
+BEGIN { current = out1 }
+$0 == marker { current = out2; print $0 > current; next }
+{ print $0 > current }
+' "${PLAIN_TXT}"
+
+echo "Done."
+echo "  - ${PLAIN_TXT}"
+echo "  - ${PART1_TXT}"
+echo "  - ${PART2_TXT}"
+

thesis/thesis_preamble/abstract.tex

@@ -1,3 +1,9 @@
-\addcontentsline{toc}{chapter}{Abstract (English)}
-\begin{center}\Large\bfseries Abstract (English)\end{center}\vspace*{1cm}\noindent 
-Write some fancy abstract here!
+\addcontentsline{toc}{chapter}{Abstract}
+\begin{center}\Large\bfseries Abstract\end{center}\vspace*{1cm}\noindent
+Autonomous robots are increasingly used in search and rescue (SAR) missions. In these missions, LiDAR sensors are often the most important source of environmental data. However, LiDAR data can degrade under hazardous conditions, especially when airborne particles such as smoke or dust are present. This degradation can lead to errors in mapping and navigation and may endanger both the robot and humans. Therefore, robots need a way to estimate the reliability of their LiDAR data, so that they can make better-informed decisions.
+\bigskip
+
+This thesis investigates whether anomaly detection methods can be used to quantify LiDAR data degradation caused by airborne particles such as smoke and dust. We apply a semi-supervised deep learning approach called DeepSAD, which produces an anomaly score for each LiDAR scan, serving as a measure of data reliability.
+\bigskip
+
+We evaluate this method against baseline methods on a subterranean dataset that includes LiDAR scans degraded by artificial smoke. Our results show that DeepSAD consistently outperforms the baselines and can clearly distinguish degraded from normal scans. At the same time, we find that the limited availability of labeled data and the lack of robust ground truth remain major challenges. Despite these limitations, our work demonstrates that anomaly detection methods are a promising tool for LiDAR degradation quantification in SAR scenarios.

thesis/thesis_preamble/acknowledgements.tex

@@ -1,3 +1,3 @@
-\addcontentsline{toc}{chapter}{Acknowledgements}
-\begin{center}\Large\bfseries Acknowledgements\end{center}\vspace*{1cm}\noindent 
-Here you can tell us, how thankful you are for this amazing template ;)
+\addcontentsline{toc}{chapter}{Artificial Intelligence Usage Disclaimer}
+\begin{center}\Large\bfseries Artificial Intelligence Usage Disclaimer\end{center}\vspace*{1cm}\noindent
+During the creation of this thesis, an LLM-based Artificial Intelligence tool was used for stylistic and grammatical revision of the author's own work.

thesis/third_party/PlotNeuralNet/deepsad/Makefile

@@ -1,6 +1,6 @@
 # ====== CONFIG ======
 # Add names (without extension). Example: NAMES = report thesis notes
-NAMES = subter_lenet_arch subter_ef_arch
+NAMES = arch_ef_encoder arch_ef_decoder arch_lenet_encoder arch_lenet_decoder
 
 TEX = $(NAMES:%=%.tex)
 PDF = $(NAMES:%=%.pdf)
@@ -10,7 +10,7 @@ PDF = $(NAMES:%=%.pdf)
 .PRECIOUS: %.tex
 
 # Default: build all PDFs
-all: $(PDF)
+all: $(PDF) $(TEX)
 
 # ====== Rules ======
 # Generate {name}.tex from {name}.py

thesis/third_party/PlotNeuralNet/deepsad/arch_ef_decoder.py

@@ -21,130 +21,17 @@ arch = [
     to_head(".."),
     to_cor(),
     to_begin(),
-    # --------------------------- ENCODER ---------------------------
-    # Input 1×32×2048  (caption carries H×W; s_filer is numeric)
-    to_Conv(
-        "input",
-        s_filer="{{2048×32}}",
-        n_filer=1,
-        offset="(0,0,0)",
-        to="(0,0,0)",
-        height=H32,
-        depth=D2048,
-        width=W1,
-        caption="input",
-    ),
-    # Conv1 (5x5, same): 1->8, 32×2048
-    to_Conv(
-        "dwconv1",
-        s_filer="",
-        n_filer=1,
-        offset="(2,0,0)",
-        to="(input-east)",
-        height=H32,
-        depth=D2048,
-        width=W1,
-        caption="",
-    ),
-    to_Conv(
-        "dwconv2",
-        s_filer="",
-        n_filer=16,
-        offset="(0,0,0)",
-        to="(dwconv1-east)",
-        height=H32,
-        depth=D2048,
-        width=W16,
-        caption="conv1",
-    ),
-    # Pool1 2×2: 32×2048 -> 16×1024
-    # to_connection("input", "conv1"),
-    to_Pool(
-        "pool1",
-        offset="(0,0,0)",
-        to="(dwconv2-east)",
-        height=H32,
-        depth=D512,
-        width=W16,
-        caption="",
-    ),
-    # Conv2 (5x5, same): 8->4, stays 16×1024
-    to_Conv(
-        "dwconv3",
-        s_filer="",
-        n_filer=1,
-        offset="(2,0,0)",
-        to="(pool1-east)",
-        height=H32,
-        depth=D512,
-        width=W1,
-        caption="",
-    ),
-    to_Conv(
-        "dwconv4",
-        s_filer="",
-        n_filer=32,
-        offset="(0,0,0)",
-        to="(dwconv3-east)",
-        height=H32,
-        depth=D512,
-        width=W32,
-        caption="conv2",
-    ),
-    # Pool2 2×2: 16×1024 -> 8×512
-    # to_connection("pool1", "conv2"),
-    to_Pool(
-        "pool2",
-        offset="(0,0,0)",
-        to="(dwconv4-east)",
-        height=H16,
-        depth=D256,
-        width=W32,
-        caption="",
-    ),
-    to_Pool(
-        "pool3",
-        offset="(0,0,0)",
-        to="(pool2-east)",
-        height=H8,
-        depth=D128,
-        width=W32,
-        caption="",
-    ),
-    to_Conv(
-        "squeeze",
-        s_filer="",
-        n_filer=8,
-        offset="(2,0,0)",
-        to="(pool3-east)",
-        height=H8,
-        depth=D128,
-        width=W8,
-        caption="squeeze",
-    ),
-    # FC -> rep_dim (use numeric n_filer)
-    to_fc(
-        "fc1",
-        n_filer="{{8×128×8}}",
-        offset="(2,0,0)",
-        to="(squeeze-east)",
-        height=H1,
-        depth=D512,
-        width=W1,
-        caption=f"FC",
-    ),
-    # to_connection("pool2", "fc1"),
-    # --------------------------- LATENT  ---------------------------
     to_Conv(
         "latent",
         n_filer="",
         s_filer="latent dim",
         offset="(2,0,0)",
-        to="(fc1-east)",
+        to="(0,0,0)",
         height=H8 * 1.6,
         depth=D1,
         width=W1,
-        caption=f"Latent Space",
+        caption="Latent Space",
+        captionshift=0,
     ),
     # to_connection("fc1", "latent"),
     # --------------------------- DECODER ---------------------------
@@ -152,29 +39,33 @@ arch = [
     to_fc(
         "fc3",
         n_filer="{{8×128×8}}",
-        offset="(2,0,0)",
+        zlabeloffset=0.5,
+        offset="(2,-.5,0)",
         to="(latent-east)",
         height=H1,
         depth=D512,
         width=W1,
         caption=f"FC",
+        captionshift=20,
     ),
     to_Conv(
         "unsqueeze",
-        s_filer="",
+        s_filer="{{128×8}}",
+        zlabeloffset=0.4,
         n_filer=32,
-        offset="(2,0,0)",
+        offset="(1.4,0,0)",
         to="(fc3-east)",
         height=H8,
         depth=D128,
         width=W32,
-        caption="unsqueeze",
+        caption="Unsqueeze",
     ),
     # to_connection("latent", "fc3"),
     # Reshape to 4×8×512
     to_UnPool(
         "up1",
-        offset="(2,0,0)",
+        offset="(1.2,0,0)",
+        n_filer=32,
         to="(unsqueeze-east)",
         height=H16,
         depth=D256,
@@ -190,11 +81,12 @@ arch = [
         height=H16,
         depth=D256,
         width=W1,
-        caption="deconv1",
+        caption="Deconv1",
     ),
     to_Conv(
         "dwdeconv2",
         s_filer="{{256×16}}",
+        zlabeloffset=0.4,
         n_filer=32,
         offset="(0,0,0)",
         to="(dwdeconv1-east)",
@@ -207,10 +99,12 @@ arch = [
         "up2",
         offset="(2,0,0)",
         to="(dwdeconv2-east)",
+        n_filer=32,
         height=H16,
         depth=D1024,
         width=W32,
-        caption="",
+        caption="Deconv2",
+        captionshift=20,
     ),
     to_Conv(
         "dwdeconv3",
@@ -221,11 +115,12 @@ arch = [
         height=H16,
         depth=D1024,
         width=W1,
-        caption="deconv2",
+        caption="",
     ),
     to_Conv(
         "dwdeconv4",
         s_filer="{{1024×16}}",
+        zlabeloffset=0.17,
         n_filer=16,
         offset="(0,0,0)",
         to="(dwdeconv3-east)",
@@ -237,11 +132,13 @@ arch = [
     to_UnPool(
         "up3",
         offset="(2,0,0)",
+        n_filer=16,
         to="(dwdeconv4-east)",
         height=H32,
         depth=D2048,
         width=W16,
-        caption="",
+        caption="Deconv3",
+        captionshift=10,
     ),
     to_Conv(
         "dwdeconv5",
@@ -252,11 +149,12 @@ arch = [
         height=H32,
         depth=D2048,
         width=W1,
-        caption="deconv3",
+        caption="",
     ),
     to_Conv(
         "dwdeconv6",
         s_filer="{{2048×32}}",
+        zlabeloffset=0.15,
         n_filer=8,
         offset="(0,0,0)",
         to="(dwdeconv5-east)",
@@ -268,26 +166,29 @@ arch = [
     to_Conv(
         "outconv",
         s_filer="{{2048×32}}",
+        zlabeloffset=0.15,
         n_filer=1,
-        offset="(2,0,0)",
+        offset="(1.5,0,0)",
         to="(dwdeconv6-east)",
         height=H32,
         depth=D2048,
         width=W1,
-        caption="deconv4",
+        caption="Deconv4",
     ),
     # to_connection("up2", "deconv2"),
     # Output
     to_Conv(
         "out",
         s_filer="{{2048×32}}",
+        zlabeloffset=0.15,
         n_filer=1,
-        offset="(2,0,0)",
+        offset="(1.5,0,0)",
         to="(outconv-east)",
         height=H32,
         depth=D2048,
         width=W1,
-        caption="output",
+        caption="Output",
+        captionshift=5,
     ),
     # to_connection("deconv2", "out"),
     to_end(),

thesis/third_party/PlotNeuralNet/deepsad/arch_ef_decoder.tex

@@ -24,149 +24,13 @@
 \tikzstyle{copyconnection}=[ultra thick,every node/.style={sloped,allow upside down},draw={rgb:blue,4;red,1;green,1;black,3},opacity=0.7]
 
 
-\pic[shift={(0,0,0)}] at (0,0,0) 
-    {Box={
-        name=input,
-        caption=input,
-        xlabel={{1, }},
-        zlabel={{2048×32}},
-        fill=\ConvColor,
-        height=26,
-        width=1,
-        depth=52
-        }
-    };
-
-
-\pic[shift={(2,0,0)}] at (input-east) 
-    {Box={
-        name=dwconv1,
-        caption=,
-        xlabel={{1, }},
-        zlabel=,
-        fill=\ConvColor,
-        height=26,
-        width=1,
-        depth=52
-        }
-    };
-
-
-\pic[shift={(0,0,0)}] at (dwconv1-east) 
-    {Box={
-        name=dwconv2,
-        caption=conv1,
-        xlabel={{16, }},
-        zlabel=,
-        fill=\ConvColor,
-        height=26,
-        width=4,
-        depth=52
-        }
-    };
-
-
-\pic[shift={ (0,0,0) }] at (dwconv2-east) 
-    {Box={
-        name=pool1,
-        caption=,
-        fill=\PoolColor,
-        opacity=0.5,
-        height=26,
-        width=4,
-        depth=24
-        }
-    };
-
-
-\pic[shift={(2,0,0)}] at (pool1-east) 
-    {Box={
-        name=dwconv3,
-        caption=,
-        xlabel={{1, }},
-        zlabel=,
-        fill=\ConvColor,
-        height=26,
-        width=1,
-        depth=24
-        }
-    };
-
-
-\pic[shift={(0,0,0)}] at (dwconv3-east) 
-    {Box={
-        name=dwconv4,
-        caption=conv2,
-        xlabel={{32, }},
-        zlabel=,
-        fill=\ConvColor,
-        height=26,
-        width=8,
-        depth=24
-        }
-    };
-
-
-\pic[shift={ (0,0,0) }] at (dwconv4-east) 
-    {Box={
-        name=pool2,
-        caption=,
-        fill=\PoolColor,
-        opacity=0.5,
-        height=18,
-        width=8,
-        depth=12
-        }
-    };
-
-
-\pic[shift={ (0,0,0) }] at (pool2-east) 
-    {Box={
-        name=pool3,
-        caption=,
-        fill=\PoolColor,
-        opacity=0.5,
-        height=12,
-        width=8,
-        depth=6
-        }
-    };
-
-
-\pic[shift={(2,0,0)}] at (pool3-east) 
-    {Box={
-        name=squeeze,
-        caption=squeeze,
-        xlabel={{8, }},
-        zlabel=,
-        fill=\ConvColor,
-        height=12,
-        width=2,
-        depth=6
-        }
-    };
-
-
-\pic[shift={(2,0,0)}] at (squeeze-east)
-    {Box={
-        name=fc1,
-        caption=FC,
-        xlabel={{" ","dummy"}},
-        zlabel={{8×128×8}},
-        fill=\FcColor,
-        opacity=0.8,
-        height=1,
-        width=1,
-        depth=24
-        }
-    };
-
-
-\pic[shift={(2,0,0)}] at (fc1-east) 
+\pic[shift={(2,0,0)}] at (0,0,0) 
     {Box={
         name=latent,
         caption=Latent Space,
+        captionshift=0,
         xlabel={{, }},
+        zlabeloffset=0.3,
         zlabel=latent dim,
         fill=\ConvColor,
         height=19.200000000000003,
@@ -176,11 +40,13 @@
     };
 
 
-\pic[shift={(2,0,0)}] at (latent-east)
+\pic[shift={(2,-.5,0)}] at (latent-east)
     {Box={
         name=fc3,
         caption=FC,
+        captionshift=20,
         xlabel={{" ","dummy"}},
+        zlabeloffset=0.5,
         zlabel={{8×128×8}},
         fill=\FcColor,
         opacity=0.8,
@@ -191,12 +57,14 @@
     };
 
 
-\pic[shift={(2,0,0)}] at (fc3-east) 
+\pic[shift={(1.4,0,0)}] at (fc3-east) 
     {Box={
         name=unsqueeze,
-        caption=unsqueeze,
+        caption=Unsqueeze,
+        captionshift=0,
         xlabel={{32, }},
-        zlabel=,
+        zlabeloffset=0.4,
+        zlabel={{128×8}},
         fill=\ConvColor,
         height=12,
         width=8,
@@ -205,12 +73,14 @@
     };
 
 
-\pic[shift={ (2,0,0) }] at (unsqueeze-east) 
+\pic[shift={ (1.2,0,0) }] at (unsqueeze-east) 
     {Box={
         name=up1,
         caption=,
+        captionshift=0,
         fill=\UnpoolColor,
         opacity=0.5,
+        xlabel={{32, }},
         height=18,
         width=8,
         depth=12
@@ -221,8 +91,10 @@
 \pic[shift={(0,0,0)}] at (up1-east) 
     {Box={
         name=dwdeconv1,
-        caption=deconv1,
+        caption=Deconv1,
+        captionshift=0,
         xlabel={{1, }},
+        zlabeloffset=0.3,
         zlabel=,
         fill=\ConvColor,
         height=18,
@@ -236,7 +108,9 @@
     {Box={
         name=dwdeconv2,
         caption=,
+        captionshift=0,
         xlabel={{32, }},
+        zlabeloffset=0.4,
         zlabel={{256×16}},
         fill=\ConvColor,
         height=18,
@@ -249,9 +123,11 @@
 \pic[shift={ (2,0,0) }] at (dwdeconv2-east) 
     {Box={
         name=up2,
-        caption=,
+        caption=Deconv2,
+        captionshift=20,
         fill=\UnpoolColor,
         opacity=0.5,
+        xlabel={{32, }},
         height=18,
         width=8,
         depth=36
@@ -262,8 +138,10 @@
 \pic[shift={(0,0,0)}] at (up2-east) 
     {Box={
         name=dwdeconv3,
-        caption=deconv2,
+        caption=,
+        captionshift=0,
         xlabel={{1, }},
+        zlabeloffset=0.3,
         zlabel=,
         fill=\ConvColor,
         height=18,
@@ -277,7 +155,9 @@
     {Box={
         name=dwdeconv4,
         caption=,
+        captionshift=0,
         xlabel={{16, }},
+        zlabeloffset=0.17,
         zlabel={{1024×16}},
         fill=\ConvColor,
         height=18,
@@ -290,9 +170,11 @@
 \pic[shift={ (2,0,0) }] at (dwdeconv4-east) 
     {Box={
         name=up3,
-        caption=,
+        caption=Deconv3,
+        captionshift=10,
         fill=\UnpoolColor,
         opacity=0.5,
+        xlabel={{16, }},
         height=26,
         width=4,
         depth=52
@@ -303,8 +185,10 @@
 \pic[shift={(0,0,0)}] at (up3-east) 
     {Box={
         name=dwdeconv5,
-        caption=deconv3,
+        caption=,
+        captionshift=0,
         xlabel={{1, }},
+        zlabeloffset=0.3,
         zlabel=,
         fill=\ConvColor,
         height=26,
@@ -318,7 +202,9 @@
     {Box={
         name=dwdeconv6,
         caption=,
+        captionshift=0,
         xlabel={{8, }},
+        zlabeloffset=0.15,
         zlabel={{2048×32}},
         fill=\ConvColor,
         height=26,
@@ -328,11 +214,13 @@
     };
 
 
-\pic[shift={(2,0,0)}] at (dwdeconv6-east) 
+\pic[shift={(1.5,0,0)}] at (dwdeconv6-east) 
     {Box={
         name=outconv,
-        caption=deconv4,
+        caption=Deconv4,
+        captionshift=0,
         xlabel={{1, }},
+        zlabeloffset=0.15,
         zlabel={{2048×32}},
         fill=\ConvColor,
         height=26,
@@ -342,11 +230,13 @@
     };
 
 
-\pic[shift={(2,0,0)}] at (outconv-east) 
+\pic[shift={(1.5,0,0)}] at (outconv-east) 
     {Box={
         name=out,
-        caption=output,
+        caption=Output,
+        captionshift=5,
         xlabel={{1, }},
+        zlabeloffset=0.15,
         zlabel={{2048×32}},
         fill=\ConvColor,
         height=26,

thesis/third_party/PlotNeuralNet/deepsad/arch_ef_encoder.py

@@ -0,0 +1,171 @@
+# subter_lenet_arch.py
+# Requires running from inside the PlotNeuralNet repo, like: python3 ../subter_lenet_arch.py
+import sys, argparse
+
+sys.path.append("../")  # import pycore from repo root
+
+from pycore.tikzeng import *
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--rep_dim", type=int, default=1024, help="latent size for FC")
+args = parser.parse_args()
+REP = int(args.rep_dim)
+
+# Visual scales so the huge width doesn't dominate the figure
+H32, H16, H8, H1 = 26, 18, 12, 1
+D2048, D1024, D512, D256, D128, D1 = 52, 36, 24, 12, 6, 1
+W1, W4, W8, W16, W32 = 1, 2, 2, 4, 8
+
+
+arch = [
+    to_head(".."),
+    to_cor(),
+    to_begin(),
+    # --------------------------- ENCODER ---------------------------
+    # Input 1×32×2048  (caption carries H×W; s_filer is numeric)
+    to_Conv(
+        "input",
+        zlabeloffset=0.2,
+        s_filer="{{2048×32}}",
+        n_filer=1,
+        offset="(0,0,0)",
+        to="(0,0,0)",
+        height=H32,
+        depth=D2048,
+        width=W1,
+        caption="Input",
+    ),
+    # Conv1 (5x5, same): 1->8, 32×2048
+    to_Conv(
+        "dwconv1",
+        s_filer="",
+        n_filer=1,
+        offset="(2,0,0)",
+        to="(input-east)",
+        height=H32,
+        depth=D2048,
+        width=W1,
+        caption="",
+    ),
+    to_Conv(
+        "dwconv2",
+        s_filer="{{2048×32}}",
+        zlabeloffset=0.15,
+        n_filer=16,
+        offset="(0,0,0)",
+        to="(dwconv1-east)",
+        height=H32,
+        depth=D2048,
+        width=W16,
+        caption="Conv1",
+    ),
+    # Pool1 2×2: 32×2048 -> 16×1024
+    # to_connection("input", "conv1"),
+    to_Pool(
+        "pool1",
+        offset="(0,0,0)",
+        zlabeloffset=0.3,
+        s_filer="{{512×32}}",
+        to="(dwconv2-east)",
+        height=H32,
+        depth=D512,
+        width=W16,
+        caption="",
+    ),
+    # Conv2 (5x5, same): 8->4, stays 16×1024
+    to_Conv(
+        "dwconv3",
+        s_filer="",
+        n_filer=1,
+        offset="(2,0,0)",
+        to="(pool1-east)",
+        height=H32,
+        depth=D512,
+        width=W1,
+        caption="",
+    ),
+    to_Conv(
+        "dwconv4",
+        n_filer=32,
+        zlabeloffset=0.3,
+        s_filer="{{512×32}}",
+        offset="(0,0,0)",
+        to="(dwconv3-east)",
+        height=H32,
+        depth=D512,
+        width=W32,
+        caption="Conv2",
+    ),
+    # Pool2 2×2: 16×1024 -> 8×512
+    # to_connection("pool1", "conv2"),
+    to_Pool(
+        "pool2",
+        offset="(0,0,0)",
+        zlabeloffset=0.45,
+        s_filer="{{256×16}}",
+        to="(dwconv4-east)",
+        height=H16,
+        depth=D256,
+        width=W32,
+        caption="",
+    ),
+    to_Pool(
+        "pool3",
+        offset="(0,0,0)",
+        zlabeloffset=0.45,
+        s_filer="{{128×8}}",
+        to="(pool2-east)",
+        height=H8,
+        depth=D128,
+        width=W32,
+        caption="",
+    ),
+    to_Conv(
+        "squeeze",
+        n_filer=8,
+        zlabeloffset=0.45,
+        s_filer="{{128×8}}",
+        offset="(1,0,0)",
+        to="(pool3-east)",
+        height=H8,
+        depth=D128,
+        width=W8,
+        caption="Squeeze",
+    ),
+    # FC -> rep_dim (use numeric n_filer)
+    to_fc(
+        "fc1",
+        n_filer="{{8×128×8}}",
+        zlabeloffset=0.5,
+        offset="(2,-.5,0)",
+        to="(squeeze-east)",
+        height=H1,
+        depth=D512,
+        width=W1,
+        caption="FC",
+        captionshift=0,
+    ),
+    # to_connection("pool2", "fc1"),
+    # --------------------------- LATENT  ---------------------------
+    to_Conv(
+        "latent",
+        n_filer="",
+        s_filer="latent dim",
+        offset="(1.3,0.5,0)",
+        to="(fc1-east)",
+        height=H8 * 1.6,
+        depth=D1,
+        width=W1,
+        caption=f"Latent Space",
+    ),
+    to_end(),
+]
+
+
+def main():
+    name = "subter_lenet_arch"
+    to_generate(arch, name + ".tex")
+
+
+if __name__ == "__main__":
+    main()

thesis/third_party/PlotNeuralNet/deepsad/arch_ef_encoder.tex

@@ -0,0 +1,209 @@
+
+\documentclass[border=8pt, multi, tikz]{standalone} 
+\usepackage{import}
+\subimport{../layers/}{init}
+\usetikzlibrary{positioning}
+\usetikzlibrary{3d} %for including external image 
+
+
+\def\ConvColor{rgb:yellow,5;red,2.5;white,5}
+\def\ConvReluColor{rgb:yellow,5;red,5;white,5}
+\def\PoolColor{rgb:red,1;black,0.3}
+\def\UnpoolColor{rgb:blue,2;green,1;black,0.3}
+\def\FcColor{rgb:blue,5;red,2.5;white,5}
+\def\FcReluColor{rgb:blue,5;red,5;white,4}
+\def\SoftmaxColor{rgb:magenta,5;black,7}   
+\def\SumColor{rgb:blue,5;green,15}
+
+
+\newcommand{\copymidarrow}{\tikz \draw[-Stealth,line width=0.8mm,draw={rgb:blue,4;red,1;green,1;black,3}] (-0.3,0) -- ++(0.3,0);}
+
+\begin{document}
+\begin{tikzpicture}
+\tikzstyle{connection}=[ultra thick,every node/.style={sloped,allow upside down},draw=\edgecolor,opacity=0.7]
+\tikzstyle{copyconnection}=[ultra thick,every node/.style={sloped,allow upside down},draw={rgb:blue,4;red,1;green,1;black,3},opacity=0.7]
+
+
+\pic[shift={(0,0,0)}] at (0,0,0) 
+    {Box={
+        name=input,
+        caption=Input,
+        captionshift=0,
+        xlabel={{1, }},
+        zlabeloffset=0.2,
+        zlabel={{2048×32}},
+        fill=\ConvColor,
+        height=26,
+        width=1,
+        depth=52
+        }
+    };
+
+
+\pic[shift={(2,0,0)}] at (input-east) 
+    {Box={
+        name=dwconv1,
+        caption=,
+        captionshift=0,
+        xlabel={{1, }},
+        zlabeloffset=0.3,
+        zlabel=,
+        fill=\ConvColor,
+        height=26,
+        width=1,
+        depth=52
+        }
+    };
+
+
+\pic[shift={(0,0,0)}] at (dwconv1-east) 
+    {Box={
+        name=dwconv2,
+        caption=Conv1,
+        captionshift=0,
+        xlabel={{16, }},
+        zlabeloffset=0.15,
+        zlabel={{2048×32}},
+        fill=\ConvColor,
+        height=26,
+        width=4,
+        depth=52
+        }
+    };
+
+
+\pic[shift={ (0,0,0) }] at (dwconv2-east) 
+    {Box={
+        name=pool1,
+        xlabel={{, }},
+        zlabeloffset=0.3,
+        zlabel={{512×32}},
+        caption=,
+        captionshift=0,
+        fill=\PoolColor,
+        opacity=0.5,
+        height=26,
+        width=4,
+        depth=24
+        }
+    };
+
+
+\pic[shift={(2,0,0)}] at (pool1-east) 
+    {Box={
+        name=dwconv3,
+        caption=,
+        captionshift=0,
+        xlabel={{1, }},
+        zlabeloffset=0.3,
+        zlabel=,
+        fill=\ConvColor,
+        height=26,
+        width=1,
+        depth=24
+        }
+    };
+
+
+\pic[shift={(0,0,0)}] at (dwconv3-east) 
+    {Box={
+        name=dwconv4,
+        caption=Conv2,
+        captionshift=0,
+        xlabel={{32, }},
+        zlabeloffset=0.3,
+        zlabel={{512×32}},
+        fill=\ConvColor,
+        height=26,
+        width=8,
+        depth=24
+        }
+    };
+
+
+\pic[shift={ (0,0,0) }] at (dwconv4-east) 
+    {Box={
+        name=pool2,
+        xlabel={{, }},
+        zlabeloffset=0.45,
+        zlabel={{256×16}},
+        caption=,
+        captionshift=0,
+        fill=\PoolColor,
+        opacity=0.5,
+        height=18,
+        width=8,
+        depth=12
+        }
+    };
+
+
+\pic[shift={ (0,0,0) }] at (pool2-east) 
+    {Box={
+        name=pool3,
+        xlabel={{, }},
+        zlabeloffset=0.45,
+        zlabel={{128×8}},
+        caption=,
+        captionshift=0,
+        fill=\PoolColor,
+        opacity=0.5,
+        height=12,
+        width=8,
+        depth=6
+        }
+    };
+
+
+\pic[shift={(1,0,0)}] at (pool3-east) 
+    {Box={
+        name=squeeze,
+        caption=Squeeze,
+        captionshift=0,
+        xlabel={{8, }},
+        zlabeloffset=0.45,
+        zlabel={{128×8}},
+        fill=\ConvColor,
+        height=12,
+        width=2,
+        depth=6
+        }
+    };
+
+
+\pic[shift={(2,-.5,0)}] at (squeeze-east)
+    {Box={
+        name=fc1,
+        caption=FC,
+        captionshift=0,
+        xlabel={{" ","dummy"}},
+        zlabeloffset=0.5,
+        zlabel={{8×128×8}},
+        fill=\FcColor,
+        opacity=0.8,
+        height=1,
+        width=1,
+        depth=24
+        }
+    };
+
+
+\pic[shift={(1.3,0.5,0)}] at (fc1-east) 
+    {Box={
+        name=latent,
+        caption=Latent Space,
+        captionshift=0,
+        xlabel={{, }},
+        zlabeloffset=0.3,
+        zlabel=latent dim,
+        fill=\ConvColor,
+        height=19.200000000000003,
+        width=1,
+        depth=1
+        }
+    };
+
+
+\end{tikzpicture}
+\end{document}
+

thesis/third_party/PlotNeuralNet/deepsad/arch_lenet_decoder.py

@@ -21,84 +21,12 @@ arch = [
     to_head(".."),
     to_cor(),
     to_begin(),
-    # --------------------------- ENCODER ---------------------------
-    # Input 1×32×2048  (caption carries H×W; s_filer is numeric)
-    to_Conv(
-        "input",
-        s_filer="{{2048×32}}",
-        n_filer=1,
-        offset="(0,0,0)",
-        to="(0,0,0)",
-        height=H32,
-        depth=D2048,
-        width=W1,
-        caption="input",
-    ),
-    # Conv1 (5x5, same): 1->8, 32×2048
-    to_Conv(
-        "conv1",
-        s_filer="{{1024×16}}",
-        n_filer=8,
-        offset="(2,0,0)",
-        to="(input-east)",
-        height=H32,
-        depth=D2048,
-        width=W8,
-        caption="conv1",
-    ),
-    # Pool1 2×2: 32×2048 -> 16×1024
-    # to_connection("input", "conv1"),
-    to_Pool(
-        "pool1",
-        offset="(0,0,0)",
-        to="(conv1-east)",
-        height=H16,
-        depth=D1024,
-        width=W8,
-        caption="",
-    ),
-    # Conv2 (5x5, same): 8->4, stays 16×1024
-    to_Conv(
-        "conv2",
-        s_filer="{{512×8}}",
-        n_filer=4,
-        offset="(2,0,0)",
-        to="(pool1-east)",
-        height=H16,
-        depth=D1024,
-        width=W4,
-        caption="conv2",
-    ),
-    # Pool2 2×2: 16×1024 -> 8×512
-    # to_connection("pool1", "conv2"),
-    to_Pool(
-        "pool2",
-        offset="(0,0,0)",
-        to="(conv2-east)",
-        height=H8,
-        depth=D512,
-        width=W4,
-        caption="",
-    ),
-    # FC -> rep_dim (use numeric n_filer)
-    to_fc(
-        "fc1",
-        n_filer="{{4×512×8}}",
-        offset="(2,0,0)",
-        to="(pool2-east)",
-        height=1.3,
-        depth=D512,
-        width=W1,
-        caption=f"FC",
-    ),
-    # to_connection("pool2", "fc1"),
-    # --------------------------- LATENT  ---------------------------
     to_Conv(
         "latent",
         n_filer="",
         s_filer="latent dim",
         offset="(2,0,0)",
-        to="(fc1-east)",
+        to="(0,0,0)",
         height=H8 * 1.6,
         depth=1.3,
         width=W1,
@@ -110,18 +38,21 @@ arch = [
     to_fc(
         "fc3",
         n_filer="{{4×512×8}}",
-        offset="(2,0,0)",
+        zlabeloffset=0.35,
+        offset="(2,-.5,0)",
         to="(latent-east)",
         height=1.3,
         depth=D512,
         width=W1,
         caption=f"FC",
+        captionshift=20,
     ),
     # to_connection("latent", "fc3"),
     # Reshape to 4×8×512
     to_UnPool(
         "up1",
-        offset="(2,0,0)",
+        n_filer=4,
+        offset="(2.5,0,0)",
         to="(fc3-east)",
         height=H16,
         depth=D1024,
@@ -133,50 +64,56 @@ arch = [
     to_Conv(
         "deconv1",
         s_filer="{{1024×16}}",
+        zlabeloffset=0.2,
         n_filer=8,
         offset="(0,0,0)",
         to="(up1-east)",
         height=H16,
         depth=D1024,
         width=W8,
-        caption="deconv1",
+        caption="Deconv1",
     ),
     # to_connection("fc3", "up1"),
     # Up ×2: 16×1024 -> 32×2048
     to_UnPool(
         "up2",
         offset="(2,0,0)",
+        n_filer=8,
         to="(deconv1-east)",
         height=H32,
         depth=D2048,
         width=W8,
-        caption="",
+        caption="Deconv2",
+        captionshift=10,
     ),
     # to_connection("deconv1", "up2"),
     # DeConv2 (5×5, same): 8->1, 32×2048
     to_Conv(
         "deconv2",
         s_filer="{{2048×32}}",
+        zlabeloffset=0.15,
         n_filer=1,
         offset="(0,0,0)",
         to="(up2-east)",
         height=H32,
         depth=D2048,
         width=W1,
-        caption="deconv2",
+        caption="",
     ),
     # to_connection("up2", "deconv2"),
     # Output
     to_Conv(
         "out",
         s_filer="{{2048×32}}",
+        zlabeloffset=0.15,
         n_filer=1,
         offset="(2,0,0)",
         to="(deconv2-east)",
         height=H32,
         depth=D2048,
         width=1.0,
-        caption="output",
+        caption="Output",
+        captionshift=5,
     ),
     # to_connection("deconv2", "out"),
     to_end(),

thesis/third_party/PlotNeuralNet/deepsad/arch_lenet_decoder.tex

@@ -0,0 +1,140 @@
+
+\documentclass[border=8pt, multi, tikz]{standalone} 
+\usepackage{import}
+\subimport{../layers/}{init}
+\usetikzlibrary{positioning}
+\usetikzlibrary{3d} %for including external image 
+
+
+\def\ConvColor{rgb:yellow,5;red,2.5;white,5}
+\def\ConvReluColor{rgb:yellow,5;red,5;white,5}
+\def\PoolColor{rgb:red,1;black,0.3}
+\def\UnpoolColor{rgb:blue,2;green,1;black,0.3}
+\def\FcColor{rgb:blue,5;red,2.5;white,5}
+\def\FcReluColor{rgb:blue,5;red,5;white,4}
+\def\SoftmaxColor{rgb:magenta,5;black,7}   
+\def\SumColor{rgb:blue,5;green,15}
+
+
+\newcommand{\copymidarrow}{\tikz \draw[-Stealth,line width=0.8mm,draw={rgb:blue,4;red,1;green,1;black,3}] (-0.3,0) -- ++(0.3,0);}
+
+\begin{document}
+\begin{tikzpicture}
+\tikzstyle{connection}=[ultra thick,every node/.style={sloped,allow upside down},draw=\edgecolor,opacity=0.7]
+\tikzstyle{copyconnection}=[ultra thick,every node/.style={sloped,allow upside down},draw={rgb:blue,4;red,1;green,1;black,3},opacity=0.7]
+
+
+\pic[shift={(2,0,0)}] at (0,0,0) 
+    {Box={
+        name=latent,
+        caption=Latent Space,
+        captionshift=0,
+        xlabel={{, }},
+        zlabeloffset=0.3,
+        zlabel=latent dim,
+        fill=\ConvColor,
+        height=19.200000000000003,
+        width=1,
+        depth=1.3
+        }
+    };
+
+
+\pic[shift={(2,-.5,0)}] at (latent-east)
+    {Box={
+        name=fc3,
+        caption=FC,
+        captionshift=20,
+        xlabel={{" ","dummy"}},
+        zlabeloffset=0.35,
+        zlabel={{4×512×8}},
+        fill=\FcColor,
+        opacity=0.8,
+        height=1.3,
+        width=1,
+        depth=24
+        }
+    };
+
+
+\pic[shift={ (2.5,0,0) }] at (fc3-east) 
+    {Box={
+        name=up1,
+        caption=,
+        captionshift=0,
+        fill=\UnpoolColor,
+        opacity=0.5,
+        xlabel={{4, }},
+        height=18,
+        width=2,
+        depth=36
+        }
+    };
+
+
+\pic[shift={(0,0,0)}] at (up1-east) 
+    {Box={
+        name=deconv1,
+        caption=Deconv1,
+        captionshift=0,
+        xlabel={{8, }},
+        zlabeloffset=0.2,
+        zlabel={{1024×16}},
+        fill=\ConvColor,
+        height=18,
+        width=4,
+        depth=36
+        }
+    };
+
+
+\pic[shift={ (2,0,0) }] at (deconv1-east) 
+    {Box={
+        name=up2,
+        caption=Deconv2,
+        captionshift=10,
+        fill=\UnpoolColor,
+        opacity=0.5,
+        xlabel={{8, }},
+        height=26,
+        width=4,
+        depth=52
+        }
+    };
+
+
+\pic[shift={(0,0,0)}] at (up2-east) 
+    {Box={
+        name=deconv2,
+        caption=,
+        captionshift=0,
+        xlabel={{1, }},
+        zlabeloffset=0.15,
+        zlabel={{2048×32}},
+        fill=\ConvColor,
+        height=26,
+        width=1,
+        depth=52
+        }
+    };
+
+
+\pic[shift={(2,0,0)}] at (deconv2-east) 
+    {Box={
+        name=out,
+        caption=Output,
+        captionshift=5,
+        xlabel={{1, }},
+        zlabeloffset=0.15,
+        zlabel={{2048×32}},
+        fill=\ConvColor,
+        height=26,
+        width=1.0,
+        depth=52
+        }
+    };
+
+
+\end{tikzpicture}
+\end{document}
+

thesis/third_party/PlotNeuralNet/deepsad/arch_lenet_encoder.py

@@ -0,0 +1,126 @@
+# subter_lenet_arch.py
+# Requires running from inside the PlotNeuralNet repo, like: python3 ../subter_lenet_arch.py
+import sys, argparse
+
+sys.path.append("../")  # import pycore from repo root
+
+from pycore.tikzeng import *
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--rep_dim", type=int, default=1024, help="latent size for FC")
+args = parser.parse_args()
+REP = int(args.rep_dim)
+
+# Visual scales so the huge width doesn't dominate the figure
+H32, H16, H8 = 26, 18, 12
+D2048, D1024, D512 = 52, 36, 24
+W1, W4, W8 = 1, 2, 4
+
+
+arch = [
+    to_head(".."),
+    to_cor(),
+    to_begin(),
+    # --------------------------- ENCODER ---------------------------
+    # Input 1×32×2048  (caption carries H×W; s_filer is numeric)
+    to_Conv(
+        "input",
+        s_filer="{{2048×32}}",
+        zlabeloffset=0.15,
+        n_filer=1,
+        offset="(0,0,0)",
+        to="(0,0,0)",
+        height=H32,
+        depth=D2048,
+        width=W1,
+        caption="Input",
+    ),
+    # Conv1 (5x5, same): 1->8, 32×2048
+    to_Conv(
+        "conv1",
+        s_filer="{{2048×32}}",
+        zlabeloffset=0.15,
+        n_filer=8,
+        offset="(2,0,0)",
+        to="(input-east)",
+        height=H32,
+        depth=D2048,
+        width=W8,
+        caption="Conv1",
+    ),
+    # Pool1 2×2: 32×2048 -> 16×1024
+    # to_connection("input", "conv1"),
+    to_Pool(
+        "pool1",
+        offset="(0,0,0)",
+        s_filer="{{1024×16}}",
+        zlabeloffset=0.3,
+        to="(conv1-east)",
+        height=H16,
+        depth=D1024,
+        width=W8,
+        caption="",
+    ),
+    # Conv2 (5x5, same): 8->4, stays 16×1024
+    to_Conv(
+        "conv2",
+        s_filer="{{1024×16\hspace{2.5em}512×8}}",
+        zlabeloffset=0.4,
+        n_filer=4,
+        offset="(2,0,0)",
+        to="(pool1-east)",
+        height=H16,
+        depth=D1024,
+        width=W4,
+        caption="Conv2",
+    ),
+    # Pool2 2×2: 16×1024 -> 8×512
+    # to_connection("pool1", "conv2"),
+    to_Pool(
+        "pool2",
+        s_filer="{{}}",
+        offset="(0,0,0)",
+        zlabeloffset=0.3,
+        to="(conv2-east)",
+        height=H8,
+        depth=D512,
+        width=W4,
+        caption="",
+    ),
+    # FC -> rep_dim (use numeric n_filer)
+    to_fc(
+        "fc1",
+        n_filer="{{4×512×8}}",
+        offset="(2,-.5,0)",
+        zlabeloffset=0.5,
+        to="(pool2-east)",
+        height=1.3,
+        depth=D512,
+        width=W1,
+        caption=f"FC",
+        captionshift=20,
+    ),
+    # to_connection("pool2", "fc1"),
+    # --------------------------- LATENT  ---------------------------
+    to_Conv(
+        "latent",
+        n_filer="",
+        s_filer="latent dim",
+        offset="(2,0,0)",
+        to="(fc1-east)",
+        height=H8 * 1.6,
+        depth=1.3,
+        width=W1,
+        caption=f"Latent Space",
+    ),
+    to_end(),
+]
+
+
+def main():
+    name = "subter_lenet_arch"
+    to_generate(arch, name + ".tex")
+
+
+if __name__ == "__main__":
+    main()

thesis/third_party/PlotNeuralNet/deepsad/arch_lenet_encoder.tex

@@ -27,8 +27,10 @@
 \pic[shift={(0,0,0)}] at (0,0,0) 
     {Box={
         name=input,
-        caption=input,
+        caption=Input,
+        captionshift=0,
         xlabel={{1, }},
+        zlabeloffset=0.15,
         zlabel={{2048×32}},
         fill=\ConvColor,
         height=26,
@@ -41,9 +43,11 @@
 \pic[shift={(2,0,0)}] at (input-east) 
     {Box={
         name=conv1,
-        caption=conv1,
+        caption=Conv1,
+        captionshift=0,
         xlabel={{8, }},
-        zlabel={{1024×16}},
+        zlabeloffset=0.15,
+        zlabel={{2048×32}},
         fill=\ConvColor,
         height=26,
         width=4,
@@ -55,7 +59,11 @@
 \pic[shift={ (0,0,0) }] at (conv1-east) 
     {Box={
         name=pool1,
+        xlabel={{, }},
+        zlabeloffset=0.3,
+        zlabel={{1024×16}},
         caption=,
+        captionshift=0,
         fill=\PoolColor,
         opacity=0.5,
         height=18,
@@ -68,9 +76,11 @@
 \pic[shift={(2,0,0)}] at (pool1-east) 
     {Box={
         name=conv2,
-        caption=conv2,
+        caption=Conv2,
+        captionshift=0,
         xlabel={{4, }},
-        zlabel={{512×8}},
+        zlabeloffset=0.4,
+        zlabel={{1024×16\hspace{2.5em}512×8}},
         fill=\ConvColor,
         height=18,
         width=2,
@@ -82,7 +92,11 @@
 \pic[shift={ (0,0,0) }] at (conv2-east) 
     {Box={
         name=pool2,
+        xlabel={{, }},
+        zlabeloffset=0.3,
+        zlabel={{}},
         caption=,
+        captionshift=0,
         fill=\PoolColor,
         opacity=0.5,
         height=12,
@@ -92,11 +106,13 @@
     };
 
 
-\pic[shift={(2,0,0)}] at (pool2-east)
+\pic[shift={(2,-.5,0)}] at (pool2-east)
     {Box={
         name=fc1,
         caption=FC,
+        captionshift=20,
         xlabel={{" ","dummy"}},
+        zlabeloffset=0.5,
         zlabel={{4×512×8}},
         fill=\FcColor,
         opacity=0.8,
@@ -111,7 +127,9 @@
     {Box={
         name=latent,
         caption=Latent Space,
+        captionshift=0,
         xlabel={{, }},
+        zlabeloffset=0.3,
         zlabel=latent dim,
         fill=\ConvColor,
         height=19.200000000000003,
@@ -121,89 +139,6 @@
     };
 
 
-\pic[shift={(2,0,0)}] at (latent-east)
-    {Box={
-        name=fc3,
-        caption=FC,
-        xlabel={{" ","dummy"}},
-        zlabel={{4×512×8}},
-        fill=\FcColor,
-        opacity=0.8,
-        height=1.3,
-        width=1,
-        depth=24
-        }
-    };
-
-
-\pic[shift={ (2,0,0) }] at (fc3-east) 
-    {Box={
-        name=up1,
-        caption=,
-        fill=\UnpoolColor,
-        opacity=0.5,
-        height=18,
-        width=2,
-        depth=36
-        }
-    };
-
-
-\pic[shift={(0,0,0)}] at (up1-east) 
-    {Box={
-        name=deconv1,
-        caption=deconv1,
-        xlabel={{8, }},
-        zlabel={{1024×16}},
-        fill=\ConvColor,
-        height=18,
-        width=4,
-        depth=36
-        }
-    };
-
-
-\pic[shift={ (2,0,0) }] at (deconv1-east) 
-    {Box={
-        name=up2,
-        caption=,
-        fill=\UnpoolColor,
-        opacity=0.5,
-        height=26,
-        width=4,
-        depth=52
-        }
-    };
-
-
-\pic[shift={(0,0,0)}] at (up2-east) 
-    {Box={
-        name=deconv2,
-        caption=deconv2,
-        xlabel={{1, }},
-        zlabel={{2048×32}},
-        fill=\ConvColor,
-        height=26,
-        width=1,
-        depth=52
-        }
-    };
-
-
-\pic[shift={(2,0,0)}] at (deconv2-east) 
-    {Box={
-        name=out,
-        caption=output,
-        xlabel={{1, }},
-        zlabel={{2048×32}},
-        fill=\ConvColor,
-        height=26,
-        width=1.0,
-        depth=52
-        }
-    };
-
-
 \end{tikzpicture}
 \end{document}
 

thesis/third_party/PlotNeuralNet/layers/Box.sty

@@ -42,14 +42,27 @@
 
 \coordinate (a1) at (0 , \y/2 , \z/2);
 \coordinate (b1) at (0 ,-\y/2 , \z/2);
-\tikzstyle{depthlabel}=[pos=0.2,text width=14*\z,text centered,sloped]       
+
+\tikzset{depthlabel/.style={pos=\zlabeloffset, text width=14*\z, text centered, sloped}}
+
+%\tikzstyle{depthlabel}=[pos=0.3,text width=14*\z,text centered,sloped]       
+%\tikzstyle{depthlabel0}=[pos=0,text width=14*\z,text centered,sloped]       
+%\tikzstyle{depthlabel1}=[pos=0.1,text width=14*\z,text centered,sloped]       
+%\tikzstyle{depthlabel2}=[pos=0.2,text width=14*\z,text centered,sloped]       
+%\tikzstyle{depthlabel3}=[pos=0.3,text width=14*\z,text centered,sloped]       
+%\tikzstyle{depthlabel4}=[pos=0.4,text width=14*\z,text centered,sloped]       
+%\tikzstyle{depthlabel5}=[pos=0.5,text width=14*\z,text centered,sloped]       
 
 \path (c) edge ["\small\zlabel"',depthlabel](f); %depth label
 \path (b1) edge ["\ylabel",midway] (a1);  %height label
 
 
-\tikzstyle{captionlabel}=[text width=15*\LastEastx/\scale,text centered]       
-\path (\LastEastx/2,-\y/2,+\z/2) + (0,-25pt) coordinate (cap) 
+% \tikzstyle{captionlabel}=[text width=15*\LastEastx/\scale,text centered,xshift=\captionshift pt]       
+% \path (\LastEastx/2,-\y/2,+\z/2) + (0,-25pt) coordinate (cap) 
+% edge ["\textcolor{black}{ \bf \caption}"',captionlabel](cap) ; %Block caption/pic object label
+
+% Place caption: shift the coordinate by captionshift (NEW)
+\path (\LastEastx/2,-\y/2,+\z/2) + (\captionshift pt,-25pt) coordinate (cap)
 edge ["\textcolor{black}{ \bf \caption}"',captionlabel](cap) ; %Block caption/pic object label
 
 %Define nodes to be used outside on the pic object
@@ -92,7 +105,9 @@ scale/.store        in=\scale,
 xlabel/.store       in=\boxlabels,
 ylabel/.store       in=\ylabel,
 zlabel/.store       in=\zlabel,
+zlabeloffset/.store in=\zlabeloffset,
 caption/.store      in=\caption,
+captionshift/.store in=\captionshift,
 name/.store         in=\name,
 fill/.store         in=\fill,
 opacity/.store      in=\opacity,
@@ -105,6 +120,8 @@ scale=.2,
 xlabel={{"","","","","","","","","",""}},
 ylabel=,
 zlabel=,
+zlabeloffset=0.3,
 caption=,
+captionshift=0,
 name=,
 }

thesis/third_party/PlotNeuralNet/pycore/tikzeng.py

@@ -69,11 +69,13 @@ def to_Conv(
     s_filer=256,
     n_filer=64,
     offset="(0,0,0)",
+    zlabeloffset=0.3,
     to="(0,0,0)",
     width=1,
     height=40,
     depth=40,
     caption=" ",
+    captionshift=0,
 ):
     return (
         r"""
@@ -89,9 +91,15 @@ def to_Conv(
         caption="""
         + caption
         + r""",
+        captionshift="""
+        + str(captionshift)
+        + """,
         xlabel={{"""
         + str(n_filer)
         + """, }},
+        zlabeloffset="""
+        + str(zlabeloffset)
+        + """,
         zlabel="""
         + str(s_filer)
         + """,
@@ -168,13 +176,17 @@ def to_ConvConvRelu(
 # Pool
 def to_Pool(
     name,
+    n_filer="",
+    s_filer="",
     offset="(0,0,0)",
+    zlabeloffset=0.3,
     to="(0,0,0)",
     width=1,
     height=32,
     depth=32,
     opacity=0.5,
     caption=" ",
+    captionshift=0,
 ):
     return (
         r"""
@@ -187,9 +199,21 @@ def to_Pool(
         name="""
         + name
         + """,
+        xlabel={{"""
+        + str(n_filer)
+        + """, }},
+        zlabeloffset="""
+        + str(zlabeloffset)
+        + """,
+        zlabel="""
+        + str(s_filer)
+        + """,
         caption="""
         + caption
         + r""",
+        captionshift="""
+        + str(captionshift)
+        + """,
         fill=\PoolColor,
         opacity="""
         + str(opacity)
@@ -212,6 +236,7 @@ def to_Pool(
 # unpool4,
 def to_UnPool(
     name,
+    n_filer="",
     offset="(0,0,0)",
     to="(0,0,0)",
     width=1,
@@ -219,6 +244,7 @@ def to_UnPool(
     depth=32,
     opacity=0.5,
     caption=" ",
+    captionshift=0,
 ):
     return (
         r"""
@@ -234,10 +260,16 @@ def to_UnPool(
         caption="""
         + caption
         + r""",
+        captionshift="""
+        + str(captionshift)
+        + r""",
         fill=\UnpoolColor,
         opacity="""
         + str(opacity)
         + """,
+        xlabel={{"""
+        + str(n_filer)
+        + """, }},
         height="""
         + str(height)
         + """,
@@ -315,6 +347,7 @@ def to_ConvSoftMax(
     height=40,
     depth=40,
     caption=" ",
+    captionshift=0,
 ):
     return (
         r"""
@@ -330,6 +363,9 @@ def to_ConvSoftMax(
         caption="""
         + caption
         + """,
+        captionshift="""
+        + str(captionshift)
+        + """,
         zlabel="""
         + str(s_filer)
         + """,
@@ -360,6 +396,8 @@ def to_SoftMax(
     depth=25,
     opacity=0.8,
     caption=" ",
+    captionshift=0,
+    z_label_offset=0,
 ):
     return (
         r"""
@@ -375,6 +413,9 @@ def to_SoftMax(
         caption="""
         + caption
         + """,
+        captionshift="""
+        + str(captionshift)
+        + """,
         xlabel={{" ","dummy"}},
         zlabel="""
         + str(s_filer)
@@ -428,11 +469,13 @@ def to_fc(
     name,
     n_filer=120,
     offset="(0,0,0)",
+    zlabeloffset=0.3,
     to="(0,0,0)",
     width=2,
     height=2,
     depth=10,
     caption=" ",
+    captionshift=0,
     # titlepos=0,
 ):
     return (
@@ -449,7 +492,13 @@ def to_fc(
         caption="""
         + caption
         + """,
+        captionshift="""
+        + str(captionshift)
+        + """,
         xlabel={{" ","dummy"}},
+        zlabeloffset="""
+        + str(zlabeloffset)
+        + """,
         zlabel="""
         + str(n_filer)
         + """,

tools/.gitignore

@@ -7,4 +7,7 @@ tmp
 .envrc
 .vscode
 test
+*.jpg
+*.jpeg
+*.png
 

tools/ae_elbow_eval.py

@@ -0,0 +1,174 @@
+# loads results from autoencoder training form a pickle file and evaluates results and visualizes them to find traiing elbow
+
+import pickle
+import unittest
+from pathlib import Path
+from typing import Dict, List
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+results_folder = Path(
+    "/home/fedex/mt/projects/thesis-kowalczyk-jan/Deep-SAD-PyTorch/test/DeepSAD/subter_ae_elbow/"
+)
+
+# Find all result files matching the pattern
+result_files = sorted(
+    results_folder.glob("ae_elbow_results_subter_LeNet_dim_*_kfold.pkl")
+)
+
+# Initialize data structures for both classes
+dimensions = []
+normal_means = []
+normal_stds = []
+anomaly_means = []
+anomaly_stds = []
+
+BATCH_SIZE = 256  # Add this constant at the top of the file
+
+
+def calculate_batch_mean_loss(scores, batch_size=BATCH_SIZE):
+    """Calculate mean loss over batches similar to the original testing code."""
+    n_samples = len(scores)
+    n_batches = (n_samples + batch_size - 1) // batch_size  # ceiling division
+
+    # Split scores into batches
+    batch_losses = []
+    for i in range(0, n_samples, batch_size):
+        batch_scores = scores[i : i + batch_size]
+        batch_losses.append(np.mean(batch_scores))
+
+    return np.sum(batch_losses) / n_batches
+
+
+def test_loss_calculation(results: Dict, batch_size: int = BATCH_SIZE) -> None:
+    """Test if our loss calculation matches the original implementation."""
+    test = unittest.TestCase()
+    folds = results["ae_results"]
+    dim = results["dimension"]
+
+    for fold_key in folds:
+        fold_data = folds[fold_key]["test"]
+        scores = np.array(fold_data["scores"])
+        original_loss = fold_data["loss"]
+        calculated_loss = calculate_batch_mean_loss(scores)
+
+        try:
+            test.assertAlmostEqual(
+                original_loss,
+                calculated_loss,
+                places=5,
+                msg=f"Loss mismatch for dim={dim}, {fold_key}",
+            )
+        except AssertionError as e:
+            print(f"Warning: {str(e)}")
+            print(f"Original: {original_loss:.6f}, Calculated: {calculated_loss:.6f}")
+            raise
+
+
+# Load and verify data
+print("Verifying loss calculation implementation...")
+for result_file in result_files:
+    with open(result_file, "rb") as f:
+        results = pickle.load(f)
+        test_loss_calculation(results)
+print("Loss calculation verified successfully!")
+
+# Continue with actual data processing
+for result_file in result_files:
+    with open(result_file, "rb") as f:
+        results = pickle.load(f)
+    dim = int(results["dimension"])
+    folds = results["ae_results"]
+
+    normal_fold_losses = []
+    anomaly_fold_losses = []
+
+    for fold_key in folds:
+        fold_data = folds[fold_key]["test"]
+        scores = np.array(fold_data["scores"])
+        labels = np.array(fold_data["labels_exp_based"])
+
+        # Calculate mean loss for normal and anomaly samples
+        normal_scores = scores[labels == 1]
+        anomaly_scores = scores[labels == -1]
+
+        # Calculate losses using batch means
+        normal_fold_losses.append(calculate_batch_mean_loss(normal_scores))
+        anomaly_fold_losses.append(calculate_batch_mean_loss(anomaly_scores))
+
+    dimensions.append(dim)
+    normal_means.append(np.mean(normal_fold_losses))
+    normal_stds.append(np.std(normal_fold_losses))
+    anomaly_means.append(np.mean(anomaly_fold_losses))
+    anomaly_stds.append(np.std(anomaly_fold_losses))
+
+# Sort by dimension
+dims, n_means, n_stds, a_means, a_stds = zip(
+    *sorted(zip(dimensions, normal_means, normal_stds, anomaly_means, anomaly_stds))
+)
+
+# Calculate overall means and stds
+means = [(n + a) / 2 for n, a in zip(n_means, a_means)]
+stds = [(ns + as_) / 2 for ns, as_ in zip(n_stds, a_stds)]
+
+
+def plot_loss_curve(dims, means, stds, title, color, output_path):
+    """Create and save a single loss curve plot.
+
+    Args:
+        dims: List of latent dimensions
+        means: List of mean losses
+        stds: List of standard deviations
+        title: Plot title
+        color: Color for plot and fill
+        output_path: Where to save the plot
+    """
+    plt.figure(figsize=(8, 5))
+    plt.plot(dims, means, marker="o", color=color, label="Mean Test Loss")
+    plt.fill_between(
+        dims,
+        np.array(means) - np.array(stds),
+        np.array(means) + np.array(stds),
+        color=color,
+        alpha=0.2,
+        label="Std Dev",
+    )
+    plt.xlabel("Latent Dimension")
+    plt.ylabel("Test Loss")
+    plt.title(title)
+    plt.legend()
+    plt.grid(True, alpha=0.3)
+    plt.xticks(dims)  # Set x-ticks exactly at all data points
+    plt.tight_layout()
+    plt.savefig(output_path, dpi=150, bbox_inches="tight")
+    plt.close()
+
+
+# Create the three plots
+plot_loss_curve(
+    dims,
+    means,
+    stds,
+    "Overall Test Loss vs. Latent Dimension",
+    "blue",
+    results_folder / "ae_elbow_test_loss_overall.png",
+)
+
+plot_loss_curve(
+    dims,
+    n_means,
+    n_stds,
+    "Normal Class Test Loss vs. Latent Dimension",
+    "green",
+    results_folder / "ae_elbow_test_loss_normal.png",
+)
+
+plot_loss_curve(
+    dims,
+    a_means,
+    a_stds,
+    "Anomaly Class Test Loss vs. Latent Dimension",
+    "red",
+    results_folder / "ae_elbow_test_loss_anomaly.png",
+)

tools/calculate_conv_dimensions.py

@@ -0,0 +1,23 @@
+def calculate_conv_dimensions(W_in, H_in, K, S, P):
+    """
+    Calculate the output dimensions of a convolutional layer.
+
+    Parameters:
+    W_in (int): Width of the input image
+    H_in (int): Height of the input image
+    K (int): Size of the filter (assumed to be square)
+    S (int): Stride
+    P (int): Padding
+
+    Returns:
+    (int, int): Width and height of the output activation map
+    """
+    W_out = (W_in - K + (2 * P)) / S + 1
+    H_out = (H_in - K + (2 * P)) / S + 1
+
+    return W_out, H_out
+
+
+print(f"w, h = {calculate_conv_dimensions(W_in=2048, H_in=32, K=11, S=4, P=2)=}")
+# w, h = calculate_conv_dimensions(W_in=2048, H_in=32, K=11, S=4, P=2)
+# print(f"{calculate_conv_dimensions(W_in=w, H_in=h, K=11, S=4, P=2)=}")

tools/calculate_rf_sizes.py

@@ -0,0 +1,124 @@
+def calculate_receptive_field_size(layers):
+    """
+    Parameters
+    ----------
+    layers : list of dict
+        Each dict must contain
+        ─ 'kernel_size' : (k_h, k_w)
+        ─ 'stride'      : (s_h, s_w)
+        ─ 'dilation'    : (d_h, d_w)  # optional; defaults to (1, 1)
+
+    Returns
+    -------
+    (rf_h, rf_w) : tuple of ints
+        Receptive-field size in pixels for height and width.
+    """
+    rf_h = rf_w = 1  # start with a 1×1 pixel
+    jump_h = jump_w = 1  # effective stride so far
+
+    for layer in layers:
+        k_h, k_w = layer["kernel_size"]
+        s_h, s_w = layer["stride"]
+        d_h, d_w = layer.get("dilation", (1, 1))  # default = 1
+
+        # Dumoulin & Visin recurrence, now with dilation
+        rf_h += (k_h - 1) * d_h * jump_h
+        rf_w += (k_w - 1) * d_w * jump_w
+
+        jump_h *= s_h
+        jump_w *= s_w
+
+    return rf_h, rf_w
+
+
+def calculate_angular_receptive_field(
+    rf_height: int,
+    rf_width: int,
+    vertical_res: int,
+    horizontal_res: int,
+    vertical_fov: float,
+    horizontal_fov: float,
+) -> tuple[float, float]:
+    """Calculate the angular size of a receptive field in degrees.
+
+    Args:
+        rf_height: Receptive field height in pixels
+        rf_width: Receptive field width in pixels
+        vertical_res: Vertical resolution of input in pixels
+        horizontal_res: Horizontal resolution of input in pixels
+        vertical_fov: Vertical field of view in degrees
+        horizontal_fov: Horizontal field of view in degrees
+
+    Returns:
+        tuple[float, float]: Angular size (height, width) in degrees
+    """
+    # Calculate degrees per pixel for each axis
+    vertical_deg_per_pixel = vertical_fov / vertical_res
+    horizontal_deg_per_pixel = horizontal_fov / horizontal_res
+
+    # Calculate angular size of receptive field
+    rf_vertical_deg = rf_height * vertical_deg_per_pixel
+    rf_horizontal_deg = rf_width * horizontal_deg_per_pixel
+
+    return rf_vertical_deg, rf_horizontal_deg
+
+
+def calculate_pixels_per_degree(resolution: int, fov: float) -> float:
+    """Calculate pixels per degree for a given resolution and field of view.
+
+    Args:
+        resolution: Number of pixels
+        fov: Field of view in degrees
+
+    Returns:
+        float: Pixels per degree
+    """
+    return resolution / fov
+
+
+horizontal_resolution = 2048
+horizontal_fov = 360.0
+vertical_resolution = 32
+vertical_fov = 31.76
+
+lenet_network_config = [
+    {"name": "Conv1", "kernel_size": (5, 5), "stride": (1, 1), "dilation": (1, 1)},
+    {"name": "MaxPool1", "kernel_size": (2, 2), "stride": (2, 2), "dilation": (1, 1)},
+    {"name": "Conv2", "kernel_size": (5, 5), "stride": (1, 1), "dilation": (1, 1)},
+    {"name": "MaxPool2", "kernel_size": (2, 2), "stride": (2, 2), "dilation": (1, 1)},
+]
+
+# Calculate and print results for LeNet
+rf_h, rf_w = calculate_receptive_field_size(lenet_network_config)
+rf_vert_deg, rf_horiz_deg = calculate_angular_receptive_field(
+    rf_h, rf_w, vertical_resolution, horizontal_resolution, vertical_fov, horizontal_fov
+)
+print(f"SubTer LeNet RF size: {rf_h} × {rf_w} pixels")
+print(f"SubTer LeNet RF angular size: {rf_vert_deg:.2f}° × {rf_horiz_deg:.2f}°")
+
+
+asym_network_config = [
+    {"name": "Conv1", "kernel_size": (3, 17), "stride": (1, 1), "dilation": (1, 1)},
+    {"name": "MaxPool1", "kernel_size": (2, 2), "stride": (2, 2), "dilation": (1, 1)},
+    {"name": "Conv2", "kernel_size": (3, 17), "stride": (1, 1), "dilation": (1, 1)},
+    {"name": "MaxPool2", "kernel_size": (2, 2), "stride": (2, 2), "dilation": (1, 1)},
+]
+
+# Calculate and print results for Asymmetric network
+rf_h, rf_w = calculate_receptive_field_size(asym_network_config)
+rf_vert_deg, rf_horiz_deg = calculate_angular_receptive_field(
+    rf_h, rf_w, vertical_resolution, horizontal_resolution, vertical_fov, horizontal_fov
+)
+print(f"SubTer LeNet (Asymmetric kernels) RF size: {rf_h} × {rf_w} pixels")
+print(
+    f"SubTer LeNet (Asymmetric kernels) RF angular size: {rf_vert_deg:.2f}° × {rf_horiz_deg:.2f}°"
+)
+
+# Calculate pixels per degree
+horizontal_ppd = calculate_pixels_per_degree(horizontal_resolution, horizontal_fov)
+vertical_ppd = calculate_pixels_per_degree(vertical_resolution, vertical_fov)
+
+print("\nPixels per Degree:")
+print(f"Horizontal: {horizontal_ppd:.2f} px/°")
+print(f"Vertical:   {vertical_ppd:.2f} px/°")
+print()

tools/compare_projections.py

@@ -0,0 +1,205 @@
+from pathlib import Path
+from typing import Any, Dict, Optional, Tuple
+
+import numpy as np
+
+# Configuration
+old_projections_path = Path("/home/fedex/mt/data/subter")
+new_projections_path = Path("/home/fedex/mt/data/subter/new_projection")
+
+
+def get_file_info(file_path: Path) -> Optional[dict]:
+    """Get detailed information about a .npy file."""
+    if not file_path.exists():
+        return None
+    try:
+        with file_path.open("rb") as f:
+            return {
+                "size": file_path.stat().st_size,
+                "format": np.lib.format.read_magic(f),
+                "header": np.lib.format.read_array_header_1_0(f),
+            }
+    except Exception as e:
+        print(f"Error reading file {file_path}: {e}")
+        return None
+
+
+def get_array_info(arr: np.ndarray) -> dict:
+    """Get detailed information about a numpy array."""
+    return {
+        "dtype": str(arr.dtype),
+        "itemsize": arr.itemsize,
+        "nbytes": arr.nbytes,
+        "byteorder": arr.dtype.byteorder,
+        "shape": arr.shape,
+        "nan_count": np.sum(np.isnan(arr)),
+        "inf_count": np.sum(np.isinf(arr)),
+    }
+
+
+def compare_arrays(old_arr: np.ndarray, new_arr: np.ndarray) -> dict:
+    """Compare two numpy arrays and return detailed differences."""
+    # Get basic array information
+    old_info = get_array_info(old_arr)
+    new_info = get_array_info(new_arr)
+
+    differences = {
+        "old_info": old_info,
+        "new_info": new_info,
+        "shape_mismatch": old_arr.shape != new_arr.shape,
+    }
+
+    if differences["shape_mismatch"]:
+        return differences
+
+    # Compare values
+    both_nan = np.isnan(old_arr) & np.isnan(new_arr)
+    unequal = (old_arr != new_arr) & ~both_nan
+    mismatch_indices = np.where(unequal)
+
+    differences.update(
+        {
+            "num_mismatches": mismatch_indices[0].size,
+            "mismatch_indices": mismatch_indices
+            if mismatch_indices[0].size > 0
+            else None,
+            "mean_difference": np.mean(np.abs(old_arr - new_arr)),
+            "std_difference": np.std(old_arr - new_arr),
+            "new_zeros": np.sum(new_arr == 0),
+        }
+    )
+
+    # If there are mismatches, store some example values
+    if differences["num_mismatches"] > 0:
+        old_values = old_arr[mismatch_indices][:10]
+        new_values = new_arr[mismatch_indices][:10]
+        differences.update(
+            {
+                "example_old_values": old_values,
+                "example_new_values": new_values,
+                "all_new_mismatches_zero": np.all(new_arr[mismatch_indices] == 0),
+            }
+        )
+
+    return differences
+
+
+def print_detailed_comparison(name: str, diff: dict):
+    """Print detailed comparison results for a single file."""
+    print(f"\nDetailed comparison for: {name}")
+    print("=" * 80)
+
+    # Storage information
+    old_info = diff["old_info"]
+    new_info = diff["new_info"]
+    print(f"Storage Information:")
+    print(f"  Dtype: {old_info['dtype']} → {new_info['dtype']}")
+    print(f"  Item size: {old_info['itemsize']} → {new_info['itemsize']} bytes")
+    print(f"  Total bytes: {old_info['nbytes']} → {new_info['nbytes']}")
+    print(f"  Byte ratio: {new_info['nbytes'] / old_info['nbytes']:.2f}")
+
+    # Shape information
+    if diff["shape_mismatch"]:
+        print(f"Shape mismatch: {old_info['shape']} → {new_info['shape']}")
+        return
+
+    # Value differences
+    if diff["num_mismatches"] > 0:
+        print(f"\nValue Differences:")
+        print(f"  Number of mismatches: {diff['num_mismatches']}")
+        print(f"  Mean difference: {diff['mean_difference']:.2e}")
+        print(f"  Std difference: {diff['std_difference']:.2e}")
+        print(f"  Example mismatches (old → new):")
+        for old_val, new_val in zip(
+            diff["example_old_values"], diff["example_new_values"]
+        ):
+            print(f"    {old_val:.6e} → {new_val:.6e}")
+        if diff["all_new_mismatches_zero"]:
+            print("  Note: All mismatched values in new array are zero")
+    else:
+        print("\nNo value mismatches found.")
+
+
+def summarize_differences(all_differences: Dict[str, dict]) -> str:
+    """Create a summary of all differences."""
+    summary = []
+    summary.append("\nSUMMARY OF ALL DIFFERENCES")
+    summary.append("=" * 80)
+
+    total_files = len(all_differences)
+    files_with_mismatches = sum(
+        1 for d in all_differences.values() if d["num_mismatches"] > 0
+    )
+    files_with_shape_mismatch = sum(
+        1 for d in all_differences.values() if d["shape_mismatch"]
+    )
+
+    summary.append(f"Total files compared: {total_files}")
+    summary.append(f"Files with shape mismatches: {files_with_shape_mismatch}")
+    summary.append(f"Files with value mismatches: {files_with_mismatches}")
+
+    if files_with_mismatches > 0:
+        summary.append("\nFiles with differences:")
+        for name, diff in all_differences.items():
+            if diff["num_mismatches"] > 0:
+                summary.append(f"  {name}:")
+                summary.append(f"    Mismatches: {diff['num_mismatches']}")
+                summary.append(f"    Mean difference: {diff['mean_difference']:.2e}")
+                if diff.get("all_new_mismatches_zero", False):
+                    summary.append("    All mismatches are zeros in new file")
+
+    return "\n".join(summary)
+
+
+def main():
+    # Get list of all .npy files
+    old_files = list(old_projections_path.glob("*.npy"))
+    new_files = list(new_projections_path.glob("*.npy"))
+
+    # Check for missing files
+    old_names = {f.stem for f in old_files}
+    new_names = {f.stem for f in new_files}
+    if missing := (old_names - new_names):
+        print(f"Files missing in new directory: {missing}")
+    if missing := (new_names - old_names):
+        print(f"Files missing in old directory: {missing}")
+
+    # Compare common files
+    all_differences = {}
+    for old_file in old_files:
+        if old_file.stem not in new_names:
+            continue
+
+        print(f"\nComparing {old_file.stem}...")
+        new_file = new_projections_path / f"{old_file.stem}.npy"
+
+        # Check file info before loading
+        old_info = get_file_info(old_file)
+        new_info = get_file_info(new_file)
+
+        if not old_info or not new_info:
+            print(f"Skipping {old_file.stem} due to file reading errors")
+            continue
+
+        try:
+            # Load arrays
+            old_arr = np.load(old_file)
+            new_arr = np.load(new_file)
+
+            # Compare and print detailed results
+            differences = compare_arrays(old_arr, new_arr)
+            print_detailed_comparison(old_file.stem, differences)
+            all_differences[old_file.stem] = differences
+        except Exception as e:
+            print(f"Error processing {old_file.stem}: {e}")
+            continue
+
+    # Print summary
+    if all_differences:
+        print(summarize_differences(all_differences))
+    else:
+        print("\nNo valid comparisons were made.")
+
+
+if __name__ == "__main__":
+    main()

tools/demo_loaded_data.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+
+import polars as pl
+from load_results import (
+    load_pretraining_results_dataframe,
+    load_results_dataframe,
+)
+
+
+# ------------------------------------------------------------
+# Example “analysis-ready” queries (Polars idioms)
+# ------------------------------------------------------------
+def demo_queries(df: pl.DataFrame):
+    # q1: lazy is fine, then collect
+    q1 = (
+        df.lazy()
+        .filter(
+            (pl.col("network") == "LeNet")
+            & (pl.col("latent_dim") == 1024)
+            & (pl.col("semi_normals") == 0)
+            & (pl.col("semi_anomalous") == 0)
+            & (pl.col("eval") == "exp_based")
+        )
+        .group_by(["model"])
+        .agg(pl.col("auc").mean().alias("mean_auc"))
+        .sort(["mean_auc"], descending=True)
+        .collect()
+    )
+
+    # q2: do the filtering eagerly, then pivot (LazyFrame has no .pivot)
+    base = df.filter(
+        (pl.col("model") == "deepsad")
+        & (pl.col("eval") == "exp_based")
+        & (pl.col("network") == "LeNet")
+        & (pl.col("semi_normals") == 0)
+        & (pl.col("semi_anomalous") == 0)
+    ).select("fold", "latent_dim", "auc")
+    q2 = base.pivot(
+        values="auc",
+        index="fold",
+        columns="latent_dim",
+        aggregate_function="first",  # or "mean" if duplicates exist
+    ).sort("fold")
+
+    # roc_subset: eager filter/select, then explode struct fields
+    roc_subset = (
+        df.filter(
+            (pl.col("model") == "ocsvm")
+            & (pl.col("eval") == "manual_based")
+            & (pl.col("network") == "efficient")
+            & (pl.col("latent_dim") == 1024)
+            & (pl.col("semi_normals") == 0)
+            & (pl.col("semi_anomalous") == 0)
+        )
+        .select("fold", "roc_curve")
+        .with_columns(
+            pl.col("roc_curve").struct.field("fpr").alias("fpr"),
+            pl.col("roc_curve").struct.field("tpr").alias("tpr"),
+            pl.col("roc_curve").struct.field("thr").alias("thr"),
+        )
+    )
+
+    return q1, q2, roc_subset
+
+
+def main():
+    root = Path("/home/fedex/mt/results/done")
+    df = load_results_dataframe(root, allow_cache=True)
+    demo_queries(df)
+
+
+if __name__ == "__main__":
+    main()

tools/devenv.lock

@@ -0,0 +1,103 @@
+{
+  "nodes": {
+    "devenv": {
+      "locked": {
+        "dir": "src/modules",
+        "lastModified": 1754730435,
+        "owner": "cachix",
+        "repo": "devenv",
+        "rev": "d1388a093a7225c2abe8c244109c5a4490de4077",
+        "type": "github"
+      },
+      "original": {
+        "dir": "src/modules",
+        "owner": "cachix",
+        "repo": "devenv",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1747046372,
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1754416808,
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "9c52372878df6911f9afc1e2a1391f55e4dfc864",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "git-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1754299112,
+        "owner": "cachix",
+        "repo": "devenv-nixpkgs",
+        "rev": "16c21c9f5c6fb978466e91182a248dd8ca1112ac",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "ref": "rolling",
+        "repo": "devenv-nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "devenv": "devenv",
+        "git-hooks": "git-hooks",
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks": [
+          "git-hooks"
+        ]
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

tools/devenv.nix

@@ -0,0 +1,27 @@
+{ pkgs, ... }:
+let
+  native_dependencies = with pkgs.python311Packages; [
+    torch-bin
+    torchvision-bin
+    aggdraw # for visualtorch
+    numpy
+    scipy
+    matplotlib
+  ];
+  tools = with pkgs; [
+    ruff
+  ];
+in
+{
+  packages = native_dependencies ++ tools;
+  languages.python = {
+    enable = true;
+    package = pkgs.python311;
+    uv = {
+      enable = true;
+      sync.enable = true;
+    };
+    venv.enable = true;
+  };
+
+}

tools/devenv.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://devenv.sh/devenv.schema.json
+inputs:
+  nixpkgs:
+    url: github:cachix/devenv-nixpkgs/rolling
+
+allowUnfree: true
+cudaSupport: true
+# If you're using non-OSS software, you can set allowUnfree to true.
+# allowUnfree: true
+
+# If you're willing to use a package that's vulnerable
+# permittedInsecurePackages:
+#  - "openssl-1.1.1w"
+
+# If you have more than one devenv you can merge them
+#imports:
+# - ./backend

tools/elbow_structure.txt

@@ -0,0 +1,600 @@
+
+=== Results for ae_elbow_results_subter_LeNet_dim_32_kfold.pkl ===
+dimension:
+  32
+ae_results:
+  fold_0:
+    train:
+      time:
+        47.35341715812683
+      indices:
+        [ 6547  6096 17322 ... 14590  6764  6219]
+      labels_exp_based:
+        [1 1 1 ... 1 1 1]
+      labels_manual_based:
+        [1 1 1 ... 1 1 1]
+      semi_targets:
+        [1 1 1 ... 1 1 1]
+      file_ids:
+        [8 5 0 ... 1 9 1]
+      frame_ids:
+        [1260  569 2387 ... 3064  205 1096]
+      scores:
+        [0.06785126 0.13130851 0.10918648 ... 0.04708466 0.03133186 0.05559417]
+      loss:
+        0.07233340218663216
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+    test:
+      time:
+        3.2727890014648438
+      indices:
+        [   0    1    2 ... 5883 5884 5885]
+      labels_exp_based:
+        [ 1  1  1 ... -1  1 -1]
+      labels_manual_based:
+        [ 1  1  1 ... -1  1 -1]
+      semi_targets:
+        [ 1  1  1 ... -1  1 -1]
+      file_ids:
+        [ 0  1  3 ... 11  0 12]
+      frame_ids:
+        [3373 2260   81 ...  405 1342  335]
+      scores:
+        [0.28987885 0.06485476 0.03065375 ... 0.26596928 0.03807792 0.46061364]
+      loss:
+        0.06829473586833995
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+  fold_1:
+    train:
+      time:
+        46.584895849227905
+      indices:
+        [14909  2898 15177 ...  3185  2297  2518]
+      labels_exp_based:
+        [ 1  1  1 ...  1 -1  1]
+      labels_manual_based:
+        [ 1  1  1 ...  1 -1  1]
+      semi_targets:
+        [ 1  1  1 ...  1 -1  1]
+      file_ids:
+        [ 5  6  1 ...  7 13  4]
+      frame_ids:
+        [ 936  606 3193 ...  989   86  844]
+      scores:
+        [0.10743871 0.17937626 0.3385203  ... 0.02518196 0.3909377  0.02823789]
+      loss:
+        0.06587159360448519
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+    test:
+      time:
+        3.254101037979126
+      indices:
+        [ 5886  5887  5888 ... 11769 11770 11771]
+      labels_exp_based:
+        [ 1  1 -1 ...  1  1  1]
+      labels_manual_based:
+        [1 1 0 ... 1 1 1]
+      semi_targets:
+        [1 1 0 ... 1 1 1]
+      file_ids:
+        [ 0  1 12 ...  0  1  7]
+      frame_ids:
+        [2943    4  179 ...  348  663  423]
+      scores:
+        [0.02836892 0.03712069 0.13963991 ... 0.07210366 0.02482101 0.07186633]
+      loss:
+        0.059809695119443146
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+  fold_2:
+    train:
+      time:
+        47.0039427280426
+      indices:
+        [ 4709 10566  1669 ...  4887  6760  1348]
+      labels_exp_based:
+        [1 1 1 ... 1 1 1]
+      labels_manual_based:
+        [1 1 1 ... 1 1 1]
+      semi_targets:
+        [1 1 1 ... 1 1 1]
+      file_ids:
+        [0 6 1 ... 8 5 0]
+      frame_ids:
+        [1701  990 2584 ... 1036  610  449]
+      scores:
+        [0.07715754 0.07710524 0.11718763 ... 0.03398419 0.07341428 0.03260035]
+      loss:
+        0.06793700895375676
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+    test:
+      time:
+        3.3094074726104736
+      indices:
+        [11772 11773 11774 ... 17655 17656 17657]
+      labels_exp_based:
+        [1 1 1 ... 1 1 1]
+      labels_manual_based:
+        [1 1 1 ... 1 1 1]
+      semi_targets:
+        [1 1 1 ... 1 1 1]
+      file_ids:
+        [7 8 0 ... 4 5 0]
+      frame_ids:
+        [  98 1000  873 ...  674  541 2732]
+      scores:
+        [0.033662   0.03786198 0.04839528 ... 0.07525856 0.07365932 0.0603102 ]
+      loss:
+        0.06470830592772235
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+k_fold:
+  True
+k_fold_num:
+  3
+
+=== Results for ae_elbow_results_subter_LeNet_dim_64_kfold.pkl ===
+dimension:
+  64
+ae_results:
+  fold_0:
+    train:
+      time:
+        47.33047151565552
+      indices:
+        [11876 12634 14187 ... 13309  8275 16184]
+      labels_exp_based:
+        [ 1 -1  1 ...  1  1  1]
+      labels_manual_based:
+        [1 0 1 ... 1 1 1]
+      semi_targets:
+        [1 0 1 ... 1 1 1]
+      file_ids:
+        [ 4 12  8 ...  0  7  4]
+      frame_ids:
+        [ 545   63 1159 ...  441  992  700]
+      scores:
+        [0.13897061 0.08483056 0.07720029 ... 0.02398726 0.02211376 0.05582099]
+      loss:
+        0.05624731986059083
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+    test:
+      time:
+        3.393460512161255
+      indices:
+        [   0    1    2 ... 5883 5884 5885]
+      labels_exp_based:
+        [ 1  1  1 ... -1  1 -1]
+      labels_manual_based:
+        [ 1  1  1 ... -1  1 -1]
+      semi_targets:
+        [ 1  1  1 ... -1  1 -1]
+      file_ids:
+        [ 0  1  3 ... 11  0 12]
+      frame_ids:
+        [3373 2260   81 ...  405 1342  335]
+      scores:
+        [0.19174853 0.04987323 0.0225831  ... 0.19634478 0.03153864 0.39142317]
+      loss:
+        0.05257830946989681
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+  fold_1:
+    train:
+      time:
+        47.514824867248535
+      indices:
+        [13393  1405 12413 ... 14812   397 14801]
+      labels_exp_based:
+        [ 1 -1  1 ... -1  1  1]
+      labels_manual_based:
+        [ 1  0  1 ... -1  1  1]
+      semi_targets:
+        [ 1  0  1 ... -1  1  1]
+      file_ids:
+        [ 3 10  8 ... 10  9  7]
+      frame_ids:
+        [1059  161  236 ...  368  480  145]
+      scores:
+        [0.08272791 0.17799513 0.07856707 ... 0.16969317 0.09914576 0.04465985]
+      loss:
+        0.08280573851532406
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+    test:
+      time:
+        3.303018808364868
+      indices:
+        [ 5886  5887  5888 ... 11769 11770 11771]
+      labels_exp_based:
+        [ 1  1 -1 ...  1  1  1]
+      labels_manual_based:
+        [1 1 0 ... 1 1 1]
+      semi_targets:
+        [1 1 0 ... 1 1 1]
+      file_ids:
+        [ 0  1 12 ...  0  1  7]
+      frame_ids:
+        [2943    4  179 ...  348  663  423]
+      scores:
+        [0.03737867 0.05766786 0.15279752 ... 0.07271019 0.04369381 0.10203677]
+      loss:
+        0.07809325761121252
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+  fold_2:
+    train:
+      time:
+        49.39735269546509
+      indices:
+        [6159 8197  617 ... 5118 8505 8730]
+      labels_exp_based:
+        [1 1 1 ... 1 1 1]
+      labels_manual_based:
+        [1 1 1 ... 1 1 1]
+      semi_targets:
+        [1 1 1 ... 1 1 1]
+      file_ids:
+        [1 1 0 ... 6 5 0]
+      frame_ids:
+        [ 113  169   48 ...  741  231 2666]
+      scores:
+        [0.0654774  0.06249695 0.06091163 ... 0.01456711 0.0128163  0.0202495 ]
+      loss:
+        0.03804728595746888
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+    test:
+      time:
+        3.464538812637329
+      indices:
+        [11772 11773 11774 ... 17655 17656 17657]
+      labels_exp_based:
+        [1 1 1 ... 1 1 1]
+      labels_manual_based:
+        [1 1 1 ... 1 1 1]
+      semi_targets:
+        [1 1 1 ... 1 1 1]
+      file_ids:
+        [7 8 0 ... 4 5 0]
+      frame_ids:
+        [  98 1000  873 ...  674  541 2732]
+      scores:
+        [0.01374984 0.01219883 0.01543648 ... 0.01943305 0.01877283 0.02498569]
+      loss:
+        0.03719014423372953
+      file_names:
+        0:
+          1_loop_closure_illuminated_2023-01-23.npy
+        1:
+          1_loop_closure_illuminated_two_LiDARs_2023-01-23.npy
+        2:
+          2_human_static_illuminated_2023-01-23-001.npy
+        3:
+          2_human_static_no_illumination_2023-01-23-001.npy
+        4:
+          2_human_walking_illuminated_2023-01-23-001.npy
+        5:
+          2_human_walking_no_illumination_2023-01-23-001.npy
+        6:
+          2_static_artifacts_illuminated_2023-01-23-001.npy
+        7:
+          2_static_artifacts_no_illumination_2023-01-23-001.npy
+        8:
+          2_static_no_artifacts_illuminated_2023-01-23-001.npy
+        9:
+          2_static_no_artifacts_no_illumination_2023-01-23-001.npy
+        10:
+          3_smoke_artifacts_human_static_2023-01-23-001.npy
+        11:
+          3_smoke_human_walking_2023-01-23.npy
+        12:
+          3_smoke_no_artifacts_2023-01-23.npy
+        13:
+          3_smoke_robot_stationary_static_excess_smoke_2023-01-23.npy
+k_fold:
+  True
+k_fold_num:
+  3

tools/evaluate_roc.py

@@ -1,13 +1,18 @@
+import json
 import pickle
 from pathlib import Path
 
 import matplotlib.pyplot as plt
 import numpy as np
+from rich.progress import track
 from scipy.stats import sem, t
-from sklearn.metrics import auc
+
+models = ["deepsad", "isoforest", "ocsvm"]
+evaluation_types = ["exp_based", "manual_based"]
+parent_results_path = Path("/home/fedex/mt/results/done")
+base_output_path = Path("/home/fedex/mt/results/tmp_plots")
 
 
-# Confidence interval function
 def confidence_interval(data, confidence=0.95):
     n = len(data)
     mean = np.mean(data)
@@ -16,67 +21,178 @@ def confidence_interval(data, confidence=0.95):
     return mean, h
 
 
-# Load ROC and AUC values from pickle files
-roc_data = []
-auc_scores = []
-isoforest_roc_data = []
-isoforest_auc_scores = []
+def load_results_data(folder):
+    experiment_data = {}
 
-results_path = Path(
-    "/home/fedex/mt/projects/thesis-kowalczyk-jan/Deep-SAD-PyTorch/log/DeepSAD/subter_kfold_0_0"
-)
+    json_config_path = folder / "config.json"
+    with json_config_path.open("r") as f:
+        config = json.load(f)
+    try:
+        net = config["net_name"]
+        num_known_normal, num_known_anomalous = (
+            config["num_known_normal"],
+            config["num_known_outlier"],
+        )
+        semi_known_nums = (num_known_normal, num_known_anomalous)
+        latent_dim = config["latent_space_dim"]
 
-for i in range(5):
-    with (results_path / f"results_{i}.pkl").open("rb") as f:
-        data = pickle.load(f)
-        roc_data.append(data["test_roc"])
-        auc_scores.append(data["test_auc"])
-    with (results_path / f"results.isoforest_{i}.pkl").open("rb") as f:
-        data = pickle.load(f)
-        isoforest_roc_data.append(data["test_roc"])
-        isoforest_auc_scores.append(data["test_auc"])
+        exp_title = f"{net} - {num_known_normal} normal, {num_known_anomalous} anomalous, latent dim {latent_dim}"
 
-# Calculate mean and confidence interval for AUC scores
-mean_auc, auc_ci = confidence_interval(auc_scores)
+        if not config["k_fold"]:
+            raise ValueError(f"{folder.name} was not trained as k-fold. Exiting...")
 
-# Combine ROC curves
-mean_fpr = np.linspace(0, 1, 100)
-tprs = []
+        k_fold_num = config["k_fold_num"]
+    except KeyError as e:
+        print(f"Missing key in config.json for experiment folder {folder.name}: {e}")
+        raise
 
-for fpr, tpr, _ in roc_data:
-    interp_tpr = np.interp(mean_fpr, fpr, tpr)
-    interp_tpr[0] = 0.0
-    tprs.append(interp_tpr)
+    experiment_data["exp_title"] = exp_title
+    experiment_data["k_fold_num"] = k_fold_num
+    experiment_data["semi_known_nums"] = semi_known_nums
+    experiment_data["folder"] = folder
+    experiment_data["net"] = net
+    experiment_data["latent_dim"] = latent_dim
 
-mean_tpr = np.mean(tprs, axis=0)
-mean_tpr[-1] = 1.0
-std_tpr = np.std(tprs, axis=0)
+    roc_data = {}
+    roc_auc_data = {}
+    prc_data = {}
 
-# Plot ROC curves with confidence margins
-plt.figure()
-plt.plot(
-    mean_fpr,
-    mean_tpr,
-    color="b",
-    label=f"Mean ROC (AUC = {mean_auc:.2f} ± {auc_ci:.2f})",
-)
-plt.fill_between(
-    mean_fpr,
-    mean_tpr - std_tpr,
-    mean_tpr + std_tpr,
-    color="b",
-    alpha=0.2,
-    label="± 1 std. dev.",
-)
+    for model in models:
+        # You can adjust the number of folds if needed
+        for fold_idx in range(k_fold_num):
+            results_file = folder / f"results_{model}_{fold_idx}.pkl"
+            if not results_file.exists():
+                print(
+                    f"Expected results file {results_file.name} does not exist. Skipping..."
+                )
+            with results_file.open("rb") as f:
+                data = pickle.load(f)
+                try:
+                    if model == "deepsad":
+                        test_results = data["test"]
+                        for evaluation_type in evaluation_types:
+                            eval_type_results = test_results[evaluation_type]
+                            roc_data.setdefault(model, {}).setdefault(
+                                evaluation_type, {}
+                            )[fold_idx] = eval_type_results["roc"]
+                            roc_auc_data.setdefault(model, {}).setdefault(
+                                evaluation_type, {}
+                            )[fold_idx] = eval_type_results["auc"]
+                            prc_data.setdefault(model, {}).setdefault(
+                                evaluation_type, {}
+                            )[fold_idx] = eval_type_results["prc"]
+                    elif model in ["isoforest", "ocsvm"]:
+                        for evaluation_type in evaluation_types:
+                            roc_data.setdefault(model, {}).setdefault(
+                                evaluation_type, {}
+                            )[fold_idx] = data[f"test_roc_{evaluation_type}"]
+                            roc_auc_data.setdefault(model, {}).setdefault(
+                                evaluation_type, {}
+                            )[fold_idx] = data[f"test_auc_{evaluation_type}"]
+                            prc_data.setdefault(model, {}).setdefault(
+                                evaluation_type, {}
+                            )[fold_idx] = data[f"test_prc_{evaluation_type}"]
 
-# Plot each fold's ROC curve (optional)
- for i, (fpr, tpr, _) in enumerate(roc_data):
-     plt.plot(fpr, tpr, lw=1, alpha=0.3, label=f"Fold {i + 1} ROC")
+                except KeyError as e:
+                    print(f"Missing key in results file {results_file.name}: {e}")
+                    raise
 
-# Labels and legend
-plt.plot([0, 1], [0, 1], "k--", label="Chance")
-plt.xlabel("False Positive Rate")
-plt.ylabel("True Positive Rate")
-plt.title("ROC Curve with 5-Fold Cross-Validation")
-plt.legend(loc="lower right")
-plt.savefig("roc_curve_0_0.png")
+    experiment_data["roc_data"] = roc_data
+    experiment_data["roc_auc_data"] = roc_auc_data
+    experiment_data["prc_data"] = prc_data
+    return experiment_data
+
+
+def plot_roc_curve(experiment_data, output_path):
+    try:
+        k_fold_num = experiment_data["k_fold_num"]
+        roc_data = experiment_data["roc_data"]
+        roc_auc_data = experiment_data["roc_auc_data"]
+        folder = experiment_data["folder"]
+        exp_title = experiment_data["exp_title"]
+    except KeyError as e:
+        print(f"Missing key in experiment data: {e}")
+        raise
+    for evaluation_type in evaluation_types:
+        plt.figure(figsize=(8, 6))
+        for model in models:
+            # Gather all folds' ROC data for this model and evaluation_type
+            fold_rocs = []
+            auc_scores = []
+            for fold_idx in range(k_fold_num):
+                try:
+                    fpr, tpr, thresholds = roc_data[model][evaluation_type][fold_idx]
+                    fold_rocs.append((fpr, tpr))
+                    auc_scores.append(roc_auc_data[model][evaluation_type][fold_idx])
+                except KeyError:
+                    continue
+
+            if not fold_rocs:
+                print(
+                    f"No ROC data for model {model}, evaluation {evaluation_type} in {folder.name}"
+                )
+                continue
+
+            # Interpolate TPRs to a common FPR grid
+            mean_fpr = np.linspace(0, 1, 100)
+            interp_tprs = []
+            for fpr, tpr in fold_rocs:
+                interp_tpr = np.interp(mean_fpr, fpr, tpr)
+                interp_tpr[0] = 0.0
+                interp_tprs.append(interp_tpr)
+            mean_tpr = np.mean(interp_tprs, axis=0)
+            std_tpr = np.std(interp_tprs, axis=0)
+            mean_tpr[-1] = 1.0
+
+            # Mean and CI for AUC
+            mean_auc, auc_ci = confidence_interval(auc_scores)
+
+            # Plot mean ROC and std band
+            plt.plot(
+                mean_fpr,
+                mean_tpr,
+                label=f"{model} (AUC={mean_auc:.2f}±{auc_ci:.2f})",
+            )
+            plt.fill_between(
+                mean_fpr,
+                mean_tpr - std_tpr,
+                mean_tpr + std_tpr,
+                alpha=0.15,
+            )
+
+        plt.plot([0, 1], [0, 1], "k--", label="Chance")
+        plt.xlabel("False Positive Rate")
+        plt.ylabel("True Positive Rate")
+        plt.title(f"ROC Curve ({exp_title} - {evaluation_type})")
+        plt.legend(loc="lower right")
+        plt.tight_layout()
+        plt.savefig(
+            (output_path / f"roc_curve_{folder.name}_{evaluation_type}.png").as_posix()
+        )
+        plt.close()
+
+
+def main():
+    base_output_path.mkdir(exist_ok=True, parents=True)
+    # Find all subfolders (skip files)
+    subfolders = [f for f in parent_results_path.iterdir() if f.is_dir()]
+    print(f"Found {len(subfolders)} subfolders in {parent_results_path}")
+    all_experiments_data = []
+    for folder in track(
+        subfolders, description="[cyan]Loading data...", total=len(subfolders)
+    ):
+        all_experiments_data.append(load_results_data(folder))
+
+    print("Data loading complete. Plotting ROC curves...")
+    roc_curves_output_path = base_output_path / "roc_curves"
+    roc_curves_output_path.mkdir(exist_ok=True, parents=True)
+    for experiment_data in track(
+        all_experiments_data,
+        description="[green]Plotting ROC curves...",
+        total=len(all_experiments_data),
+    ):
+        plot_roc_curve(experiment_data, roc_curves_output_path)
+
+
+if __name__ == "__main__":
+    main()

tools/flake.lock

@@ -1,192 +0,0 @@
-{
-  "nodes": {
-    "flake-utils": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "flake-utils_2": {
-      "inputs": {
-        "systems": "systems_2"
-      },
-      "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "nix-github-actions": {
-      "inputs": {
-        "nixpkgs": [
-          "poetry2nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1703863825,
-        "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=",
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "type": "github"
-      }
-    },
-    "nixpkgs": {
-      "locked": {
-        "lastModified": 1718676766,
-        "narHash": "sha256-0se0JqeNSZcNmqhsHMN9N4cVV/XkPhtSVJwhLs2RGUg=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "31e107dc564e53cf2843bedf6a8b85faa2f845e3",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-unstable-small",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-newest": {
-      "locked": {
-        "lastModified": 1745391562,
-        "narHash": "sha256-sPwcCYuiEopaafePqlG826tBhctuJsLx/mhKKM5Fmjo=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "8a2f738d9d1f1d986b5a4cd2fd2061a7127237d7",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "poetry2nix": {
-      "inputs": {
-        "flake-utils": "flake-utils_2",
-        "nix-github-actions": "nix-github-actions",
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "systems": "systems_3",
-        "treefmt-nix": "treefmt-nix"
-      },
-      "locked": {
-        "lastModified": 1718656656,
-        "narHash": "sha256-/8pXTFOfb7+KrFi+g8G/dFehDkc96/O5eL8L+FjzG1w=",
-        "owner": "nix-community",
-        "repo": "poetry2nix",
-        "rev": "2c6d07717af20e45fa5b2c823729126be91a3cdf",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "poetry2nix",
-        "type": "github"
-      }
-    },
-    "root": {
-      "inputs": {
-        "flake-utils": "flake-utils",
-        "nixpkgs": "nixpkgs",
-        "nixpkgs-newest": "nixpkgs-newest",
-        "poetry2nix": "poetry2nix"
-      }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
-    "systems_2": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
-    "systems_3": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "id": "systems",
-        "type": "indirect"
-      }
-    },
-    "treefmt-nix": {
-      "inputs": {
-        "nixpkgs": [
-          "poetry2nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1718522839,
-        "narHash": "sha256-ULzoKzEaBOiLRtjeY3YoGFJMwWSKRYOic6VNw2UyTls=",
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "rev": "68eb1dc333ce82d0ab0c0357363ea17c31ea1f81",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "type": "github"
-      }
-    }
-  },
-  "root": "root",
-  "version": 7
-}

tools/flake.nix

@@ -1,70 +0,0 @@
-{
-  description = "Application packaged using poetry2nix";
-
-  inputs = {
-    flake-utils.url = "github:numtide/flake-utils";
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
-    # Added newest nixpkgs for an updated poetry package.
-    nixpkgs-newest.url = "github:NixOS/nixpkgs/nixos-unstable";
-    poetry2nix = {
-      url = "github:nix-community/poetry2nix";
-      inputs.nixpkgs.follows = "nixpkgs";
-    };
-  };
-
-  outputs =
-    {
-      self,
-      nixpkgs,
-      flake-utils,
-      poetry2nix,
-      nixpkgs-newest,
-    }:
-    flake-utils.lib.eachDefaultSystem (
-      system:
-      let
-        pkgs = nixpkgs.legacyPackages.${system};
-        # Use the newest nixpkgs exclusively for the poetry package.
-        pkgsNew = nixpkgs-newest.legacyPackages.${system};
-        inherit (poetry2nix.lib.mkPoetry2Nix { inherit pkgs; }) mkPoetryApplication defaultPoetryOverrides;
-        inherit poetry2nix;
-      in
-      {
-        packages = {
-          myapp = mkPoetryApplication {
-            projectDir = self;
-            preferWheels = true;
-            overrides = defaultPoetryOverrides.extend (
-              self: super: {
-                umap = super.umap.overridePythonAttrs (old: {
-                  buildInputs = (old.buildInputs or [ ]) ++ [ super.setuptools ];
-                });
-              }
-            );
-          };
-          default = self.packages.${system}.myapp;
-        };
-
-        # Shell for app dependencies.
-        #
-        #     nix develop
-        #
-        # Use this shell for developing your app.
-        devShells.default = pkgs.mkShell {
-          inputsFrom = [
-            self.packages.${system}.myapp
-          ];
-        };
-
-        # Shell for poetry.
-        #
-        #     nix develop .#poetry
-        #
-        # Here we use the poetry package from the newest nixpkgs input while keeping
-        # all other dependencies locked.
-        devShells.poetry = pkgs.mkShell {
-          packages = [ pkgsNew.poetry ];
-        };
-      }
-    );
-}

tools/gridsearch_new.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+"""
+Grid-search launcher (2-GPU, fixed per-GPU batch sizes)
+
+• one job at a time per GPU
+• watchdog heartbeats + error pushes
+
+"""
+
+import datetime
+import itertools
+import queue
+import signal
+import subprocess
+import sys
+import threading
+import time
+from os import environ
+from pathlib import Path
+
+import GPUtil
+import requests
+
+# ─────────────── Watchdog / host configuration ────────────────
+WATCHDOG_ROOT   = "https://mt.the-k.at"
+BASIC_AUTH      = ("jan", "jruuI3GZ@9Rnt7")
+HEARTBEAT_EVERY = 60
+HOSTNAME = "gpu-node-01"
+OUT_FOLDER = Path("./grid_search")
+DONE_FOLDER = OUT_FOLDER / "done"
+FAILED_FOLDER = OUT_FOLDER / "failed"
+
+PING_URL, ERR_URL = f"{WATCHDOG_ROOT}/ping", f"{WATCHDOG_ROOT}/error"
+HEADERS = {"Content-Type": "application/json"}
+
+# ───────────────────── Hyper-parameter grid ─────────────────────
+PARAM_GRID = {
+    "network_name": ["subter_LeNet", "subter_efficient"],
+    "latent_space_dim": [32, 64, 128, 256, 512, 768, 1024],
+    "semi_label": [(0, 0), (50, 10), (500, 100)],  # (normal, anomaly)
+}
+
+# ────────────────────── FIXED PARAMETERS ──────────────────────
+BATCH_SIZE = 128  # fixed batch size per GPU
+EPOCHS_AE = 50  # autoencoder epochs
+EPOCHS_DEEPSAD = 120  # Deep SAD epochs
+
+CMD_TEMPLATE = (
+    "python src/main.py train subter {network_name} {exp_folder} "
+    "~/jan/data --k_fold True --k_fold_num 5 "
+    "--num_known_normal {num_known_normal} --num_known_outlier {num_known_anomaly} "
+    "--lr 0.0001 --n_epochs {deepsad_epochs} --lr_milestone 50 --batch_size {batch_size} "
+    "--weight_decay 0.5e-6 --latent_space_dim {latent_space_dim} "
+    "--pretrain True --ae_lr 0.0001 --ae_n_epochs {ae_epochs} --ae_batch_size {batch_size} "
+    "--ae_weight_decay 0.5e-3 --normal_class 0 --known_outlier_class 1 "
+    "--n_known_outlier_classes 1 --seed 3 --device cuda:0"
+)
+
+# ──────────────── global bookkeeping / queues ──────────────────
+STOP_EVT = threading.Event()
+JOB_TABLE = []  # [{ …, status }]
+
+
+# ─────────────────────── helper functions ──────────────────────
+def post_json(url, payload):
+    try:
+        requests.post(
+            url, json=payload, headers=HEADERS, auth=BASIC_AUTH, timeout=5
+        ).raise_for_status()
+    except Exception as exc:
+        print(f"[warn] POST {url} failed: {exc}", file=sys.stderr)
+
+
+def gpu_stats():
+    stats = []
+    for g in GPUtil.getGPUs():
+        stats.append(
+            {
+                "id": g.id,
+                "util": int(g.load * 100),
+                "mem": {"used": int(g.memoryUsed), "total": int(g.memoryTotal)},
+            }
+        )
+    return stats
+
+
+def summarise_jobs():
+    out = {"pending": 0, "running": 0, "ok": 0, "fail": 0}
+    for j in JOB_TABLE:
+        out[j["status"]] += 1
+    return out
+
+
+def heartbeater():
+    while not STOP_EVT.wait(HEARTBEAT_EVERY):
+        post_json(
+            PING_URL, {"host": HOSTNAME, "gpu": gpu_stats(), "status": summarise_jobs()}
+        )
+
+
+# ───────────────────────── worker logic ────────────────────────
+def worker(device: str, q: "queue.Queue[dict]"):
+    while True:
+        job = q.get()
+        if job is None:
+            break  # sentinel for shutdown
+        wait_until_allowed_hours()
+        run_job(job, device)
+        q.task_done()
+
+
+def exp_folder_name(params):
+    num_norm, num_anom = params["semi_label"]
+    return (
+        f"{params['network_name']}_"
+        f"latent{params['latent_space_dim']}_"
+        f"n{num_norm}_a{num_anom}"
+    )
+
+
+def move_exp_folder(exp_folder: Path, target_folder: Path):
+    """Move the experiment folder to the target folder."""
+    destination_folder = target_folder / exp_folder.name
+    if destination_folder.exists():
+        destination_folder.unlink()
+    target_folder.mkdir(parents=True, exist_ok=True)
+    exp_folder.rename(target_folder / exp_folder.name)
+
+
+def already_done_set():
+    if not DONE_FOLDER.exists():
+        return set()
+    return set([f.name for f in DONE_FOLDER.iterdir() if f.is_dir()])
+
+
+def run_job(job: dict, device: str):
+    num_norm, num_anom = job["params"]["semi_label"]
+    exp_folder = job["exp_folder"]
+    exp_folder.mkdir(parents=True, exist_ok=True)
+
+    cmd = CMD_TEMPLATE.format(
+        network_name=job["params"]["network_name"],
+        exp_folder=exp_folder.as_posix(),
+        num_known_normal=num_norm,
+        num_known_anomaly=num_anom,
+        latent_space_dim=job["params"]["latent_space_dim"],
+        batch_size=BATCH_SIZE,
+        ae_epochs=EPOCHS_AE,
+        deepsad_epochs=EPOCHS_DEEPSAD,
+    )
+
+    job.update({"device": device, "status": "running", "start": time.time()})
+    post_json(
+        PING_URL, {"host": HOSTNAME, "msg": f"starting {exp_folder.name} on {device}"}
+    )
+    print(f"EXEC [{device}] {cmd}", flush=True)
+
+    try:
+        env = environ.copy()
+        env["CUDA_VISIBLE_DEVICES"] = device.split(":")[1]  # "0" or "1"
+        res = subprocess.run(
+            cmd, shell=True, capture_output=True, text=True, env=env, check=False
+        )
+        job.update({"end": time.time(), "returncode": res.returncode})
+        job["status"] = "ok" if res.returncode == 0 else "fail"
+        if res.returncode == 0:
+            move_exp_folder(exp_folder, DONE_FOLDER)
+        else:
+            move_exp_folder(exp_folder, FAILED_FOLDER)
+            tail = (res.stderr or "no-stderr")[-500:]
+            post_json(
+                ERR_URL,
+                {
+                    "host": HOSTNAME,
+                    "msg": f"rc {res.returncode} {exp_folder.name} → {tail}",
+                },
+            )
+            print(
+                f"ERRR [{device}] rc {res.returncode} {exp_folder.name} → {tail}",
+                flush=True,
+            )
+    except Exception as exc:
+        job.update({"end": time.time(), "status": "fail", "returncode": -999})
+        move_exp_folder(exp_folder, FAILED_FOLDER)
+        post_json(
+            ERR_URL,
+            {"host": HOSTNAME, "msg": f'exception "{exc}" in {exp_folder.name}'},
+        )
+
+
+# ────────────────────── graceful shutdown ──────────────────────
+def shutdown(reason):
+    post_json(ERR_URL, {"host": HOSTNAME, "msg": f"run stopped: {reason}"})
+    STOP_EVT.set()
+    sys.exit(0)
+
+
+for sig in (signal.SIGINT, signal.SIGTERM):
+    signal.signal(sig, lambda s, f, sig=sig: shutdown(signal.Signals(sig).name))
+
+
+# ─────────────────────────── main() ────────────────────────────
+def wait_until_allowed_hours():
+    """Sleep until the current time is between 22:00 and 04:00."""
+    while True:
+        now = datetime.datetime.now()
+        hour = now.hour
+        # Allowed window: 22:00 (22) to 04:00 (4)
+        if hour >= 22 or hour < 4:
+            break
+        # Calculate seconds until 22:00 today or 22:00 tomorrow if after 4am
+        if hour < 22:
+            next_allowed = now.replace(hour=22, minute=0, second=0, microsecond=0)
+        else:
+            # After 4am but before 22:00, so next allowed is tonight
+            next_allowed = now.replace(hour=22, minute=0, second=0, microsecond=0)
+        if next_allowed < now:
+            next_allowed += datetime.timedelta(days=1)
+        wait_seconds = (next_allowed - now).total_seconds()
+        print(f"[info] Waiting until 22:00 to start new jobs ({int(wait_seconds/60)} min)...")
+        time.sleep(min(wait_seconds, 60*10))  # Sleep in chunks of max 10 minutes
+
+def main():
+    threading.Thread(target=heartbeater, daemon=True).start()
+
+    # two worker queues, one per GPU
+    q0, q1 = queue.Queue(), queue.Queue()
+    threading.Thread(target=worker, args=("cuda:0", q0), daemon=True).start()
+    threading.Thread(target=worker, args=("cuda:1", q1), daemon=True).start()
+
+    # build full parameter combinations as jobs
+    keys = list(PARAM_GRID)
+    jobs = []
+    for combo in itertools.product(*PARAM_GRID.values()):
+        params = dict(zip(keys, combo))
+        exp_folder = OUT_FOLDER / exp_folder_name(params)
+        job = {
+            "params": params,
+            "exp_folder": exp_folder,
+            "status": "pending",
+            "start": None,
+            "end": None,
+            "returncode": None,
+        }
+        jobs.append(job)
+
+    # Check which jobs are already done and mark them
+    done_exps = already_done_set()
+    #print(f"Already done jobs found: {done_exps}")
+    
+    for job in jobs:
+        if job["exp_folder"].name in done_exps:
+            job["status"] = "ok"
+
+    # Print summary of job statuses before starting
+    n_done = sum(1 for job in jobs if job["status"] == "ok")
+    n_pending = sum(1 for job in jobs if job["status"] != "ok")
+    print(f"[info] {n_done} jobs already done, {n_pending} jobs pending.")
+
+    # Add all jobs to global JOB_TABLE for bookkeeping
+    JOB_TABLE.extend(jobs)
+
+    # Only enqueue jobs that are not already done
+    for job in jobs:
+        if job["status"] == "ok":
+            continue
+        # Hardcode device assignment
+        if job["params"]["network_name"] == "subter_LeNet":
+            q1.put(job)  # cuda:1
+        elif job["params"]["network_name"] == "subter_efficient":
+            q0.put(job)  # cuda:0
+        else:
+            # fallback: load-balance
+            (q0 if q0.qsize() <= q1.qsize() else q1).put(job)
+
+    q0.join()
+    q1.join()  # wait for all jobs to drain
+    q0.put(None)
+    q1.put(None)  # terminate workers
+    STOP_EVT.set()
+
+    post_json(
+        PING_URL,
+        {
+            "host": HOSTNAME,
+            "msg": "all jobs done",
+            "gpu": gpu_stats(),
+            "status": summarise_jobs(),
+        },
+    )
+    print("[info] grid search completed")
+
+
+if __name__ == "__main__":
+    main()

tools/plot_scripts/ae_elbow_lenet.py

@@ -0,0 +1,269 @@
+# ae_elbow_from_df.py
+
+from __future__ import annotations
+
+import json
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+import polars as pl
+
+# CHANGE THIS IMPORT IF YOUR LOADER MODULE IS NAMED DIFFERENTLY
+from load_results import load_pretraining_results_dataframe
+
+# ----------------------------
+# Config
+# ----------------------------
+ROOT = Path("/home/fedex/mt/results/done")  # experiments root you pass to the loader
+OUTPUT_DIR = Path("/home/fedex/mt/plots/ae_elbow_lenet_from_df")
+
+# Which label field to use from the DF; "labels_exp_based" or "labels_manual_based"
+LABEL_FIELD = "labels_exp_based"
+
+
+# ----------------------------
+# Helpers
+# ----------------------------
+def canonicalize_network(name: str) -> str:
+    """Map various net_name strings to clean labels for plotting."""
+    low = (name or "").lower()
+    if "lenet" in low:
+        return "LeNet"
+    if "efficient" in low:
+        return "Efficient"
+    # fallback: show whatever was stored
+    return name or "unknown"
+
+
+def calculate_batch_mean_loss(scores: np.ndarray, batch_size: int) -> float:
+    """Mean of per-batch means (matches how the original test loss was computed)."""
+    n = len(scores)
+    if n == 0:
+        return np.nan
+    if batch_size <= 0:
+        batch_size = n  # single batch fallback
+    n_batches = (n + batch_size - 1) // batch_size
+    acc = 0.0
+    for i in range(0, n, batch_size):
+        acc += float(np.mean(scores[i : i + batch_size]))
+    return acc / n_batches
+
+
+def extract_batch_size(cfg_json: str) -> int:
+    """
+    Prefer AE batch size; fall back to general batch_size; then a safe default.
+    We only rely on config_json (no lifted fields).
+    """
+    try:
+        cfg = json.loads(cfg_json) if cfg_json else {}
+    except Exception:
+        cfg = {}
+    return int(cfg.get("ae_batch_size") or cfg.get("batch_size") or 256)
+
+
+def build_arch_curves_from_df(
+    df: pl.DataFrame,
+    label_field: str = "labels_exp_based",
+    only_nets: set[str] | None = None,
+):
+    """
+    From the AE pretraining DF, compute (dims, means, stds) for normal/anomaly/overall
+    grouped by network and latent_dim. Returns:
+        { net_label: {
+            "normal":  (dims, means, stds),
+            "anomaly": (dims, means, stds),
+            "overall": (dims, means, stds),
+        } }
+    """
+    # if "split" not in df.columns:
+    #     raise ValueError("Expected 'split' column in AE dataframe.")
+    if "scores" not in df.columns:
+        raise ValueError("Expected 'scores' column in AE dataframe.")
+    if "network" not in df.columns or "latent_dim" not in df.columns:
+        raise ValueError("Expected 'network' and 'latent_dim' columns in AE dataframe.")
+    if label_field not in df.columns:
+        raise ValueError(f"Expected '{label_field}' column in AE dataframe.")
+
+    # Keep only test split
+    # df = df.filter(pl.col("split") == "test")
+
+    groups: dict[tuple[str, int], dict[str, list[float]]] = {}
+
+    for row in df.iter_rows(named=True):
+        net_label = canonicalize_network(row["network"])
+        if only_nets and net_label not in only_nets:
+            continue
+
+        dim = int(row["latent_dim"])
+        batch_size = extract_batch_size(row.get("config_json"))
+        scores = np.asarray(row["scores"] or [], dtype=float)
+
+        labels = row.get(label_field)
+        labels = np.asarray(labels, dtype=int) if labels is not None else None
+
+        overall_loss = calculate_batch_mean_loss(scores, batch_size)
+
+        # Split by labels if available; otherwise we only aggregate overall
+        normal_loss = np.nan
+        anomaly_loss = np.nan
+        if labels is not None and labels.size == scores.size:
+            normal_scores = scores[labels == 1]
+            anomaly_scores = scores[labels == -1]
+            if normal_scores.size > 0:
+                normal_loss = calculate_batch_mean_loss(normal_scores, batch_size)
+            if anomaly_scores.size > 0:
+                anomaly_loss = calculate_batch_mean_loss(anomaly_scores, batch_size)
+
+        key = (net_label, dim)
+        if key not in groups:
+            groups[key] = {"normal": [], "anomaly": [], "overall": []}
+        groups[key]["overall"].append(overall_loss)
+        groups[key]["normal"].append(normal_loss)
+        groups[key]["anomaly"].append(anomaly_loss)
+
+    # Aggregate across folds -> per (net, dim) mean/std
+    per_net_dims: dict[str, set[int]] = {}
+    for net, dim in groups:
+        per_net_dims.setdefault(net, set()).add(dim)
+
+    result: dict[str, dict[str, tuple[list[int], list[float], list[float]]]] = {}
+    for net, dims in per_net_dims.items():
+        dims_sorted = sorted(dims)
+
+        def collect(kind: str):
+            means, stds = [], []
+            for d in dims_sorted:
+                xs = [
+                    x
+                    for (n2, d2), v in groups.items()
+                    if n2 == net and d2 == d
+                    for x in v[kind]
+                    if x is not None and not np.isnan(x)
+                ]
+                if len(xs) == 0:
+                    means.append(np.nan)
+                    stds.append(np.nan)
+                else:
+                    means.append(float(np.mean(xs)))
+                    stds.append(float(np.std(xs)))
+            return dims_sorted, means, stds
+
+        result[net] = {
+            "normal": collect("normal"),
+            "anomaly": collect("anomaly"),
+            "overall": collect("overall"),
+        }
+
+    return result
+
+
+def plot_multi_loss_curve(arch_results, title, output_path, colors=None):
+    """
+    arch_results: {arch_name: (dims, means, stds)}
+    """
+    plt.figure(figsize=(10, 6))
+
+    # default color map if not provided
+    if colors is None:
+        colors = {
+            "LeNet": "tab:blue",
+            "Efficient": "tab:orange",
+        }
+
+    # Get unique dimensions across all architectures
+    all_dims = sorted(
+        set(dim for _, (dims, _, _) in arch_results.items() for dim in dims)
+    )
+
+    for arch_name, (dims, means, stds) in arch_results.items():
+        color = colors.get(arch_name)
+        # Plot line
+        if color is None:
+            plt.plot(dims, means, marker="o", label=arch_name)
+            plt.fill_between(
+                dims,
+                np.array(means) - np.array(stds),
+                np.array(means) + np.array(stds),
+                alpha=0.2,
+            )
+        else:
+            plt.plot(dims, means, marker="o", color=color, label=arch_name)
+            plt.fill_between(
+                dims,
+                np.array(means) - np.array(stds),
+                np.array(means) + np.array(stds),
+                color=color,
+                alpha=0.2,
+            )
+
+    plt.xlabel("Latent Dimensionality")
+    plt.ylabel("Test Loss")
+    # plt.title(title)
+    plt.legend()
+    plt.grid(True, alpha=0.3)
+    plt.xticks(all_dims)
+    plt.tight_layout()
+    plt.savefig(output_path, dpi=150, bbox_inches="tight")
+    plt.close()
+
+
+def main():
+    # Load AE DF (uses your cache if enabled in the loader)
+    df = load_pretraining_results_dataframe(ROOT, allow_cache=True)
+
+    # Optional: filter to just LeNet vs Efficient; drop this set() to plot all nets
+    wanted_nets = {"LeNet", "Efficient"}
+
+    curves = build_arch_curves_from_df(
+        df,
+        label_field=LABEL_FIELD,
+        only_nets=wanted_nets,
+    )
+
+    # Prepare output dirs
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    ts_dir = OUTPUT_DIR / "archive" / datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    ts_dir.mkdir(parents=True, exist_ok=True)
+
+    def pick(kind: str):
+        # kind in {"normal","anomaly","overall"}
+        return {name: payload[kind] for name, payload in curves.items()}
+
+    plot_multi_loss_curve(
+        pick("normal"),
+        "Normal Class Test Loss vs. Latent Dimensionality",
+        ts_dir / "ae_elbow_test_loss_normal.png",
+    )
+
+    plot_multi_loss_curve(
+        pick("anomaly"),
+        "Anomaly Class Test Loss vs. Latent Dimensionality",
+        ts_dir / "ae_elbow_test_loss_anomaly.png",
+    )
+
+    plot_multi_loss_curve(
+        pick("overall"),
+        "Overall Test Loss vs. Latent Dimensionality",
+        ts_dir / "ae_elbow_test_loss_overall.png",
+    )
+
+    # Copy this script to preserve the code used for the outputs
+    script_path = Path(__file__)
+    shutil.copy2(script_path, ts_dir)
+
+    # Optionally mirror latest
+    latest = OUTPUT_DIR / "latest"
+    latest.mkdir(exist_ok=True, parents=True)
+    for f in ts_dir.iterdir():
+        if f.is_file():
+            shutil.copy2(f, latest / f.name)
+
+    print(f"Saved plots to: {ts_dir}")
+    print(f"Also updated: {latest}")
+
+
+if __name__ == "__main__":
+    main()

tools/plot_scripts/background_ml_illustrations.py

@@ -0,0 +1,223 @@
+"""
+Downloads the cats_vs_dogs dataset, then generates four PNGs:
+ - supervised_grid.png
+ - unsupervised_clusters.png
+ - semi_supervised_classification.png
+ - semi_supervised_clustering.png
+"""
+
+import os
+import random
+
+import matplotlib.pyplot as plt
+import numpy as np
+import tensorflow as tf
+import tensorflow_datasets as tfds
+from sklearn.cluster import KMeans
+from sklearn.decomposition import PCA
+from sklearn.linear_model import LogisticRegression
+from sklearn.semi_supervised import LabelSpreading
+
+# -----------------------------------------------------------------------------
+# CONFIGURATION
+# -----------------------------------------------------------------------------
+NUM_SUPERVISED = 16
+GRID_ROWS = 4
+GRID_COLS = 4
+
+UNSUP_SAMPLES = 200
+
+# how many labeled points to “seed” semi-sup methods
+N_LABELED_CLASS = 10  # for classification demo
+N_SEEDS_PER_CLASS = 3  # for clustering demo
+
+OUTPUT_DIR = "outputs"
+
+
+# -----------------------------------------------------------------------------
+# UTILITIES
+# -----------------------------------------------------------------------------
+def ensure_dir(path):
+    os.makedirs(path, exist_ok=True)
+
+
+# -----------------------------------------------------------------------------
+# 1) Supervised grid
+# -----------------------------------------------------------------------------
+def plot_supervised_grid(ds, info, num, rows, cols, outpath):
+    plt.figure(figsize=(cols * 2, rows * 2))
+    for i, (img, lbl) in enumerate(ds.take(num)):
+        ax = plt.subplot(rows, cols, i + 1)
+        ax.imshow(img.numpy().astype("uint8"))
+        ax.axis("off")
+        cname = info.features["label"].int2str(lbl.numpy())
+        ax.set_title(cname, fontsize=9)
+    plt.tight_layout()
+    plt.savefig(outpath, dpi=150)
+    plt.close()
+    print(f"✔ Saved supervised grid → {outpath}")
+
+
+# -----------------------------------------------------------------------------
+# 2) Unsupervised clustering (PCA + KMeans)
+# -----------------------------------------------------------------------------
+def plot_unsupervised_clusters(ds, outpath):
+    imgs = []
+    for img, _ in ds.take(UNSUP_SAMPLES):
+        arr = (
+            tf.image.resize(img, (64, 64)).numpy().astype("float32").mean(axis=2)
+        )  # resize and grayscale to speed up
+        imgs.append(arr.ravel() / 255.0)
+    X = np.stack(imgs)
+    pca = PCA(n_components=2, random_state=0)
+    X2 = pca.fit_transform(X)
+
+    km = KMeans(n_clusters=2, random_state=0)
+    clusters = km.fit_predict(X2)
+
+    plt.figure(figsize=(6, 6))
+    plt.scatter(X2[:, 0], X2[:, 1], c=clusters, s=15, alpha=0.6)
+    plt.title("Unsupervised: PCA + KMeans")
+    plt.xlabel("PCA 1")
+    plt.ylabel("PCA 2")
+    plt.tight_layout()
+    plt.savefig(outpath, dpi=150)
+    plt.close()
+    print(f"✔ Saved unsupervised clusters → {outpath}")
+
+    return X2, clusters  # return for reuse
+
+
+# -----------------------------------------------------------------------------
+# 3) Semi‐supervised classification demo
+# -----------------------------------------------------------------------------
+def plot_semi_supervised_classification(X2, y_true, outpath):
+    n = X2.shape[0]
+    all_idx = list(range(n))
+    labeled_idx = random.sample(all_idx, N_LABELED_CLASS)
+    unlabeled_idx = list(set(all_idx) - set(labeled_idx))
+
+    # pure supervised
+    clf = LogisticRegression().fit(X2[labeled_idx], y_true[labeled_idx])
+
+    # semi‐supervised
+    y_train = np.full(n, -1, dtype=int)
+    y_train[labeled_idx] = y_true[labeled_idx]
+    ls = LabelSpreading().fit(X2, y_train)
+
+    # grid for decision boundary
+    x_min, x_max = X2[:, 0].min() - 1, X2[:, 0].max() + 1
+    y_min, y_max = X2[:, 1].min() - 1, X2[:, 1].max() + 1
+    xx, yy = np.meshgrid(np.linspace(x_min, x_max, 200), np.linspace(y_min, y_max, 200))
+    grid = np.c_[xx.ravel(), yy.ravel()]
+    pred_sup = clf.predict(grid).reshape(xx.shape)
+    pred_semi = ls.predict(grid).reshape(xx.shape)
+
+    fig, axes = plt.subplots(1, 2, figsize=(12, 5))
+    for ax, Z, title in zip(
+        axes, [pred_sup, pred_semi], ["Supervised only", "Semi-supervised"]
+    ):
+        ax.contourf(xx, yy, Z, alpha=0.3)
+        ax.scatter(X2[unlabeled_idx, 0], X2[unlabeled_idx, 1], s=20, alpha=0.4)
+        ax.scatter(
+            X2[labeled_idx, 0],
+            X2[labeled_idx, 1],
+            c=y_true[labeled_idx],
+            s=80,
+            edgecolor="k",
+        )
+        ax.set_title(title)
+        ax.set_xlabel("PCA 1")
+        ax.set_ylabel("PCA 2")
+    plt.tight_layout()
+    plt.savefig(outpath, dpi=150)
+    plt.close()
+    print(f"✔ Saved semi-supervised classification → {outpath}")
+
+
+# -----------------------------------------------------------------------------
+# 4) Semi‐supervised clustering (seeded KMeans)
+# -----------------------------------------------------------------------------
+def plot_semi_supervised_clustering(X2, y_true, outpath):
+    # pick a few seeds per class
+    cats = np.where(y_true == 0)[0]
+    dogs = np.where(y_true == 1)[0]
+    seeds = list(np.random.choice(cats, N_SEEDS_PER_CLASS, replace=False)) + list(
+        np.random.choice(dogs, N_SEEDS_PER_CLASS, replace=False)
+    )
+
+    # pure KMeans
+    km1 = KMeans(n_clusters=2, random_state=0).fit(X2)
+    lab1 = km1.labels_
+
+    # seeded: init centers from seed means
+    ctr0 = X2[seeds[:N_SEEDS_PER_CLASS]].mean(axis=0)
+    ctr1 = X2[seeds[N_SEEDS_PER_CLASS:]].mean(axis=0)
+    km2 = KMeans(
+        n_clusters=2, init=np.vstack([ctr0, ctr1]), n_init=1, random_state=0
+    ).fit(X2)
+    lab2 = km2.labels_
+
+    fig, axes = plt.subplots(1, 2, figsize=(12, 5))
+    for ax, labels, title in zip(axes, [lab1, lab2], ["Pure KMeans", "Seeded KMeans"]):
+        ax.scatter(X2[:, 0], X2[:, 1], c=labels, s=20, alpha=0.6)
+        ax.scatter(
+            X2[seeds, 0],
+            X2[seeds, 1],
+            c=y_true[seeds],
+            edgecolor="k",
+            marker="x",
+            s=100,
+            linewidths=2,
+        )
+        ax.set_title(title)
+        ax.set_xlabel("PCA 1")
+        ax.set_ylabel("PCA 2")
+    plt.tight_layout()
+    plt.savefig(outpath, dpi=150)
+    plt.close()
+    print(f"✔ Saved semi-supervised clustering → {outpath}")
+
+
+# -----------------------------------------------------------------------------
+# MAIN
+# -----------------------------------------------------------------------------
+if __name__ == "__main__":
+    ensure_dir(OUTPUT_DIR)
+
+    # load
+    print("▶ Loading cats_vs_dogs...")
+    ds, info = tfds.load(
+        "cats_vs_dogs", split="train", with_info=True, as_supervised=True
+    )
+    ds = ds.shuffle(1000, reshuffle_each_iteration=False).cache()
+
+    # supervised
+    plot_supervised_grid(
+        ds,
+        info,
+        NUM_SUPERVISED,
+        GRID_ROWS,
+        GRID_COLS,
+        os.path.join(OUTPUT_DIR, "supervised_grid.png"),
+    )
+
+    # unsupervised
+    # also returns X2 for downstream demos
+    X2, _ = plot_unsupervised_clusters(
+        ds, os.path.join(OUTPUT_DIR, "unsupervised_clusters.png")
+    )
+
+    # collect true labels for that same subset
+    # (we need a y_true array for semi-sup demos)
+    y_true = np.array([lbl.numpy() for _, lbl in ds.take(UNSUP_SAMPLES)])
+
+    # semi-supervised classification
+    plot_semi_supervised_classification(
+        X2, y_true, os.path.join(OUTPUT_DIR, "semi_supervised_classification.png")
+    )
+
+    # semi-supervised clustering
+    plot_semi_supervised_clustering(
+        X2, y_true, os.path.join(OUTPUT_DIR, "semi_supervised_clustering.png")
+    )

tools/plot_scripts/background_ml_illustrations/flake.nix

@@ -1,40 +0,0 @@
-{
-  description = "Python 3.13 devshell with tensorflow-datasets, matplotlib, scikit-learn and numpy";
-
-  inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
-    flake-utils.url = "github:numtide/flake-utils";
-  };
-
-  outputs =
-    {
-      self,
-      nixpkgs,
-      flake-utils,
-      ...
-    }:
-    flake-utils.lib.eachDefaultSystem (
-      system:
-      let
-        pkgs = import nixpkgs {
-          inherit system;
-          # optional: config = { allowUnfree = true; };
-        };
-      in
-      {
-        devShells.default = pkgs.mkShell {
-          name = "py313-devshell";
-          # bring in the Python 3.13 packages
-          buildInputs =
-            with pkgs;
-            [ python313 ]
-            ++ (with pkgs.python313Packages; [
-              tensorflow-datasets
-              matplotlib
-              scikit-learn
-              numpy
-            ]);
-        };
-      }
-    );
-}

tools/plot_scripts/background_ml_semisupervised.py

@@ -0,0 +1,274 @@
+"""
+Downloads the cats_vs_dogs dataset, extracts deep features using MobileNetV2,
+then generates two clustering comparison illustrations using two different embedding pipelines:
+  - PCA followed by t-SNE (saved as: semi_supervised_clustering_tsne.png)
+  - PCA followed by UMAP  (saved as: semi_supervised_clustering_umap.png)
+
+Each illustration compares:
+   - Unsupervised clustering using the deep embedding + KMeans
+   - Semi-supervised clustering using Label Spreading with a few labeled seeds
+
+This script saves outputs in a datetime folder and also copies the latest outputs
+to a "latest" folder. All versions of the outputs and scripts are archived.
+"""
+
+import random
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+import tensorflow as tf
+import tensorflow_datasets as tfds
+import umap
+from scipy.optimize import linear_sum_assignment
+from sklearn.cluster import KMeans
+from sklearn.decomposition import PCA
+from sklearn.manifold import TSNE
+from sklearn.semi_supervised import LabelSpreading
+
+# -----------------------------------------------------------------------------
+# CONFIGURATION
+# -----------------------------------------------------------------------------
+UNSUP_SAMPLES = 200  # number of samples to use for the demo
+N_LABELED_CLASS = 20  # number of labeled seeds for the semi-supervised approach
+
+output_path = Path("/home/fedex/mt/plots/background_ml_semisupervised")
+datetime_folder_name = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+latest_folder_path = output_path / "latest"
+archive_folder_path = output_path / "archive"
+output_datetime_path = output_path / datetime_folder_name
+
+
+# -----------------------------------------------------------------------------
+# UTILITIES
+# -----------------------------------------------------------------------------
+def ensure_dir(directory: Path):
+    directory.mkdir(exist_ok=True, parents=True)
+
+
+def cluster_accuracy(y_true, y_pred):
+    """
+    Compute clustering accuracy by determining the optimal mapping
+    between predicted clusters and true labels using the Hungarian algorithm.
+    """
+    y_true = y_true.astype(np.int64)
+    y_pred = y_pred.astype(np.int64)
+    labels = np.unique(y_true)
+    clusters = np.unique(y_pred)
+    contingency = np.zeros((labels.size, clusters.size), dtype=np.int64)
+    for i, label in enumerate(labels):
+        for j, cluster in enumerate(clusters):
+            contingency[i, j] = np.sum((y_true == label) & (y_pred == cluster))
+    row_ind, col_ind = linear_sum_assignment(-contingency)
+    accuracy = contingency[row_ind, col_ind].sum() / y_true.size
+    return accuracy
+
+
+def plot_clustering_comparison_embedding(embedding, y_true, outpath, method_name=""):
+    """
+    Given a 2D data embedding (e.g., from PCA+t-SNE or PCA+UMAP), this function:
+      - Performs unsupervised clustering with KMeans.
+      - Performs semi-supervised clustering with Label Spreading using a few labeled seeds.
+      - Computes accuracy via the Hungarian algorithm.
+      - Plots the decision boundaries from both methods overlaid with the true labels.
+      - Annotates the plot with the accuracy results.
+
+    The 'method_name' is used in the plot title to indicate which embedding is used.
+    """
+    n = embedding.shape[0]
+    all_idx = list(range(n))
+    labeled_idx = random.sample(all_idx, N_LABELED_CLASS)
+
+    # Unsupervised clustering using KMeans on all embedded data
+    km = KMeans(n_clusters=2, random_state=0).fit(embedding)
+    unsup_pred = km.predict(embedding)
+    unsup_accuracy = cluster_accuracy(y_true, unsup_pred)
+
+    # Create a grid over the space for decision boundaries
+    x_min, x_max = embedding[:, 0].min() - 1, embedding[:, 0].max() + 1
+    y_min, y_max = embedding[:, 1].min() - 1, embedding[:, 1].max() + 1
+    xx, yy = np.meshgrid(np.linspace(x_min, x_max, 200), np.linspace(y_min, y_max, 200))
+    grid = np.c_[xx.ravel(), yy.ravel()]
+    pred_unsup = km.predict(grid).reshape(xx.shape)
+
+    # Semi-supervised clustering using Label Spreading with labeled seeds
+    y_train = np.full(n, -1, dtype=int)
+    y_train[labeled_idx] = y_true[labeled_idx]
+    ls = LabelSpreading().fit(embedding, y_train)
+    semi_pred = ls.predict(embedding)
+    semi_accuracy = cluster_accuracy(y_true, semi_pred)
+    pred_semi = ls.predict(grid).reshape(xx.shape)
+
+    cmap = plt.cm.coolwarm
+
+    fig, axes = plt.subplots(1, 2, figsize=(12, 6))
+
+    # Unsupervised plot:
+    ax = axes[0]
+    ax.contourf(xx, yy, pred_unsup, alpha=0.2, cmap=cmap)
+    sc1 = ax.scatter(
+        embedding[:, 0],
+        embedding[:, 1],
+        c=y_true,
+        cmap=cmap,
+        s=30,
+        alpha=0.8,
+        edgecolor="k",
+    )
+    ax.set_title(
+        f"{method_name}\nUnsupervised (KMeans) - Acc: {unsup_accuracy:.2f}", fontsize=10
+    )
+    ax.set_xlabel("Dim 1")
+    ax.set_ylabel("Dim 2")
+    handles = []
+    for cl in np.unique(y_true):
+        handles.append(
+            plt.Line2D(
+                [],
+                [],
+                marker="o",
+                linestyle="",
+                color=cmap(cl / (np.max(y_true) + 1)),
+                label=f"Class {cl}",
+                markersize=6,
+            )
+        )
+    ax.legend(handles=handles, loc="upper right")
+
+    # Semi-supervised plot:
+    ax = axes[1]
+    ax.contourf(xx, yy, pred_semi, alpha=0.2, cmap=cmap)
+    sc2 = ax.scatter(
+        embedding[:, 0],
+        embedding[:, 1],
+        c=y_true,
+        cmap=cmap,
+        s=30,
+        alpha=0.8,
+        edgecolor="none",
+    )
+    sc3 = ax.scatter(
+        embedding[labeled_idx, 0],
+        embedding[labeled_idx, 1],
+        c=y_true[labeled_idx],
+        cmap=cmap,
+        s=80,
+        edgecolor="k",
+        marker="o",
+        label="Labeled Seeds",
+    )
+    ax.set_title(
+        f"{method_name}\nSemi-Supervised (Label Spreading) - Acc: {semi_accuracy:.2f}",
+        fontsize=10,
+    )
+    ax.set_xlabel("Dim 1")
+    ax.set_ylabel("Dim 2")
+    handles = []
+    for cl in np.unique(y_true):
+        handles.append(
+            plt.Line2D(
+                [],
+                [],
+                marker="o",
+                linestyle="",
+                color=cmap(cl / (np.max(y_true) + 1)),
+                label=f"Class {cl}",
+                markersize=6,
+            )
+        )
+    handles.append(
+        plt.Line2D(
+            [],
+            [],
+            marker="o",
+            linestyle="",
+            color="black",
+            label="Labeled Seed",
+            markersize=8,
+        )
+    )
+    ax.legend(handles=handles, loc="upper right")
+
+    plt.tight_layout()
+    plt.savefig(outpath, dpi=150)
+    plt.close()
+    print(f"✔ Saved clustering comparison illustration → {outpath}")
+    print(f"Unsupervised Accuracy: {unsup_accuracy:.2f}")
+    print(f"Semi-Supervised Accuracy: {semi_accuracy:.2f}")
+
+
+# -----------------------------------------------------------------------------
+# MAIN WITH PRE-TRAINED CNN FEATURE EXTRACTION
+# -----------------------------------------------------------------------------
+if __name__ == "__main__":
+    # Create output directories
+    ensure_dir(output_path)
+    ensure_dir(output_datetime_path)
+    ensure_dir(latest_folder_path)
+    ensure_dir(archive_folder_path)
+
+    print("▶ Loading cats_vs_dogs dataset...")
+    ds, info = tfds.load(
+        "cats_vs_dogs", split="train", with_info=True, as_supervised=True
+    )
+    ds = ds.shuffle(1000, reshuffle_each_iteration=False).cache()
+
+    # Load a pre-trained CNN (MobileNetV2) for feature extraction.
+    cnn_model = tf.keras.applications.MobileNetV2(
+        include_top=False, weights="imagenet", pooling="avg", input_shape=(224, 224, 3)
+    )
+
+    # Extract deep features for all samples.
+    features_list = []
+    labels = []
+    for img, lbl in ds.take(UNSUP_SAMPLES):
+        # Resize to 224x224 and keep 3 channels.
+        img_resized = tf.image.resize(img, (224, 224))
+        # Preprocess the image for MobileNetV2.
+        img_preprocessed = tf.keras.applications.mobilenet_v2.preprocess_input(
+            img_resized
+        )
+        # Expand dims for batch, run through CNN, then squeeze.
+        features = cnn_model(tf.expand_dims(img_preprocessed, axis=0))
+        features = features.numpy().squeeze()
+        features_list.append(features)
+        labels.append(lbl.numpy())
+    X = np.stack(features_list)
+    y_true = np.array(labels)
+
+    # First, apply PCA to reduce dimensionality to 50
+    pca_50 = PCA(n_components=50, random_state=0).fit_transform(X)
+
+    # Then compute embedding with t-SNE
+    from sklearn.manifold import TSNE
+
+    X_tsne = TSNE(n_components=2, random_state=0, init="pca").fit_transform(pca_50)
+    outfile_tsne = output_datetime_path / "semi_supervised_clustering_tsne.png"
+    plot_clustering_comparison_embedding(
+        X_tsne, y_true, outfile_tsne, "CNN + PCA + t-SNE"
+    )
+
+    # Then compute embedding with UMAP
+    X_umap = umap.UMAP(n_components=2, random_state=0).fit_transform(pca_50)
+    outfile_umap = output_datetime_path / "semi_supervised_clustering_umap.png"
+    plot_clustering_comparison_embedding(
+        X_umap, y_true, outfile_umap, "CNN + PCA + UMAP"
+    )
+
+    # -----------------------------------------------------------------------------
+    # Update the 'latest' results folder: remove previous and copy current outputs
+    # -----------------------------------------------------------------------------
+    shutil.rmtree(latest_folder_path, ignore_errors=True)
+    ensure_dir(latest_folder_path)
+    for file in output_datetime_path.iterdir():
+        shutil.copy2(file, latest_folder_path)
+
+    # Copy this script to preserve the code used for the outputs
+    script_path = Path(__file__)
+    shutil.copy2(script_path, output_datetime_path)
+    shutil.copy2(script_path, latest_folder_path)
+
+    # Archive the outputs
+    shutil.move(output_datetime_path, archive_folder_path)

tools/plot_scripts/background_ml_supervised.py

@@ -0,0 +1,99 @@
+"""
+Downloads the cats_vs_dogs dataset, then generates a supervised grid image:
+ - supervised_grid.png
+
+This script saves outputs in a datetime folder and also copies the latest outputs to a "latest" folder.
+All versions of the outputs and scripts are archived.
+"""
+
+from datetime import datetime
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import tensorflow_datasets as tfds
+
+# -----------------------------------------------------------------------------
+# CONFIGURATION
+# -----------------------------------------------------------------------------
+# Number of supervised samples and grid dimensions
+NUM_SUPERVISED = 16
+GRID_ROWS = 4
+GRID_COLS = 4
+
+# Output directories for saving plots and scripts
+output_path = Path("/home/fedex/mt/plots/background_ml_supervised")
+datetime_folder_name = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+
+latest_folder_path = output_path / "latest"
+archive_folder_path = output_path / "archive"
+output_datetime_path = output_path / datetime_folder_name
+
+
+# -----------------------------------------------------------------------------
+# UTILITIES
+# -----------------------------------------------------------------------------
+def ensure_dir(directory: Path):
+    directory.mkdir(exist_ok=True, parents=True)
+
+
+# Create required output directories
+ensure_dir(output_path)
+ensure_dir(output_datetime_path)
+ensure_dir(latest_folder_path)
+ensure_dir(archive_folder_path)
+
+
+# -----------------------------------------------------------------------------
+# Supervised grid plot
+# -----------------------------------------------------------------------------
+def plot_supervised_grid(ds, info, num, rows, cols, outpath):
+    """
+    Plots a grid of images from the dataset with their corresponding labels.
+    """
+    plt.figure(figsize=(cols * 2, rows * 2))
+    for i, (img, lbl) in enumerate(ds.take(num)):
+        ax = plt.subplot(rows, cols, i + 1)
+        ax.imshow(img.numpy().astype("uint8"))
+        ax.axis("off")
+        cname = info.features["label"].int2str(lbl.numpy())
+        ax.set_title(cname, fontsize=9)
+    plt.tight_layout()
+    plt.savefig(outpath, dpi=150)
+    plt.close()
+    print(f"✔ Saved supervised grid → {outpath}")
+
+
+# -----------------------------------------------------------------------------
+# MAIN
+# -----------------------------------------------------------------------------
+if __name__ == "__main__":
+    # Download and prepare the dataset
+    print("▶ Loading cats_vs_dogs dataset...")
+    ds, info = tfds.load(
+        "cats_vs_dogs", split="train", with_info=True, as_supervised=True
+    )
+    ds = ds.shuffle(1000, reshuffle_each_iteration=False).cache()
+
+    # Generate the supervised grid image
+    supervised_outfile = output_datetime_path / "supervised_grid.png"
+    plot_supervised_grid(
+        ds, info, NUM_SUPERVISED, GRID_ROWS, GRID_COLS, supervised_outfile
+    )
+
+    # -----------------------------------------------------------------------------
+    # Update the 'latest' results folder: remove previous and copy current outputs
+    # -----------------------------------------------------------------------------
+    import shutil
+
+    shutil.rmtree(latest_folder_path, ignore_errors=True)
+    ensure_dir(latest_folder_path)
+    for file in output_datetime_path.iterdir():
+        shutil.copy2(file, latest_folder_path)
+
+    # Copy this script to the output folder and to the latest folder to preserve the used code
+    script_path = Path(__file__)
+    shutil.copy2(script_path, output_datetime_path)
+    shutil.copy2(script_path, latest_folder_path)
+
+    # Move the output datetime folder to the archive folder for record keeping
+    shutil.move(output_datetime_path, archive_folder_path)

tools/plot_scripts/background_ml_unsupervised.py

@@ -0,0 +1,105 @@
+"""
+Downloads the cats_vs_dogs dataset, then generates an unsupervised clusters image:
+ - unsupervised_clusters.png
+
+This script saves outputs in a datetime folder and also copies the latest outputs to a "latest" folder.
+All versions of the outputs and scripts are archived.
+"""
+
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+import tensorflow as tf
+import tensorflow_datasets as tfds
+from sklearn.cluster import KMeans
+from sklearn.decomposition import PCA
+
+# -----------------------------------------------------------------------------
+# CONFIGURATION
+# -----------------------------------------------------------------------------
+UNSUP_SAMPLES = 200
+
+output_path = Path("/home/fedex/mt/plots/background_ml_unsupervised")
+datetime_folder_name = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+
+latest_folder_path = output_path / "latest"
+archive_folder_path = output_path / "archive"
+output_datetime_path = output_path / datetime_folder_name
+
+
+# -----------------------------------------------------------------------------
+# UTILITIES
+# -----------------------------------------------------------------------------
+def ensure_dir(directory: Path):
+    directory.mkdir(exist_ok=True, parents=True)
+
+
+# Create required output directories
+ensure_dir(output_path)
+ensure_dir(output_datetime_path)
+ensure_dir(latest_folder_path)
+ensure_dir(archive_folder_path)
+
+
+# -----------------------------------------------------------------------------
+# Unsupervised Clustering Plot (PCA + KMeans)
+# -----------------------------------------------------------------------------
+def plot_unsupervised_clusters(ds, outpath):
+    """
+    Processes a subset of images from the dataset, reduces their dimensionality with PCA,
+    applies KMeans clustering, and saves a scatterplot of the clusters.
+    """
+    imgs = []
+    for img, _ in ds.take(UNSUP_SAMPLES):
+        # resize to 64x64, convert to grayscale by averaging channels
+        arr = tf.image.resize(img, (64, 64)).numpy().astype("float32").mean(axis=2)
+        imgs.append(arr.ravel() / 255.0)
+    X = np.stack(imgs)
+    pca = PCA(n_components=2, random_state=0)
+    X2 = pca.fit_transform(X)
+
+    km = KMeans(n_clusters=2, random_state=0)
+    clusters = km.fit_predict(X2)
+
+    plt.figure(figsize=(6, 6))
+    plt.scatter(X2[:, 0], X2[:, 1], c=clusters, s=15, alpha=0.6)
+    plt.title("Unsupervised: PCA + KMeans")
+    plt.xlabel("PCA 1")
+    plt.ylabel("PCA 2")
+    plt.tight_layout()
+    plt.savefig(outpath, dpi=150)
+    plt.close()
+    print(f"✔ Saved unsupervised clusters → {outpath}")
+
+
+# -----------------------------------------------------------------------------
+# MAIN
+# -----------------------------------------------------------------------------
+if __name__ == "__main__":
+    # Load and prepare the dataset
+    print("▶ Loading cats_vs_dogs dataset...")
+    ds, _ = tfds.load("cats_vs_dogs", split="train", with_info=True, as_supervised=True)
+    ds = ds.shuffle(1000, reshuffle_each_iteration=False).cache()
+
+    # Generate the unsupervised clusters image
+    unsupervised_outfile = output_datetime_path / "unsupervised_clusters.png"
+    plot_unsupervised_clusters(ds, unsupervised_outfile)
+
+    # -----------------------------------------------------------------------------
+    # Update the 'latest' results folder: remove previous and copy current outputs
+    # -----------------------------------------------------------------------------
+    shutil.rmtree(latest_folder_path, ignore_errors=True)
+    ensure_dir(latest_folder_path)
+    for file in output_datetime_path.iterdir():
+        shutil.copy2(file, latest_folder_path)
+
+    # Copy this script to the output folder and to the latest folder to preserve the used code
+    script_path = Path(__file__)
+    shutil.copy2(script_path, output_datetime_path)
+    shutil.copy2(script_path, latest_folder_path)
+
+    # Move the output datetime folder to the archive folder for record keeping
+    shutil.move(output_datetime_path, archive_folder_path)

tools/plot_scripts/data_anomalies_timeline.py

@@ -0,0 +1,256 @@
+import json
+import pickle
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+# define data paths
+all_data_path = Path("/home/fedex/mt/data/subter")
+output_path = Path("/home/fedex/mt/plots/data_anomalies_timeline")
+datetime_folder_name = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+
+latest_folder_path = output_path / "latest"
+archive_folder_path = output_path / "archive"
+output_datetime_path = output_path / datetime_folder_name
+cache_path = output_path
+
+# if output does not exist, create it
+output_path.mkdir(exist_ok=True, parents=True)
+output_datetime_path.mkdir(exist_ok=True, parents=True)
+latest_folder_path.mkdir(exist_ok=True, parents=True)
+archive_folder_path.mkdir(exist_ok=True, parents=True)
+
+data_resolution = 32 * 2048
+
+# find all bag files and sort them correctly by name
+normal_experiment_paths, anomaly_experiment_paths = [], []
+for bag_file_path in all_data_path.iterdir():
+    if bag_file_path.suffix != ".bag":
+        continue
+    if "smoke" in bag_file_path.name:
+        anomaly_experiment_paths.append(bag_file_path)
+    else:
+        normal_experiment_paths.append(bag_file_path)
+
+# Load manually labeled frames
+with open(
+    cache_path / "manually_labeled_anomaly_frames.json", "r"
+) as frame_borders_file:
+    manually_labeled_anomaly_frames_json = json.load(frame_borders_file)
+    if not manually_labeled_anomaly_frames_json:
+        print("No manually labeled anomaly frames found. Exiting...")
+        exit(1)
+    manually_labeled_anomaly_frames = {}
+    try:
+        for file in manually_labeled_anomaly_frames_json["files"]:
+            if file["filename"] not in (
+                p.with_suffix(".npy").name for p in anomaly_experiment_paths
+            ):
+                print(
+                    f"File {file['filename']} from manually labeled frames not found in anomaly experiments. Exiting..."
+                )
+                exit(1)
+            manually_labeled_anomaly_frames[file["filename"]] = (
+                file["semi_target_begin_frame"],
+                file["semi_target_end_frame"],
+            )
+    except KeyError as e:
+        print(f"Missing key in manually labeled frames JSON: {e}")
+        exit(1)
+
+
+def plot_combined_timeline(
+    normal_experiment_paths, anomaly_experiment_paths, title, num_bins=50
+):
+    """Plot both missing points and near-sensor measurements over normalized timeline"""
+    # Sort experiments by filesize first (to match original processing order)
+    normal_experiment_paths = sorted(
+        normal_experiment_paths, key=lambda path: path.stat().st_size
+    )
+    anomaly_experiment_paths = sorted(
+        anomaly_experiment_paths, key=lambda path: path.stat().st_size
+    )
+
+    # Get largest normal experiment and moving anomaly experiments
+    baseline_path = normal_experiment_paths[-3]  # largest normal experiment
+    moving_exp_indices = [
+        i
+        for i, path in enumerate(anomaly_experiment_paths)
+        if "stationary" not in path.name
+    ]
+    moving_anomaly_paths = [anomaly_experiment_paths[i] for i in moving_exp_indices]
+
+    # Load missing points data
+    missing_points_cache = Path(cache_path / "missing_points.pkl")
+    if not missing_points_cache.exists():
+        print("Missing points cache not found!")
+        return
+
+    # Load near-sensor data (using 500mm threshold)
+    near_sensor_cache = Path(cache_path / "particles_near_sensor_counts_500.pkl")
+    if not near_sensor_cache.exists():
+        print("Near-sensor measurements cache not found!")
+        return
+
+    # Load both cached datasets
+    with open(missing_points_cache, "rb") as file:
+        missing_points_normal, missing_points_anomaly = pickle.load(file)
+    with open(near_sensor_cache, "rb") as file:
+        near_sensor_normal, near_sensor_anomaly = pickle.load(file)
+
+    # Get data for baseline and moving experiments
+    missing_data = [missing_points_normal[-3]] + [
+        missing_points_anomaly[i] for i in moving_exp_indices
+    ]
+    near_sensor_data = [near_sensor_normal[-3]] + [
+        near_sensor_anomaly[i] for i in moving_exp_indices
+    ]
+    all_paths = [baseline_path] + moving_anomaly_paths
+
+    # Create figure with two y-axes and dynamic layout
+    fig, ax1 = plt.subplots(figsize=(12, 6), constrained_layout=True)
+    ax2 = ax1.twinx()
+
+    # Color schemes - gray for baseline, colors for anomaly experiments
+    experiment_colors = ["#808080"] + ["#1f77b4", "#ff7f0e", "#2ca02c"]  # gray + colors
+
+    # First create custom legend handles for the metrics
+    from matplotlib.lines import Line2D
+
+    metric_legend = [
+        Line2D([0], [0], color="gray", linestyle="-", label="Missing Points"),
+        Line2D([0], [0], color="gray", linestyle="--", label="Near-Sensor (<0.5m)"),
+        Line2D([0], [0], color="gray", linestyle=":", label="Manually Labeled Borders"),
+    ]
+
+    # Plot each experiment's data
+    for i, (missing_exp, near_sensor_exp) in enumerate(
+        zip(missing_data, near_sensor_data)
+    ):
+        # Get experiment name without the full path
+        exp_name = all_paths[i].stem
+        # Shorten experiment name if needed
+        exp_name = exp_name.replace("experiment_smoke_", "exp_")
+
+        # Convert both to percentages
+        missing_pct = np.array(missing_exp) / data_resolution * 100
+        near_sensor_pct = np.array(near_sensor_exp) / data_resolution * 100
+
+        # Create normalized timeline bins for both
+        exp_len = len(missing_pct)
+        bins = np.linspace(0, exp_len - 1, num_bins)
+        missing_binned = np.zeros(num_bins)
+        near_sensor_binned = np.zeros(num_bins)
+
+        # Bin both datasets
+        for bin_idx in range(num_bins):
+            if bin_idx == num_bins - 1:
+                start_idx = int(bins[bin_idx])
+                end_idx = exp_len
+            else:
+                start_idx = int(bins[bin_idx])
+                end_idx = int(bins[bin_idx + 1])
+
+            missing_binned[bin_idx] = np.mean(missing_pct[start_idx:end_idx])
+            near_sensor_binned[bin_idx] = np.mean(near_sensor_pct[start_idx:end_idx])
+
+        # Plot both metrics with same color but different line styles
+        color = experiment_colors[i]
+        ax1.plot(
+            range(num_bins),
+            missing_binned,
+            color=color,
+            linestyle="-",
+            alpha=0.6,
+            label=exp_name,
+        )
+        ax2.plot(
+            range(num_bins), near_sensor_binned, color=color, linestyle="--", alpha=0.6
+        )
+
+        # # Add vertical lines for manually labeled frames if available
+        # if all_paths[i].with_suffix(".npy").name in manually_labeled_anomaly_frames:
+        #     begin_frame, end_frame = manually_labeled_anomaly_frames[
+        #         all_paths[i].with_suffix(".npy").name
+        #     ]
+        #     # Convert frame numbers to normalized timeline positions
+        #     begin_pos = (begin_frame / exp_len) * (num_bins - 1)
+        #     end_pos = (end_frame / exp_len) * (num_bins - 1)
+
+        #     # Add vertical lines with matching color and loose dotting
+        #     ax1.axvline(
+        #         x=begin_pos,
+        #         color=color,
+        #         linestyle=":",
+        #         alpha=0.6,
+        #     )
+        #     ax1.axvline(
+        #         x=end_pos,
+        #         color=color,
+        #         linestyle=":",
+        #         alpha=0.6,
+        #     )
+
+    # Customize axes
+    ax1.set_xlabel("Normalized Timeline")
+    ax1.set_xticks(np.linspace(0, num_bins - 1, 5))
+    ax1.set_xticklabels([f"{x:.0f}%" for x in np.linspace(0, 100, 5)])
+
+    ax1.set_ylabel("Missing Points (%)")
+    ax2.set_ylabel("Points with <0.5m Range (%)")
+
+    # plt.title(title)
+
+    # Create legends without fixed positions
+    # First get all lines and labels for experiments
+    lines1, labels1 = ax1.get_legend_handles_labels()
+
+    # Combine both legends into one
+    all_handles = (
+        lines1
+        + [Line2D([0], [0], color="gray", linestyle="-", label="", alpha=0)]
+        + metric_legend
+    )
+    all_labels = (
+        labels1
+        + [""]
+        + ["Missing Points", "Points Near Sensor (<0.5m)", "Manually Labeled Borders"]
+    )
+
+    # Create single legend in top right corner with consistent margins
+    # fig.legend(all_handles, all_labels, loc="upper right", borderaxespad=2.8)
+    fig.legend(all_handles, all_labels, bbox_to_anchor=(0.95, 0.99))
+
+    plt.grid(True, alpha=0.3)
+
+    # Save figure letting matplotlib handle the layout
+    plt.savefig(output_datetime_path / "combined_anomalies_timeline.png", dpi=150)
+    plt.close()
+
+
+# Generate the combined timeline plot
+plot_combined_timeline(
+    normal_experiment_paths,
+    anomaly_experiment_paths,
+    "Lidar Degradation Indicators Throughout Experiments\n(Baseline and Moving Anomaly Experiments)",
+)
+
+# delete current latest folder
+shutil.rmtree(latest_folder_path, ignore_errors=True)
+
+# create new latest folder
+latest_folder_path.mkdir(exist_ok=True, parents=True)
+
+# copy contents of output folder to the latest folder
+for file in output_datetime_path.iterdir():
+    shutil.copy2(file, latest_folder_path)
+
+# copy this python script to preserve the code used
+shutil.copy2(__file__, output_datetime_path)
+shutil.copy2(__file__, latest_folder_path)
+
+# move output date folder to archive
+shutil.move(output_datetime_path, archive_folder_path)

tools/plot_scripts/data_count_lidar_frames.py

@@ -122,8 +122,8 @@ def plot_data_points_pie(normal_experiment_frames, anomaly_experiment_frames):
 
     # prepare data for pie chart
     labels = [
-        "Normal Lidar Frames\nNon-Degraded Pointclouds",
-        "Anomalous Lidar Frames\nDegraded Pointclouds",
+        "Normal Lidar Frames\nNon-Degraded Point Clouds",
+        "Anomalous Lidar Frames\nDegraded Point Clouds",
     ]
     sizes = [total_normal_frames, total_anomaly_frames]
     explode = (0.1, 0)  # explode the normal slice
@@ -150,9 +150,9 @@ def plot_data_points_pie(normal_experiment_frames, anomaly_experiment_frames):
         va="center",
         color="black",
     )
-    plt.title(
-        "Distribution of Normal and Anomalous\nPointclouds in all Experiments (Lidar Frames)"
-    )
+    # plt.title(
+    #     "Distribution of Normal and Anomalous\nPointclouds in all Experiments (Lidar Frames)"
+    # )
     plt.tight_layout()
 
     # save the plot

tools/plot_scripts/data_icp_rsme.py

@@ -0,0 +1,178 @@
+import pickle
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+import open3d as o3d
+from pointcloudset import Dataset
+from pointcloudset.io.pointcloud.open3d import to_open3d
+from rich.progress import track
+
+# define data path containing the bag files
+all_data_path = Path("/home/fedex/mt/data/subter")
+
+output_path = Path("/home/fedex/mt/plots/data_icp_rmse")
+datetime_folder_name = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+
+latest_folder_path = output_path / "latest"
+archive_folder_path = output_path / "archive"
+output_datetime_path = output_path / datetime_folder_name
+
+# if output does not exist, create it
+output_path.mkdir(exist_ok=True, parents=True)
+output_datetime_path.mkdir(exist_ok=True, parents=True)
+latest_folder_path.mkdir(exist_ok=True, parents=True)
+archive_folder_path.mkdir(exist_ok=True, parents=True)
+
+# Add cache folder
+cache_folder = output_path / "cache"
+cache_folder.mkdir(exist_ok=True, parents=True)
+
+
+def get_cache_path(experiment_path: Path) -> Path:
+    """Convert experiment bag path to cache pkl path"""
+    return cache_folder / f"{experiment_path.stem}.pkl"
+
+
+# find all bag files and sort them correctly by name (experiments with smoke in the name are anomalies)
+normal_experiment_paths, anomaly_experiment_paths = [], []
+for bag_file_path in all_data_path.iterdir():
+    if bag_file_path.suffix != ".bag":
+        continue
+    if "smoke" in bag_file_path.name:
+        anomaly_experiment_paths.append(bag_file_path)
+    else:
+        normal_experiment_paths.append(bag_file_path)
+
+# sort anomaly and normal experiments by filesize, ascending
+anomaly_experiment_paths.sort(key=lambda path: path.stat().st_size)
+normal_experiment_paths.sort(key=lambda path: path.stat().st_size)
+
+
+def calculate_icp_rmse(experiment_path):
+    """Calculate ICP RMSE between consecutive frames in an experiment"""
+    cache_path = get_cache_path(experiment_path)
+
+    # Check cache first
+    if cache_path.exists():
+        with open(cache_path, "rb") as file:
+            return pickle.load(file)
+
+    dataset = Dataset.from_file(experiment_path, topic="/ouster/points")
+    rmse_values = []
+
+    # Convert iterator to list for progress tracking
+    pointclouds = list(dataset)
+
+    # Get consecutive pairs of point clouds with progress bar
+    for i in track(
+        range(len(pointclouds) - 1),
+        description=f"Processing {experiment_path.name}",
+        total=len(pointclouds) - 1,
+    ):
+        # Get consecutive point clouds and convert to Open3D format
+        source = to_open3d(pointclouds[i])
+        target = to_open3d(pointclouds[i + 1])
+
+        # Apply point-to-point ICP (much faster than point-to-plane)
+        result = o3d.pipelines.registration.registration_icp(
+            source,
+            target,
+            max_correspondence_distance=0.2,  # 20cm max correspondence
+            init=np.identity(4),  # Identity matrix as initial transformation
+            estimation_method=o3d.pipelines.registration.TransformationEstimationPointToPoint(),
+            criteria=o3d.pipelines.registration.ICPConvergenceCriteria(
+                max_iteration=50
+            ),
+        )
+
+        rmse_values.append(result.inlier_rmse)
+
+    # Cache the results
+    with open(cache_path, "wb") as file:
+        pickle.dump(rmse_values, file, protocol=pickle.HIGHEST_PROTOCOL)
+
+    return rmse_values
+
+
+def plot_statistical_comparison(normal_paths, anomaly_paths, output_path):
+    """Create statistical comparison plots between normal and anomalous experiments"""
+    # Collect all RMSE values
+    normal_rmse = []
+    anomaly_rmse = []
+
+    print("Loading cached RMSE values...")
+    for path in normal_paths:
+        normal_rmse.extend(calculate_icp_rmse(path))
+    for path in anomaly_paths:
+        anomaly_rmse.extend(calculate_icp_rmse(path))
+
+    # Convert to numpy arrays
+    normal_rmse = np.array(normal_rmse)
+    anomaly_rmse = np.array(anomaly_rmse)
+
+    # Create figure with two subplots side by sidnte
+    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))
+
+    # Box plot
+    data = [normal_rmse, anomaly_rmse]
+    ax1.boxplot(data, labels=["Normal", "Anomalous"])
+    ax1.set_ylabel("ICP RMSE (meters)")
+    ax1.set_title("Box Plot Comparison")
+    ax1.grid(True, alpha=0.3)
+
+    # Violin plot
+    ax2.violinplot(data)
+    ax2.set_xticks([1, 2])
+    ax2.set_xticklabels(["Normal", "Anomalous"])
+    ax2.set_ylabel("ICP RMSE (meters)")
+    ax2.set_title("Violin Plot Comparison")
+    ax2.grid(True, alpha=0.3)
+
+    plt.suptitle(
+        "ICP RMSE Statistical Comparison: Normal vs Anomalous Experiments", fontsize=14
+    )
+    plt.tight_layout()
+
+    # Save both linear and log scale versions
+    plt.savefig(output_path / "icp_rmse_distribution_linear.png", dpi=150)
+
+    # Log scale version
+    ax1.set_yscale("log")
+    ax2.set_yscale("log")
+    plt.savefig(output_path / "icp_rmse_distribution_log.png", dpi=150)
+    plt.close()
+
+    # Print some basic statistics
+    print("\nBasic Statistics:")
+    print(
+        f"Normal experiments - mean: {np.mean(normal_rmse):.4f}, std: {np.std(normal_rmse):.4f}"
+    )
+    print(
+        f"Anomaly experiments - mean: {np.mean(anomaly_rmse):.4f}, std: {np.std(anomaly_rmse):.4f}"
+    )
+
+
+# Replace all plotting code with just this:
+plot_statistical_comparison(
+    normal_experiment_paths, anomaly_experiment_paths, output_datetime_path
+)
+
+# delete current latest folder
+shutil.rmtree(latest_folder_path, ignore_errors=True)
+
+# create new latest folder
+latest_folder_path.mkdir(exist_ok=True, parents=True)
+
+# copy contents of output folder to the latest folder
+for file in output_datetime_path.iterdir():
+    shutil.copy2(file, latest_folder_path)
+
+# copy this python script to preserve the code used
+shutil.copy2(__file__, output_datetime_path)
+shutil.copy2(__file__, latest_folder_path)
+
+# move output date folder to archive
+shutil.move(output_datetime_path, archive_folder_path)

tools/plot_scripts/data_missing_points.py

@@ -5,7 +5,6 @@ from pathlib import Path
 
 import matplotlib.pyplot as plt
 import numpy as np
-from pointcloudset import Dataset
 
 # define data path containing the bag files
 all_data_path = Path("/home/fedex/mt/data/subter")
@@ -82,7 +81,7 @@ def plot_data_points(normal_experiment_paths, anomaly_experiment_paths, title):
     plt.figure(figsize=(10, 5))
     plt.hist(missing_points_normal, bins=100, alpha=0.5, label="Normal Experiments")
     plt.hist(missing_points_anomaly, bins=100, alpha=0.5, label="Anomaly Experiments")
-    plt.title(title)
+    # plt.title(title)
     plt.xlabel("Number of Missing Points")
     plt.ylabel("Number of Pointclouds")
     plt.legend()
@@ -109,7 +108,7 @@ def plot_data_points(normal_experiment_paths, anomaly_experiment_paths, title):
         label="Anomaly Experiments",
         orientation="horizontal",
     )
-    plt.title(title)
+    # plt.title(title)
     plt.xlabel("Number of Pointclouds")
     plt.ylabel("Number of Missing Points")
     plt.legend()
@@ -142,7 +141,7 @@ def plot_data_points(normal_experiment_paths, anomaly_experiment_paths, title):
         label="Anomaly Experiments",
         density=True,
     )
-    plt.title(title)
+    # plt.title(title)
     plt.xlabel("Number of Missing Points")
     plt.ylabel("Density")
     plt.legend()
@@ -150,7 +149,7 @@ def plot_data_points(normal_experiment_paths, anomaly_experiment_paths, title):
     plt.savefig(output_datetime_path / "missing_points_density.png")
 
     # create another density version which does not plot number of missing points but percentage of measurements that are missing (total number of points is 32*2048)
-    bins = np.linspace(0, 1, 100)
+    bins = np.linspace(0, 0.6, 100)
     plt.clf()
     plt.figure(figsize=(10, 5))
     plt.hist(
@@ -169,7 +168,7 @@ def plot_data_points(normal_experiment_paths, anomaly_experiment_paths, title):
         label="Anomaly Experiments (With Artifical Smoke)",
         density=True,
     )
-    plt.title(title)
+    # plt.title(title)
     plt.xlabel("Percentage of Missing Lidar Measurements")
     plt.ylabel("Density")
     # display the x axis as percentages
@@ -210,7 +209,7 @@ def plot_data_points(normal_experiment_paths, anomaly_experiment_paths, title):
         alpha=0.5,
         label="Anomaly Experiments",
     )
-    plt.title(title)
+    # plt.title(title)
     plt.xlabel("Number of Missing Points")
     plt.ylabel("Normalized Density")
     plt.legend()

tools/plot_scripts/data_missing_points_anomalies.py

@@ -0,0 +1,134 @@
+import pickle
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+from pointcloudset import Dataset
+
+# define data paths
+all_data_path = Path("/home/fedex/mt/data/subter")
+output_path = Path("/home/fedex/mt/plots/data_missing_points_anomalies")
+datetime_folder_name = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+
+latest_folder_path = output_path / "latest"
+archive_folder_path = output_path / "archive"
+output_datetime_path = output_path / datetime_folder_name
+
+# if output does not exist, create it
+output_path.mkdir(exist_ok=True, parents=True)
+output_datetime_path.mkdir(exist_ok=True, parents=True)
+latest_folder_path.mkdir(exist_ok=True, parents=True)
+archive_folder_path.mkdir(exist_ok=True, parents=True)
+
+data_resolution = 32 * 2048
+
+# find all bag files and sort them correctly by name
+normal_experiment_paths, anomaly_experiment_paths = [], []
+for bag_file_path in all_data_path.iterdir():
+    if bag_file_path.suffix != ".bag":
+        continue
+    if "smoke" in bag_file_path.name:
+        anomaly_experiment_paths.append(bag_file_path)
+    else:
+        normal_experiment_paths.append(bag_file_path)
+
+
+def plot_timeline_comparison(anomaly_experiment_paths, title, num_bins=50):
+    """Plot missing points percentage over normalized timeline for moving anomaly experiments"""
+    # Sort experiments by filesize first (to match original processing order)
+    anomaly_experiment_paths = sorted(
+        anomaly_experiment_paths, key=lambda path: path.stat().st_size
+    )
+
+    # Filter out stationary experiments
+    moving_exp_indices = [
+        i
+        for i, path in enumerate(anomaly_experiment_paths)
+        if "stationary" not in path.name
+    ]
+    moving_anomaly_paths = [anomaly_experiment_paths[i] for i in moving_exp_indices]
+
+    # Try to load cached data from original script's location
+    cache_path = Path("/home/fedex/mt/plots/data_missing_points/missing_points.pkl")
+    if not cache_path.exists():
+        print("No cached data found. Please run the original script first.")
+        return
+
+    with open(cache_path, "rb") as file:
+        _, missing_points_anomaly = pickle.load(file)
+
+    # Get data for moving experiments only (using original indices)
+    moving_anomaly_data = [missing_points_anomaly[i] for i in moving_exp_indices]
+
+    # Create figure
+    plt.figure(figsize=(12, 6))
+
+    # Plot each experiment's timeline
+    for i, exp_data in enumerate(moving_anomaly_data):
+        # Convert to percentage
+        percentages = np.array(exp_data) / data_resolution * 100
+
+        # Create normalized timeline bins
+        exp_len = len(percentages)
+        bins = np.linspace(0, exp_len - 1, num_bins)
+        binned_data = np.zeros(num_bins)
+
+        # Bin the data
+        for bin_idx in range(num_bins):
+            if bin_idx == num_bins - 1:
+                start_idx = int(bins[bin_idx])
+                end_idx = exp_len
+            else:
+                start_idx = int(bins[bin_idx])
+                end_idx = int(bins[bin_idx + 1])
+
+            binned_data[bin_idx] = np.mean(percentages[start_idx:end_idx])
+
+        # Plot with slight transparency to show overlaps
+        plt.plot(
+            range(num_bins),
+            binned_data,
+            alpha=0.6,
+            label=f"Experiment {moving_anomaly_paths[i].stem}",
+        )
+
+    plt.title(title)
+    plt.xlabel("Normalized Timeline")
+    # Add percentage ticks on x-axis
+    plt.xticks(
+        np.linspace(0, num_bins - 1, 5), [f"{x:.0f}%" for x in np.linspace(0, 100, 5)]
+    )
+    plt.ylabel("Missing Points (%)")
+    plt.grid(True, alpha=0.3)
+    plt.legend()
+    plt.tight_layout()
+
+    # Save the plot
+    plt.savefig(output_datetime_path / "missing_points_timeline.png", dpi=150)
+    plt.close()
+
+
+# Generate the timeline comparison plot
+plot_timeline_comparison(
+    anomaly_experiment_paths,
+    "Missing Lidar Measurements Over Time\n(Moving Anomaly Experiments Only)",
+)
+
+# delete current latest folder
+shutil.rmtree(latest_folder_path, ignore_errors=True)
+
+# create new latest folder
+latest_folder_path.mkdir(exist_ok=True, parents=True)
+
+# copy contents of output folder to the latest folder
+for file in output_datetime_path.iterdir():
+    shutil.copy2(file, latest_folder_path)
+
+# copy this python script to preserve the code used
+shutil.copy2(__file__, output_datetime_path)
+shutil.copy2(__file__, latest_folder_path)
+
+# move output date folder to archive
+shutil.move(output_datetime_path, archive_folder_path)

tools/plot_scripts/data_particles_near_sensor.py

@@ -5,7 +5,6 @@ from pathlib import Path
 
 import matplotlib.pyplot as plt
 import numpy as np
-from pointcloudset import Dataset
 
 # define data path containing the bag files
 all_data_path = Path("/home/fedex/mt/data/subter")
@@ -164,7 +163,7 @@ def plot_data_points(normal_experiment_paths, anomaly_experiment_paths, title):
         plt.gca().set_yticklabels(
             ["{:.0f}%".format(y * 100) for y in plt.gca().get_yticks()]
         )
-        plt.title("Particles Closer than 0.5m to the Sensor")
+        # plt.title("Particles Closer than 0.5m to the Sensor")
         plt.ylabel("Percentage of measurements closer than 0.5m")
         plt.tight_layout()
         plt.savefig(output_datetime_path / f"particles_near_sensor_boxplot_{rt}.png")
@@ -186,7 +185,7 @@ def plot_data_points(normal_experiment_paths, anomaly_experiment_paths, title):
         plt.gca().set_yticklabels(
             ["{:.0f}%".format(y * 100) for y in plt.gca().get_yticks()]
         )
-        plt.title("Particles Closer than 0.5m to the Sensor")
+        # plt.title("Particles Closer than 0.5m to the Sensor")
         plt.ylabel("Percentage of measurements closer than 0.5m")
         plt.ylim(0, 0.05)
         plt.tight_layout()

tools/plot_scripts/data_particles_near_sensor_anomalies.py

@@ -0,0 +1,149 @@
+import pickle
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+from pointcloudset import Dataset
+
+# define data paths
+all_data_path = Path("/home/fedex/mt/data/subter")
+output_path = Path("/home/fedex/mt/plots/data_particles_near_sensor_anomalies")
+datetime_folder_name = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+
+latest_folder_path = output_path / "latest"
+archive_folder_path = output_path / "archive"
+output_datetime_path = output_path / datetime_folder_name
+
+# if output does not exist, create it
+output_path.mkdir(exist_ok=True, parents=True)
+output_datetime_path.mkdir(exist_ok=True, parents=True)
+latest_folder_path.mkdir(exist_ok=True, parents=True)
+archive_folder_path.mkdir(exist_ok=True, parents=True)
+
+data_resolution = 32 * 2048
+
+# find all bag files and sort them correctly by name
+normal_experiment_paths, anomaly_experiment_paths = [], []
+for bag_file_path in all_data_path.iterdir():
+    if bag_file_path.suffix != ".bag":
+        continue
+    if "smoke" in bag_file_path.name:
+        anomaly_experiment_paths.append(bag_file_path)
+    else:
+        normal_experiment_paths.append(bag_file_path)
+
+
+def plot_timeline_comparison(
+    anomaly_experiment_paths, title, range_threshold, num_bins=50
+):
+    """Plot near-sensor measurements percentage over normalized timeline for moving anomaly experiments"""
+    # Sort experiments by filesize first (to match original processing order)
+    anomaly_experiment_paths = sorted(
+        anomaly_experiment_paths, key=lambda path: path.stat().st_size
+    )
+
+    # Filter out stationary experiments
+    moving_exp_indices = [
+        i
+        for i, path in enumerate(anomaly_experiment_paths)
+        if "stationary" not in path.name
+    ]
+    moving_anomaly_paths = [anomaly_experiment_paths[i] for i in moving_exp_indices]
+
+    # Try to load cached data
+    cache_path = (
+        Path("/home/fedex/mt/plots/data_particles_near_sensor")
+        / f"particles_near_sensor_counts_{range_threshold}.pkl"
+    )
+    if not cache_path.exists():
+        print(
+            f"No cached data found for range threshold {range_threshold}. Please run the original script first."
+        )
+        return
+
+    with open(cache_path, "rb") as file:
+        _, particles_near_sensor_anomaly = pickle.load(file)
+
+    # Get data for moving experiments only (using original indices)
+    moving_anomaly_data = [particles_near_sensor_anomaly[i] for i in moving_exp_indices]
+
+    # Create figure
+    plt.figure(figsize=(12, 6))
+
+    # Plot each experiment's timeline
+    for i, exp_data in enumerate(moving_anomaly_data):
+        # Convert to percentage
+        percentages = np.array(exp_data) / data_resolution * 100
+
+        # Create normalized timeline bins
+        exp_len = len(percentages)
+        bins = np.linspace(0, exp_len - 1, num_bins)
+        binned_data = np.zeros(num_bins)
+
+        # Bin the data
+        for bin_idx in range(num_bins):
+            if bin_idx == num_bins - 1:
+                start_idx = int(bins[bin_idx])
+                end_idx = exp_len
+            else:
+                start_idx = int(bins[bin_idx])
+                end_idx = int(bins[bin_idx + 1])
+
+            binned_data[bin_idx] = np.mean(percentages[start_idx:end_idx])
+
+        # Plot with slight transparency to show overlaps
+        plt.plot(
+            range(num_bins),
+            binned_data,
+            alpha=0.6,
+            label=f"Experiment {moving_anomaly_paths[i].stem}",
+        )
+
+    plt.title(f"{title}\n(Range Threshold: {range_threshold / 1000:.1f}m)")
+    plt.xlabel("Normalized Timeline")
+    # Add percentage ticks on x-axis
+    plt.xticks(
+        np.linspace(0, num_bins - 1, 5), [f"{x:.0f}%" for x in np.linspace(0, 100, 5)]
+    )
+    plt.ylabel("Near-Sensor Measurements (%)")
+    plt.grid(True, alpha=0.3)
+    plt.legend()
+    plt.tight_layout()
+
+    # Save the plot
+    plt.savefig(
+        output_datetime_path
+        / f"near_sensor_measurements_timeline_{range_threshold}.png",
+        dpi=150,
+    )
+    plt.close()
+
+
+# Generate timeline comparison plots for each range threshold
+range_thresholds = [500, 750, 1000, 1250, 1500]
+for rt in range_thresholds:
+    print(f"Processing range threshold {rt}...")
+    plot_timeline_comparison(
+        anomaly_experiment_paths,
+        "Near-Sensor Lidar Measurements Over Time\n(Moving Anomaly Experiments Only)",
+        rt,
+    )
+
+# delete current latest folder
+shutil.rmtree(latest_folder_path, ignore_errors=True)
+
+# create new latest folder
+latest_folder_path.mkdir(exist_ok=True, parents=True)
+
+# copy contents of output folder to the latest folder
+for file in output_datetime_path.iterdir():
+    shutil.copy2(file, latest_folder_path)
+
+# copy this python script to preserve the code used
+shutil.copy2(__file__, output_datetime_path)
+shutil.copy2(__file__, latest_folder_path)
+
+# move output date folder to archive
+shutil.move(output_datetime_path, archive_folder_path)

tools/plot_scripts/data_spherical_projection.py

@@ -47,16 +47,14 @@ parser.add_argument(
     "--input1",
     type=Path,
     default=Path(
-        "/home/fedex/mt/data/subter/new_projection/1_loop_closure_illuminated_2023-01-23.npy"
+        "/home/fedex/mt/data/subter/1_loop_closure_illuminated_2023-01-23.npy"
     ),
     help="Path to first .npy file containing 2D projection data",
 )
 parser.add_argument(
     "--input2",
     type=Path,
-    default=Path(
-        "/home/fedex/mt/data/subter/new_projection/3_smoke_human_walking_2023-01-23.npy"
-    ),
+    default=Path("/home/fedex/mt/data/subter/3_smoke_human_walking_2023-01-23.npy"),
     help="Path to second .npy file containing 2D projection data",
 )
 parser.add_argument(
@@ -114,25 +112,34 @@ cmap = get_colormap_with_special_missing_color(
     args.colormap, args.missing_data_color, args.reverse_colormap
 )
 
-# --- Create a figure with 2 vertical subplots ---
+# --- Create a figure with 2 vertical subplots and move titles to the left ---
 fig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, figsize=(10, 5))
-for ax, frame, title in zip(
+# leave extra left margin for the left-side labels
+fig.subplots_adjust(left=0.14, hspace=0.05)
+
+for ax, frame, label in zip(
     (ax1, ax2),
     (frame1, frame2),
-    (
-        "Projection of Lidar Frame without Degradation",
-        "Projection of Lidar Frame with Degradation (Artifical Smoke)",
-    ),
+    ("(a)", "(b)"),
 ):
     im = ax.imshow(frame, cmap=cmap, aspect="auto", vmin=global_vmin, vmax=global_vmax)
-    ax.set_title(title)
+    # place the "title" to the left, vertically centered relative to the axes
+    ax.text(
+        -0.02,  # negative x places text left of the axes (in axes coordinates)
+        0.5,
+        label,
+        transform=ax.transAxes,
+        va="center",
+        ha="right",
+        fontsize=12,
+    )
     ax.axis("off")
 
 # Adjust layout to fit margins for a paper
 plt.tight_layout(rect=[0, 0.05, 1, 1])
 # Add a colorbar with the colormap below the subplots
 cbar = fig.colorbar(im, ax=[ax1, ax2], orientation="vertical", fraction=0.05)
-cbar.set_label("Normalized Range")
+cbar.set_label("Reciprocal Range")
 
 # Add a separate colorbar for NaN values
 sm = ScalarMappable(cmap=ListedColormap([cmap.get_bad(), cmap.get_over()]))