code: complete eval pipeline (7 metrics + per-class + Wilcoxon) + Swin-UNet/TransUNet networks; remove backups/obsolete

code/envs/seggen-controlnet.yml

@@ -0,0 +1,18 @@
+# Isolated env for ControlNet (pytorch-lightning 1.5 + old torch). Needs SD v1.5 ckpt.
+# Generative-augmentation baseline only; never mix with the main env.
+name: seggen-controlnet
+channels: [conda-forge]
+dependencies:
+  - python=3.10
+  - pip
+  - pip:
+      # torch 1.12.1 (cu113 wheels bundle their own runtime):
+      #   pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 --extra-index-url https://download.pytorch.org/whl/cu113
+      - pytorch-lightning==1.5.0
+      - omegaconf==2.1.1
+      - einops==0.3.0
+      - transformers==4.19.2
+      - open-clip-torch==2.0.2
+      - gradio==3.16.2
+      - opencv-python-headless
+      - basicsr==1.4.2

code/envs/umamba.yml

@@ -1,16 +1,25 @@
-# Isolated env for U-Mamba reference baseline. mamba_ssm needs compilation; the repo
-# targets CUDA 11.8 + torch 2.0.1. Compilation on newer stacks may fail — pin to 11.8.
-# AMP can NaN in Mamba layers: use the NoAMP trainer variant.
+# Isolated env for U-Mamba reference baseline (nnU-Net v2.1.1 + Mamba).
+# VALIDATED on the A100 server (2026-06-05): mamba CUDA kernel + UMambaBot trainer
+# run end-to-end. mamba_ssm/causal-conv1d are installed from PREBUILT WHEELS to
+# avoid local CUDA compilation (system nvcc 12.8 != torch cu118).
+#
+# Reproduce with these exact commands (NOT `conda env create -f`; pip-driven):
+#
+#   conda create -n umamba python=3.10 -y && conda activate umamba
+#   pip install torch==2.0.1 torchvision==0.15.2 --index-url https://download.pytorch.org/whl/cu118
+#   pip install https://github.com/Dao-AILab/causal-conv1d/releases/download/v1.2.0.post2/causal_conv1d-1.2.0.post2+cu118torch2.0cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
+#   pip install https://github.com/state-spaces/mamba/releases/download/v1.2.0.post1/mamba_ssm-1.2.0.post1+cu118torch2.0cxx11abiFALSE-cp310-cp310-linux_x86_64.whl
+#   pip install -e sota/U-Mamba/umamba
+#   # critical pins (otherwise: transformers drops GreedySearchDecoderOnlyOutput; opencv/numpy clash):
+#   pip install "numpy<2" "transformers==4.38.2" "opencv-python-headless==4.9.0.80"
+#
+# Train (uses nnU-Net format from framework/nnunet_convert.py; A100 only — bf16/sm_80):
+#   export CUDA_VISIBLE_DEVICES=4
+#   nnUNetv2_plan_and_preprocess -d <ID> -c 2d        # umamba env does its OWN 2.1.1 preprocess
+#   cp <raw>/Dataset<ID>_*/splits_final.json $nnUNet_preprocessed/Dataset<ID>_*/splits_final.json
+#   nnUNetv2_train <ID> 2d 0 -tr nnUNetTrainerUMambaBot
+#   # AMP can NaN in Mamba: if so use -tr nnUNetTrainerUMambaEncNoAMP
 name: umamba
 channels: [conda-forge]
 dependencies:
   - python=3.10
-  - pip
-  - pip:
-      # pip install torch==2.0.1 torchvision==0.15.2 --index-url https://download.pytorch.org/whl/cu118
-      # then (in order):
-      #   pip install causal-conv1d>=1.2.0
-      #   pip install mamba-ssm --no-cache-dir
-      #   cd sota/U-Mamba/umamba && pip install -e .
-      - simpleitk
-      - nibabel

code/framework/common/subset.py

@@ -1,40 +0,0 @@
-"""Deterministic low-data subset selection.
-
-The SAME function is used by (a) the segmentation trainer's real-data loader and
-(b) the pixel-diffusion generator's data loader, so that a low-data experiment at
-fraction f trains the generator on EXACTLY the real images the segmenter sees —
-no leakage, no mismatch, fully reproducible.
-
-Selection is driven only by (fraction, fraction_seed) and the SORTED list of
-items, so it is independent of the training seed: the 3 segmentation seeds all
-use the identical real subset, only their weight init differs.
-"""
-from __future__ import annotations
-
-import math
-import random
-from typing import List, Sequence, TypeVar
-
-T = TypeVar("T")
-
-
-def select_fraction(items: Sequence[T], fraction: float, seed: int = 0) -> List[T]:
-    """Return a deterministic subset of `items` of size ceil(fraction*N).
-
-    items: any sequence (e.g. list of (image_path, mask_path) tuples). It is
-           sorted by repr() first so order of discovery never affects the subset.
-    fraction: in (0, 1]. >=1 returns all items (sorted copy).
-    seed: subset seed (NOT the training seed). Fixed across training seeds.
-    """
-    ordered = sorted(items, key=lambda x: repr(x))
-    n = len(ordered)
-    if fraction >= 1.0 or n == 0:
-        return ordered
-    if fraction <= 0.0:
-        raise ValueError(f"fraction must be in (0,1], got {fraction}")
-    k = max(1, math.ceil(fraction * n))
-    rng = random.Random(seed)
-    idx = list(range(n))
-    rng.shuffle(idx)
-    keep = sorted(idx[:k])
-    return [ordered[i] for i in keep]

code/framework/config.py

@@ -33,13 +33,6 @@ class Config:
     # Points at a dir laid out like a split: <synth_train_dir>/{images,masks}/.
     synth_train_dir: str = ""                 # "" = real data only (no generative augmentation)
 
-    # low-data regime: train on a deterministic fraction of the REAL train split.
-    # The SAME subset is used by the PixDiff generator (framework.common.subset), so
-    # real and real+synth arms never disagree. fraction_seed is FIXED across the 3
-    # training seeds (only weight init varies), so the data subset stays constant.
-    train_fraction: float = 1.0               # 1.0 = full data; e.g. 0.1 = 10%
-    fraction_seed: int = 0                    # subset seed (independent of `seed`)
-
     # ---- augmentation (conventional baseline tier) ----
     aug: str = "standard"                     # none | standard | strong  (albumentations online)
     aug_backend: str = "albumentations"       # albumentations | monai

code/framework/config.py.bak

@@ -1,109 +0,0 @@
-"""Unified experiment configuration.
-
-A single dataclass drives every run. Values can come from (in priority order):
-  1. command-line flags (argparse)  2. a YAML file (--config)  3. dataclass defaults.
-
-The same config object is used by train.py / test.py so that a training run and
-its evaluation are guaranteed to agree on dataset, model, image size, etc.
-"""
-from __future__ import annotations
-
-import argparse
-import dataclasses
-from dataclasses import dataclass, field, asdict
-from typing import Optional, List
-
-import yaml
-
-
-@dataclass
-class Config:
-    # ---- experiment identity ----
-    exp_name: str = "default"                 # results/<exp_name>/<dataset>/<arch>/seed<seed>/
-    seed: int = 0
-
-    # ---- data ----
-    data_root: str = "dataset/processed_unified"
-    dataset: str = "cvc_clinicdb"             # folder name under data_root
-    protocol: str = "official"                # e.g. official / fold01 ...
-    in_channels: int = 0                      # 0 = auto-detect from metadata/first image
-    num_classes: int = 0                      # 0 = auto-detect from metadata/masks (incl. background)
-    img_size: int = 256                       # square resize target (Swin/TransUNet need 224)
-    # extra synthetic (image,mask) pairs to MERGE into the train split.
-    # Points at a dir laid out like a split: <synth_train_dir>/{images,masks}/.
-    synth_train_dir: str = ""                 # "" = real data only (no generative augmentation)
-
-    # ---- augmentation (conventional baseline tier) ----
-    aug: str = "standard"                     # none | standard | strong  (albumentations online)
-    aug_backend: str = "albumentations"       # albumentations | monai
-    normalize: str = "auto"                   # auto(imagenet for RGB, 0.5 for gray) | imagenet | none
-
-    # ---- model ----
-    arch: str = "unet"                        # see models/registry.py REGISTRY
-    encoder: str = "resnet34"                 # SMP encoder name (ignored by non-SMP archs)
-    encoder_weights: str = "imagenet"         # imagenet | none
-    pretrained_ckpt: str = ""                 # ViT/Swin pretrain for transunet/swinunet (optional)
-
-    # ---- optimization ----
-    epochs: int = 100
-    batch_size: int = 16                      # per-GPU batch size
-    lr: float = 1e-4
-    weight_decay: float = 1e-4
-    optimizer: str = "adamw"                  # adamw | sgd
-    scheduler: str = "poly"                   # poly | cosine | none
-    warmup_epochs: int = 0
-    loss: str = "ce_dice"                     # ce_dice | ce | dice
-    num_workers: int = 8
-    grad_clip: float = 0.0                    # 0 = disabled
-
-    # ---- precision / hardware ----
-    amp: str = "bf16"                         # bf16(A100+) | fp16(V100) | fp32
-    # DDP is driven by torchrun env vars (RANK/WORLD_SIZE/LOCAL_RANK); nothing to set here.
-
-    # ---- evaluation / logging ----
-    val_interval: int = 5                     # epochs between validations
-    min_epochs: int = 0                       # never early-stop before this many epochs
-    patience: int = 0                          # early-stop after this many epochs w/o val improvement (0 = off)
-    save_interval: int = 0                    # 0 = only save best + last
-    include_background: bool = False          # include class 0 in reported Dice/IoU
-    compute_hd95: bool = True
-    out_root: str = "results"
-    resume: str = ""                          # path to checkpoint to resume from
-    visualize: bool = True                    # save overlays at test time
-    vis_max: int = 32                         # max number of overlay images to save
-
-    def out_dir(self) -> str:
-        return f"{self.out_root}/{self.exp_name}/{self.dataset}_{self.protocol}/{self.arch}/seed{self.seed}"
-
-    def to_yaml(self, path: str) -> None:
-        with open(path, "w") as f:
-            yaml.safe_dump(asdict(self), f, sort_keys=False, allow_unicode=True)
-
-    @classmethod
-    def from_args(cls, argv: Optional[List[str]] = None) -> "Config":
-        # First pass: only grab --config so YAML can set defaults that flags then override.
-        pre = argparse.ArgumentParser(add_help=False)
-        pre.add_argument("--config", type=str, default="")
-        known, _ = pre.parse_known_args(argv)
-
-        base = cls()
-        if known.config:
-            with open(known.config) as f:
-                ydata = yaml.safe_load(f) or {}
-            base = dataclasses.replace(base, **{k: v for k, v in ydata.items()
-                                                if k in {f.name for f in dataclasses.fields(cls)}})
-
-        p = argparse.ArgumentParser(parents=[pre],
-                                    description="SegGen unified segmentation framework")
-        for f in dataclasses.fields(cls):
-            default = getattr(base, f.name)
-            if f.type is bool or isinstance(default, bool):
-                # support --flag / --no-flag
-                p.add_argument(f"--{f.name}", dest=f.name, action="store_true", default=default)
-                p.add_argument(f"--no-{f.name}", dest=f.name, action="store_false")
-            else:
-                p.add_argument(f"--{f.name}", type=type(default) if default is not None else str,
-                               default=default)
-        ns = p.parse_args(argv)
-        kwargs = {f.name: getattr(ns, f.name) for f in dataclasses.fields(cls)}
-        return cls(**kwargs)

code/framework/data/loaders.py

@@ -17,7 +17,6 @@ def build_dataset(cfg, split: str) -> UnifiedSegDataset:
         data_root=cfg.data_root, dataset=cfg.dataset, protocol=cfg.protocol, split=split,
         transform=None, in_channels=cfg.in_channels, num_classes=cfg.num_classes,
         synth_dir=synth,
-        train_fraction=cfg.train_fraction, fraction_seed=cfg.fraction_seed,
     )
     ds.transform = build_transform(cfg.img_size, ds.in_channels, train=train,
                                    aug=cfg.aug, normalize=cfg.normalize)

code/framework/data/loaders.py.bak

@@ -1,40 +0,0 @@
-"""Build datasets / dataloaders from a Config, consistent across train & test."""
-from __future__ import annotations
-
-from torch.utils.data import DataLoader
-from torch.utils.data.distributed import DistributedSampler
-
-from .unified_dataset import UnifiedSegDataset
-from .transforms import build_transform
-from ..engine.distributed import is_dist
-
-
-def build_dataset(cfg, split: str) -> UnifiedSegDataset:
-    train = (split == "train")
-    synth = cfg.synth_train_dir if train else ""
-    # construct without transform first so in_channels/num_classes auto-detect runs
-    ds = UnifiedSegDataset(
-        data_root=cfg.data_root, dataset=cfg.dataset, protocol=cfg.protocol, split=split,
-        transform=None, in_channels=cfg.in_channels, num_classes=cfg.num_classes,
-        synth_dir=synth,
-    )
-    ds.transform = build_transform(cfg.img_size, ds.in_channels, train=train,
-                                   aug=cfg.aug, normalize=cfg.normalize)
-    return ds
-
-
-def build_loader(cfg, split: str, ds: UnifiedSegDataset) -> DataLoader:
-    train = (split == "train")
-    sampler = None
-    if is_dist():
-        sampler = DistributedSampler(ds, shuffle=train, drop_last=train)
-    return DataLoader(
-        ds,
-        batch_size=cfg.batch_size,
-        shuffle=(train and sampler is None),
-        sampler=sampler,
-        num_workers=cfg.num_workers,
-        pin_memory=True,
-        drop_last=(train and sampler is None),
-        persistent_workers=cfg.num_workers > 0,
-    )

code/framework/data/unified_dataset.py

@@ -124,8 +124,7 @@ class UnifiedSegDataset(Dataset):
     def __init__(self, data_root: str, dataset: str, protocol: str, split: str,
                  transform: Optional[Callable] = None,
                  in_channels: int = 0, num_classes: int = 0,
-                 synth_dir: str = "",
-                 train_fraction: float = 1.0, fraction_seed: int = 0):
+                 synth_dir: str = ""):
         self.data_root = data_root
         self.dataset = dataset
         self.split = split
@@ -142,12 +141,6 @@ class UnifiedSegDataset(Dataset):
         if not pairs:
             raise RuntimeError(f"no (image,mask) pairs found in {split_dir}")
 
-        # low-data: deterministically subsample REAL train pairs (the generator uses
-        # the SAME select_fraction, so no leakage) BEFORE merging any synthetic data.
-        if split == "train" and train_fraction < 1.0:
-            from ..common.subset import select_fraction
-            pairs = select_fraction(pairs, train_fraction, fraction_seed)
-
         # optionally merge synthetic (image,mask) pairs into the (train) split
         if synth_dir and os.path.isdir(synth_dir):
             sp = _pair_by_glob(synth_dir if os.path.isdir(os.path.join(synth_dir, "images"))

code/framework/data/unified_dataset.py.bak

@@ -1,180 +0,0 @@
-"""Dataset reader for the standardized `processed_unified` layout.
-
-Expected layout (see dataset/SEGMENTATION_WORKSPACE_README.md):
-    <data_root>/<dataset>/<protocol>/<split>/images/*.png
-    <data_root>/<dataset>/<protocol>/<split>/masks/*.png
-    <data_root>/<dataset>/metadata.json          (optional, preferred)
-    <data_root>/<dataset>/manifest.jsonl         (optional)
-
-Returns per item: {"image": FloatTensor[C,H,W], "mask": LongTensor[H,W], "name": str}.
-
-Binary and multi-class masks are both supported: masks keep their integer class
-ids (0..C-1). Auto-detection of in_channels / num_classes falls back to scanning
-files when metadata is absent, so the loader is robust to missing metadata.
-"""
-from __future__ import annotations
-
-import json
-import os
-from glob import glob
-from typing import Optional, Callable, List, Tuple
-
-import numpy as np
-import cv2
-from torch.utils.data import Dataset
-
-
-_MODALITY_CHANNELS = {  # hint table; only used when metadata lacks in_channels
-    "rgb": 3, "fundus": 3, "colonoscopy": 3, "endoscopy": 3, "histopathology": 3,
-    "ultrasound": 1, "mri": 1, "ct": 1, "grayscale": 1,
-}
-
-# Documented class counts (incl. background). metadata.json on the server has no
-# num_classes field, so this table is the fast, reliable primary source; unknown
-# datasets fall back to a FULL scan of the mask set (accurate but slower).
-_KNOWN_NUM_CLASSES = {
-    "cvc_clinicdb": 2, "kvasir_seg": 2, "fives": 2, "busi": 2,
-    "refuge2": 3, "acdc_png": 4,
-    "idridd_segmentation": 6, "pannuke_semantic": 6,
-}
-
-
-def _read_metadata(data_root: str, dataset: str) -> dict:
-    path = os.path.join(data_root, dataset, "metadata.json")
-    if os.path.isfile(path):
-        try:
-            with open(path) as f:
-                return json.load(f)
-        except Exception:
-            return {}
-    return {}
-
-
-def _pair_from_manifest(split_dir: str, manifest: str) -> Optional[List[Tuple[str, str]]]:
-    if not os.path.isfile(manifest):
-        return None
-    pairs = []
-    base = os.path.dirname(manifest)
-    with open(manifest) as f:
-        for line in f:
-            line = line.strip()
-            if not line:
-                continue
-            rec = json.loads(line)
-            img = rec.get("image") or rec.get("image_path") or rec.get("img")
-            msk = rec.get("mask") or rec.get("mask_path") or rec.get("label")
-            if img is None or msk is None:
-                return None
-            # manifest paths may be relative to dataset root or absolute
-            ip = img if os.path.isabs(img) else os.path.join(base, img)
-            mp = msk if os.path.isabs(msk) else os.path.join(base, msk)
-            # only keep entries that fall under this split dir
-            if os.path.normpath(split_dir) in os.path.normpath(ip):
-                pairs.append((ip, mp))
-    return pairs or None
-
-
-def _pair_by_glob(split_dir: str) -> List[Tuple[str, str]]:
-    img_dir = os.path.join(split_dir, "images")
-    msk_dir = os.path.join(split_dir, "masks")
-    imgs = sorted(glob(os.path.join(img_dir, "*")))
-    pairs = []
-    for ip in imgs:
-        stem = os.path.splitext(os.path.basename(ip))[0]
-        # mask may share extension or be .png
-        cands = glob(os.path.join(msk_dir, stem + ".*"))
-        if not cands:
-            continue
-        pairs.append((ip, cands[0]))
-    return pairs
-
-
-def detect_in_channels(meta: dict, sample_img: Optional[str]) -> int:
-    if meta.get("in_channels"):
-        return int(meta["in_channels"])
-    mod = str(meta.get("modality", "")).lower()
-    for k, v in _MODALITY_CHANNELS.items():
-        if k in mod:
-            return v
-    if sample_img and os.path.isfile(sample_img):
-        im = cv2.imread(sample_img, cv2.IMREAD_UNCHANGED)
-        if im is not None and im.ndim == 3 and im.shape[2] >= 3:
-            return 3
-    return 1
-
-
-def detect_num_classes(meta: dict, mask_paths: List[str], dataset: str = "") -> int:
-    if dataset in _KNOWN_NUM_CLASSES:
-        return _KNOWN_NUM_CLASSES[dataset]
-    if meta.get("num_classes"):
-        return int(meta["num_classes"])
-    # unknown dataset: scan ALL masks so a rare class is never missed
-    vals = set()
-    for mp in mask_paths:
-        m = cv2.imread(mp, cv2.IMREAD_GRAYSCALE)
-        if m is not None:
-            vals.update(np.unique(m).tolist())
-    if not vals:
-        return 2
-    maxv = max(vals)
-    return int(maxv) + 1 if maxv >= 1 else 2
-
-
-class UnifiedSegDataset(Dataset):
-    def __init__(self, data_root: str, dataset: str, protocol: str, split: str,
-                 transform: Optional[Callable] = None,
-                 in_channels: int = 0, num_classes: int = 0,
-                 synth_dir: str = ""):
-        self.data_root = data_root
-        self.dataset = dataset
-        self.split = split
-        self.transform = transform
-
-        split_dir = os.path.join(data_root, dataset, protocol, split)
-        if not os.path.isdir(split_dir):
-            raise FileNotFoundError(
-                f"split dir not found: {split_dir}\n"
-                f"(data is prepared separately; see dataset/ scripts)")
-
-        manifest = os.path.join(data_root, dataset, "manifest.jsonl")
-        pairs = _pair_from_manifest(split_dir, manifest) or _pair_by_glob(split_dir)
-        if not pairs:
-            raise RuntimeError(f"no (image,mask) pairs found in {split_dir}")
-
-        # optionally merge synthetic (image,mask) pairs into the (train) split
-        if synth_dir and os.path.isdir(synth_dir):
-            sp = _pair_by_glob(synth_dir if os.path.isdir(os.path.join(synth_dir, "images"))
-                               else os.path.dirname(synth_dir))
-            pairs = pairs + sp
-
-        self.pairs = pairs
-        meta = _read_metadata(data_root, dataset)
-        self.in_channels = in_channels or detect_in_channels(meta, pairs[0][0])
-        self.num_classes = num_classes or detect_num_classes(meta, [p[1] for p in pairs], dataset)
-
-    def __len__(self) -> int:
-        return len(self.pairs)
-
-    def _load_image(self, path: str) -> np.ndarray:
-        if self.in_channels == 1:
-            im = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
-            if im is None:
-                raise IOError(f"cannot read image {path}")
-            return im[:, :, None]                      # H,W,1
-        im = cv2.imread(path, cv2.IMREAD_COLOR)        # BGR
-        if im is None:
-            raise IOError(f"cannot read image {path}")
-        return cv2.cvtColor(im, cv2.COLOR_BGR2RGB)     # H,W,3
-
-    def __getitem__(self, idx: int):
-        ip, mp = self.pairs[idx]
-        image = self._load_image(ip)
-        mask = cv2.imread(mp, cv2.IMREAD_GRAYSCALE)
-        if mask is None:
-            raise IOError(f"cannot read mask {mp}")
-        mask = mask.astype(np.int64)
-
-        if self.transform is not None:
-            image, mask = self.transform(image, mask)
-        return {"image": image, "mask": mask,
-                "name": os.path.splitext(os.path.basename(ip))[0]}

code/framework/eval_ckpt.py

@@ -1,120 +0,0 @@
-"""Evaluate a trained framework checkpoint on a chosen dataset/split, with the full
-metric set INCLUDING boundary metrics. Two uses, one script (no retraining):
-
-  * C7 (boundary fidelity, in-domain): point --eval_dataset at the same dataset to
-    recompute boundary-Dice / NSD from a saved best.pth.
-  * C4 (cross-center): set --eval_dataset/--eval_protocol to a DIFFERENT but
-    label-compatible dataset (e.g. train busi? no — train cvc_clinicdb, eval
-    kvasir_seg; both binary polyp). num_classes/in_channels must match.
-
-Run from project root (…/NPJ), env seggen:
-  CUDA_VISIBLE_DEVICES=5 python -m framework.eval_ckpt \
-      --ckpt results/baselines/cvc_clinicdb_official/unet/seed0/best.pth \
-      --arch unet --encoder resnet50 \
-      --data_root $DR --dataset cvc_clinicdb --protocol official \
-      --eval_dataset kvasir_seg --eval_protocol official \
-      --out_json results/crosscenter/cvc2kvasir_unet_seed0.json
-"""
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import sys
-
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
-
-import numpy as np
-import torch
-from torch.utils.data import DataLoader
-
-from framework.config import Config  # noqa: F401  (kept for parity / future YAML use)
-from framework.models.registry import build_model, required_img_size
-from framework.data.unified_dataset import UnifiedSegDataset
-from framework.data.transforms import build_transform
-from framework.metrics.metrics import per_image_metrics
-from framework.metrics.boundary import boundary_metrics
-
-
-def get_args():
-    p = argparse.ArgumentParser("Evaluate a checkpoint (in-domain or cross-center) + boundary metrics")
-    p.add_argument("--ckpt", required=True)
-    p.add_argument("--arch", default="unet")
-    p.add_argument("--encoder", default="resnet50")
-    p.add_argument("--data_root", required=True)
-    p.add_argument("--dataset", required=True, help="dataset the ckpt was TRAINED on (sets in_ch/num_classes)")
-    p.add_argument("--protocol", required=True)
-    p.add_argument("--eval_dataset", default="", help="dataset to evaluate ON (default = --dataset)")
-    p.add_argument("--eval_protocol", default="", help="default = --protocol")
-    p.add_argument("--split", default="test")
-    p.add_argument("--img_size", type=int, default=256)
-    p.add_argument("--normalize", default="auto")
-    p.add_argument("--tol", type=float, default=2.0)
-    p.add_argument("--batch_size", type=int, default=16)
-    p.add_argument("--num_workers", type=int, default=6)
-    p.add_argument("--out_json", default="")
-    return p.parse_args()
-
-
-def main():
-    a = get_args()
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    img_size = required_img_size(a.arch) or a.img_size
-    eval_ds_name = a.eval_dataset or a.dataset
-    eval_proto = a.eval_protocol or a.protocol
-
-    # in_ch / num_classes are fixed by the TRAIN dataset (how the model was built)
-    train_probe = UnifiedSegDataset(a.data_root, a.dataset, a.protocol, "test", transform=None)
-    in_ch, n_cls = train_probe.in_channels, train_probe.num_classes
-
-    model = build_model(a.arch, in_channels=in_ch, num_classes=n_cls, img_size=img_size,
-                        encoder=a.encoder, encoder_weights="none", pretrained_ckpt="")
-    ckpt = torch.load(a.ckpt, map_location="cpu", weights_only=False)
-    model.load_state_dict(ckpt["model"])
-    model.to(device).eval()
-
-    tf = build_transform(img_size, in_ch, train=False, aug="none", normalize=a.normalize)
-    ds = UnifiedSegDataset(a.data_root, eval_ds_name, eval_proto, a.split,
-                           transform=tf, in_channels=in_ch, num_classes=n_cls)
-    if ds.num_classes != n_cls:
-        raise ValueError(f"label mismatch: train num_classes={n_cls} vs eval={ds.num_classes} "
-                         f"({a.dataset}->{eval_ds_name}) — not label-compatible for cross-center.")
-    cross = (eval_ds_name != a.dataset) or (eval_proto != a.protocol)
-    print(f"[eval] ckpt={os.path.basename(a.ckpt)} train={a.dataset}/{a.protocol} "
-          f"eval={eval_ds_name}/{eval_proto}/{a.split} cross_center={cross} "
-          f"in_ch={in_ch} num_classes={n_cls} n={len(ds)}", flush=True)
-
-    loader = DataLoader(ds, batch_size=a.batch_size, shuffle=False, num_workers=a.num_workers)
-    recs = []
-    for batch in loader:
-        img = batch["image"].to(device, non_blocking=True)
-        msk = batch["mask"].numpy()
-        with torch.no_grad():
-            pred = model(img).argmax(1).cpu().numpy()
-        for i in range(pred.shape[0]):
-            m = per_image_metrics(pred[i], msk[i], n_cls,
-                                  include_background=False, compute_hd95=True)
-            b = boundary_metrics(pred[i], msk[i], n_cls, tol=a.tol)
-            m.update(b)
-            recs.append(m)
-
-    keys = [k for k, val in recs[0].items() if isinstance(val, (int, float))]  # skip per_class dict
-    summary = {}
-    for k in keys:
-        v = np.array([r[k] for r in recs], dtype=np.float64)
-        v = v[~np.isnan(v)]
-        summary[k] = {"mean": round(float(v.mean()), 4) if v.size else None,
-                      "std": round(float(v.std()), 4) if v.size else None, "n": int(v.size)}
-    out = {"ckpt": a.ckpt, "train": f"{a.dataset}/{a.protocol}",
-           "eval": f"{eval_ds_name}/{eval_proto}/{a.split}", "cross_center": cross,
-           "num_images": len(ds), "metrics": summary}
-    print(json.dumps(out, indent=2), flush=True)
-    if a.out_json:
-        os.makedirs(os.path.dirname(os.path.abspath(a.out_json)) or ".", exist_ok=True)
-        with open(a.out_json, "w") as f:
-            json.dump(out, f, indent=2)
-        print(f"[eval] wrote {a.out_json}", flush=True)
-
-
-if __name__ == "__main__":
-    main()

code/framework/metrics/boundary.py

@@ -1,68 +0,0 @@
-"""Boundary-localized segmentation metrics (for the C7 boundary-fidelity analysis).
-
-Two standard measures, computed per foreground class then averaged:
-  * Normalized Surface Dice (NSD) @ tol: fraction of pred/gt surface points within
-    `tol` pixels of the other surface (Nikolov et al.); the medical-standard
-    boundary metric.
-  * Boundary-Dice @ tol: Dice between the tol-dilated pred and gt boundaries.
-
-Recomputable post-hoc from saved predictions/checkpoints, so adding it never
-requires retraining (see framework/eval_ckpt.py).
-"""
-from __future__ import annotations
-
-import numpy as np
-
-try:
-    from scipy.ndimage import binary_erosion, binary_dilation, distance_transform_edt
-    _HAVE_SCIPY = True
-except Exception:  # pragma: no cover
-    _HAVE_SCIPY = False
-
-
-def _surface(bin_mask: np.ndarray) -> np.ndarray:
-    return bin_mask ^ binary_erosion(bin_mask)
-
-
-def _nsd_binary(pred: np.ndarray, gt: np.ndarray, tol: float) -> float:
-    sp, sg = _surface(pred), _surface(gt)
-    ssum = sp.sum() + sg.sum()
-    if sp.sum() == 0 and sg.sum() == 0:
-        return 1.0
-    if ssum == 0:
-        return 0.0
-    dt_to_gt = distance_transform_edt(~sg)
-    dt_to_pred = distance_transform_edt(~sp)
-    pred_close = (dt_to_gt[sp] <= tol).sum()
-    gt_close = (dt_to_pred[sg] <= tol).sum()
-    return float((pred_close + gt_close) / ssum)
-
-
-def _bdice_binary(pred: np.ndarray, gt: np.ndarray, tol: int) -> float:
-    sp, sg = _surface(pred), _surface(gt)
-    if sp.sum() == 0 and sg.sum() == 0:
-        return 1.0
-    spd = binary_dilation(sp, iterations=int(tol))
-    sgd = binary_dilation(sg, iterations=int(tol))
-    denom = spd.sum() + sgd.sum()
-    if denom == 0:
-        return 0.0
-    return float(2.0 * (spd & sgd).sum() / denom)
-
-
-def boundary_metrics(pred: np.ndarray, gt: np.ndarray, num_classes: int,
-                     tol: float = 2.0) -> dict:
-    """Mean over foreground classes (1..num_classes-1). NaN if scipy missing."""
-    if not _HAVE_SCIPY:
-        return {"nsd": float("nan"), "boundary_dice": float("nan")}
-    nsds, bdices = [], []
-    for c in range(1, num_classes):
-        p, g = (pred == c), (gt == c)
-        if g.sum() == 0 and p.sum() == 0:
-            continue
-        nsds.append(_nsd_binary(p, g, tol))
-        bdices.append(_bdice_binary(p, g, int(round(tol))))
-    return {
-        "nsd": float(np.mean(nsds)) if nsds else float("nan"),
-        "boundary_dice": float(np.mean(bdices)) if bdices else float("nan"),
-    }

code/framework/synth/generative_baselines.py

@@ -0,0 +1,112 @@
+"""Orchestration for the generative-augmentation SOTA baselines (category B).
+
+These methods are compared against our SegGen method. Each runs in its OWN conda
+env (see envs/) because their dependency stacks conflict with the main framework.
+The shared contract: every generator must emit paired (image, mask) into
+
+    <data_root>/<dataset>/<protocol>/synth_<method>/{images,masks}/
+
+which the unified trainer then merges into the train split via --synth_train_dir.
+
+Kept baselines:
+  * SegGuidedDiff (diffusion, mask->image, medical, modern stack) -- best fit, USE-AS-IS
+  * SPADE         (GAN, mask->image)                              -- ADAPT (needs sync_bn)
+  * ControlNet    (diffusion, SD-finetune, mask->image)           -- ADAPT (needs SD ckpt)
+
+Dropped (per scoping): StyleGAN2-ADA (no masks), LDM (dep hell + AE training).
+
+This module only BUILDS the commands + assembles the standard synth dir; it does
+not import the repos (they live in separate envs). Run the printed commands in the
+matching env, then call assemble_synth_dir() (env-agnostic) to lay out pairs.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+from glob import glob
+
+SOTA = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "sota"))
+
+
+def assemble_synth_dir(generated_images_dir: str, masks_source_dir: str,
+                       out_dir: str, strip_prefix: str = "condon_",
+                       link: bool = True) -> int:
+    """Pair each generated image with the real mask it was conditioned on.
+
+    Mask-conditioned generators name outputs after the conditioning mask
+    (SegGuidedDiff: 'condon_<maskname>.png'). We recover the mask name, copy/link
+    the matching real mask, and place both under out_dir/{images,masks}/.
+    Returns the number of pairs assembled.
+    """
+    img_out = os.path.join(out_dir, "images")
+    msk_out = os.path.join(out_dir, "masks")
+    os.makedirs(img_out, exist_ok=True)
+    os.makedirs(msk_out, exist_ok=True)
+
+    n = 0
+    for gp in sorted(glob(os.path.join(generated_images_dir, "*"))):
+        base = os.path.basename(gp)
+        stem = os.path.splitext(base)[0]
+        if strip_prefix and stem.startswith(strip_prefix):
+            mask_stem = stem[len(strip_prefix):]
+        else:
+            mask_stem = stem
+        cands = glob(os.path.join(masks_source_dir, mask_stem + ".*"))
+        if not cands:
+            continue
+        out_name = f"synth_{n:06d}"
+        dst_img = os.path.join(img_out, out_name + os.path.splitext(base)[1])
+        dst_msk = os.path.join(msk_out, out_name + os.path.splitext(cands[0])[1])
+        _place(gp, dst_img, link)
+        _place(cands[0], dst_msk, link)
+        n += 1
+    return n
+
+
+def _place(src, dst, link):
+    if os.path.exists(dst):
+        os.remove(dst)
+    if link:
+        os.symlink(os.path.abspath(src), dst)
+    else:
+        shutil.copy2(src, dst)
+
+
+# ---- command builders (printed into run.sh; run in the matching conda env) ----
+
+def segguideddiff_cmds(data_root, dataset, protocol, num_classes, in_channels,
+                       img_size=256, epochs=400, sample_size=1000):
+    repo = os.path.join(SOTA, "segmentation-guided-diffusion")
+    img_dir = f"{data_root}/{dataset}/{protocol}/train/images"
+    seg_dir = f"{data_root}/{dataset}/{protocol}/train/masks"
+    train = (f"cd {repo} && python main.py --mode train --model_type DDIM "
+             f"--img_size {img_size} --num_img_channels {in_channels} --dataset {dataset} "
+             f"--img_dir {img_dir} --seg_dir {seg_dir} --segmentation_guided "
+             f"--num_segmentation_classes {num_classes} --num_epochs {epochs}")
+    synth = (f"cd {repo} && python main.py --mode eval_many --model_type DDIM "
+             f"--img_size {img_size} --num_img_channels {in_channels} --dataset {dataset} "
+             f"--seg_dir {seg_dir} --segmentation_guided "
+             f"--num_segmentation_classes {num_classes} --eval_sample_size {sample_size}")
+    return train, synth
+
+
+def spade_cmds(data_root, dataset, protocol, num_classes, img_size=256, niter=100):
+    repo = os.path.join(SOTA, "SPADE")
+    img_dir = f"{data_root}/{dataset}/{protocol}/train/images"
+    lab_dir = f"{data_root}/{dataset}/{protocol}/train/masks"
+    setup = (f"cd {repo}/models/networks && "
+             f"git clone https://github.com/vacancy/Synchronized-BatchNorm-PyTorch && "
+             f"cp -r Synchronized-BatchNorm-PyTorch/sync_batchnorm .")
+    train = (f"cd {repo} && python train.py --name {dataset}_spade --dataset_mode custom "
+             f"--label_dir {lab_dir} --image_dir {img_dir} --label_nc {num_classes} "
+             f"--no_instance --crop_size {img_size} --load_size {int(img_size*1.12)} --niter {niter}")
+    synth = (f"cd {repo} && python test.py --name {dataset}_spade --dataset_mode custom "
+             f"--label_dir {lab_dir} --image_dir {img_dir} --label_nc {num_classes} "
+             f"--no_instance --results_dir ./synth_{dataset}")
+    return setup, train, synth
+
+
+def controlnet_notes():
+    return ("ControlNet: download SD v1.5 (~4GB), run tool_add_control.py, write a "
+            "MyDataset that colorizes integer masks to RGB hints + triples grayscale "
+            "images to 3ch, then tutorial_train.py. Run in env seggen-controlnet.")

code/scripts/a100_nnum_eval512.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# nnU-Net + U-Mamba @512 re-eval on a100 (CPU-only, does NOT touch GPUs).
+# Re-scores cached predTs/predTs_umamba at eval_size 512 into results/unified512/
+# so they join the unified-512 table. ~64 cells, parallel (concurrency MAXJ).
+set -u
+cd /home/wzhang/LSC/Code/NPJ
+source /opt/anaconda3/etc/profile.d/conda.sh
+conda activate seggen
+export OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 OPENBLAS_NUM_THREADS=4 NUMEXPR_NUM_THREADS=4
+DR=/home/wzhang/LSC/Dataset/Segmentation/processed_unified
+RAW=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/raw
+PRED_NN=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/predTs
+PRED_UM=/home/wzhang/LSC/Code/NPJ/nnunet_workspace/predTs_umamba
+MAXJ=10
+
+declare -A DS=(
+  [1]=cvc_clinicdb:official [2]=kvasir_seg:official [3]=fives:official
+  [4]=refuge2:official [5]=busi:fold01 [6]=idridd_segmentation:fold01
+  [7]=acdc_png:official [8]=pannuke_semantic:fold01 [9]=medsegdb_isic2018:holdout
+  [10]=medsegdb_kits19:fold01 [11]=pannuke_semantic:fold02 [12]=pannuke_semantic:fold03
+)
+
+run=0
+for id in 1 2 3 4 5 6 7 8 9 10 11 12; do
+  IFS=: read -r ds proto <<< "${DS[$id]}"
+  for f in 0 1 2; do
+    for ap in "nnunet:$PRED_NN" "umamba:$PRED_UM"; do
+      arch=${ap%%:*}; pred=${ap#*:}
+      outdir=$pred/d${id}_f${f}
+      [ -d "$outdir" ] && ls -A "$outdir"/*.png >/dev/null 2>&1 || continue
+      (
+        python framework/nnunet_eval.py --data_root "$DR" --dataset "$ds" --protocol "$proto" \
+          --raw "$RAW" --dataset_id "$id" --fold "$f" --pred_dir "$outdir" --arch "$arch" \
+          --exp_name unified512 --eval_size 512 \
+          > /tmp/nnum512_${arch}_d${id}_f${f}.log 2>&1 \
+          && echo "[ok] $arch d${id}_f${f} ($ds $proto)" \
+          || echo "[FAIL] $arch d${id}_f${f} ($ds $proto)"
+      ) &
+      run=$((run+1))
+      if [ "$run" -ge "$MAXJ" ]; then wait -n; run=$((run-1)); fi
+    done
+  done
+done
+wait
+echo "NNUM_EVAL512_DONE"
+n=$(find results/unified512 -path '*/nnunet/*/metrics.json' -o -path '*/umamba/*/metrics.json' 2>/dev/null | wc -l)
+echo "unified512 nnunet+umamba metrics.json count: $n"

code/scripts/a100_swin_transunet_3seed_eval512.py

@@ -0,0 +1,58 @@
+"""Fill SwinUNet/TransUNet to FULL 3-seed @512 on a100 (their per-seed best.pth live here).
+For every results/baselines/<cell>/{swinunet,transunet}/seed<s>/best.pth, copy it into the
+unified512 tree and run eval_at_res.py --eval_size 512 --exp_name unified512. GPU 4/5 only
+(A100 80G; PCI_BUS_ID). 64 evals. Then metrics.json get transferred to h800 + re-aggregated.
+"""
+import os, glob, shutil, subprocess, time
+
+CODE = "/home/wzhang/LSC/Code/NPJ"
+DATA = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+PY   = "/opt/anaconda3/envs/seggen/bin/python"
+BASE = CODE + "/results/baselines"       # source per-seed weights
+UNI  = CODE + "/results/unified512"      # eval_at_res writes here (out_root=results rel to CODE)
+LOGD = "/tmp/sw_tr_3seed_logs"; os.makedirs(LOGD, exist_ok=True)
+SLOTS = [4, 4, 4, 5, 5, 5]               # GPU 4/5, 3 co-located evals each
+
+jobs = []
+for arch in ("swinunet", "transunet"):
+    for w in sorted(glob.glob(f"{BASE}/*/{arch}/seed*/best.pth")):
+        parts = w.split("/")
+        cell, seed = parts[-4], parts[-2]           # <cell>, seedN
+        sd = int(seed.replace("seed", ""))
+        # parse cell -> dataset, protocol
+        ds, proto = None, None
+        for p in ("official", "holdout", "fold01", "fold02", "fold03"):
+            if cell.endswith("_" + p):
+                ds, proto = cell[:-(len(p) + 1)], p; break
+        out = f"{UNI}/{cell}/{arch}/{seed}"
+        jobs.append({"ds": ds, "proto": proto, "arch": arch, "seed": sd, "w": w,
+                     "out": out, "mj": out + "/metrics.json", "tag": f"{cell}_{arch}_s{sd}"})
+
+pending = [j for j in jobs if not os.path.isfile(j["mj"])]
+print(f"[3seed] total={len(jobs)} done={len(jobs)-len(pending)} pending={len(pending)}", flush=True)
+
+def make_cmd(j, gpu):
+    enc = "R50-ViT-B_16" if j["arch"] == "transunet" else "resnet50"
+    os.makedirs(j["out"], exist_ok=True)
+    shutil.copy(j["w"], j["out"] + "/best.pth")
+    return (f"export CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES={gpu} "
+            f"OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 OPENBLAS_NUM_THREADS=8 && cd {CODE} && "
+            f"{PY} framework/eval_at_res.py --data_root {DATA} --dataset {j['ds']} "
+            f"--protocol {j['proto']} --arch {j['arch']} --seed {j['seed']} --eval_size 512 "
+            f"--exp_name unified512 --encoder {enc}")
+
+running = {}; free = list(SLOTS); i = 0; ok = fail = 0
+while i < len(pending) or running:
+    while free and i < len(pending):
+        gpu = free.pop(0); j = pending[i]; i += 1
+        lf = open(f"{LOGD}/{j['tag']}.log", "w")
+        p = subprocess.Popen(["bash", "-lc", make_cmd(j, gpu)], stdout=lf, stderr=subprocess.STDOUT)
+        running[id(p)] = (p, j, lf, gpu); print(f"[launch] gpu{gpu} {j['tag']}", flush=True)
+    time.sleep(6)
+    for k, (p, j, lf, gpu) in list(running.items()):
+        if p.poll() is not None:
+            lf.close(); okj = os.path.isfile(j["mj"]); ok += okj; fail += (not okj)
+            print(f"[finish] gpu{gpu} {j['tag']} rc={p.returncode} ok={okj}", flush=True)
+            del running[k]; free.append(gpu)
+print(f"[3seed] ALL DONE ok={ok} fail={fail}", flush=True)
+print("SWTR_3SEED_DONE", flush=True)

code/scripts/h800_cache_data.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Cache the dataset from the slow JuiceFS share to local RAID /data/temp for fast
+# training reads. Parallel per-dataset cp -a (preserves pannuke hard links), overlaps
+# JuiceFS small-file read latency. Run detached; log to /tmp/cache_data.log.
+SRC=/mnt/tidal-alsh-share2/dataset/qinshengqian/research/c3/NPJ-ACM/Data
+DST=/data/temp/NPJ-ACM/Data
+mkdir -p "$DST"
+echo "[start] $(date +%T)"
+for d in acdc_png busi cvc_clinicdb fives idridd_segmentation kvasir_seg \
+         medsegdb_isic2018 medsegdb_kits19 pannuke_semantic refuge2; do
+  ( cp -a "$SRC/$d" "$DST/" && echo "done $d $(date +%T)" ) &
+done
+wait
+echo "CACHE_DONE $(date +%T)"

code/scripts/h800_fetch_data.py

@@ -0,0 +1,31 @@
+"""On h800: download GenSegDataset tars from HF (via proxy+token), extract into Data/,
+then remove the tars. Produces the processed_unified layout under Data/."""
+import os, glob, tarfile
+from huggingface_hub import snapshot_download
+
+BASE = "/mnt/tidal-alsh-share2/dataset/qinshengqian/research/c3/NPJ-ACM/Data"
+TARS = os.path.join(BASE, "_tars")
+
+print("[1] downloading tars ...", flush=True)
+snapshot_download("MaybeRichard/GenSegDataset", repo_type="dataset",
+                  allow_patterns=["*.tar", "README.md"], local_dir=TARS)
+
+print("[2] extracting ...", flush=True)
+for t in sorted(glob.glob(os.path.join(TARS, "*.tar"))):
+    print("  extract", os.path.basename(t), flush=True)
+    with tarfile.open(t) as tf:
+        tf.extractall(BASE)
+
+rd = os.path.join(TARS, "README.md")
+if os.path.isfile(rd):
+    os.replace(rd, os.path.join(BASE, "README.md"))
+
+print("[3] cleanup tars ...", flush=True)
+for t in glob.glob(os.path.join(TARS, "*.tar")):
+    os.remove(t)
+try:
+    os.rmdir(TARS)
+except OSError:
+    pass
+
+print("DONE_DATA", flush=True)

code/scripts/h800_parallel_extract.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Parallel extraction of the remaining GenSegDataset tars on h800's slow network share.
+# Big archives are split by member-list into N chunks, each extracted by a separate
+# `tar -x -T <chunk>` process, to saturate the share's parallel small-file throughput.
+set -u
+BASE=/mnt/tidal-alsh-share2/dataset/qinshengqian/research/c3/NPJ-ACM/Data
+TARS=$BASE/_tars
+WORK=/tmp/pextract
+mkdir -p "$WORK"
+
+# dataset -> parallel chunk count (kits19 = most files)
+launch_ds() {
+  local ds=$1 n=$2 tar="$TARS/$1.tar"
+  [ -f "$tar" ] || { echo "MISSING $tar"; return; }
+  if [ "$n" -le 1 ]; then
+    tar -xf "$tar" -C "$BASE" &
+  else
+    tar -tf "$tar" | grep -v '/$' > "$WORK/$ds.list"
+    split -n "l/$n" -d "$WORK/$ds.list" "$WORK/$ds.chunk."
+    for c in "$WORK/$ds.chunk."*; do
+      tar -xf "$tar" -C "$BASE" -T "$c" &
+    done
+  fi
+}
+
+echo "[start] $(date +%T) launching parallel extraction"
+launch_ds medsegdb_kits19 8
+launch_ds pannuke_semantic 4
+launch_ds refuge2 1
+echo "launched $(jobs -p | wc -l) parallel tar streams"
+wait
+echo "PEXTRACT_DONE $(date +%T)"
+
+# cleanup tars + work
+rm -f "$TARS"/*.tar
+rmdir "$TARS" 2>/dev/null || true
+rm -rf "$WORK"
+echo "CLEANUP_DONE $(date +%T)"

code/scripts/h800_run_unified512.py

@@ -0,0 +1,80 @@
+"""8-GPU pool runner for the UNIFIED-512 conv retrain on h800 (L20Y x8).
+Mirrors the baseline grid: 4 conv archs x the existing (dataset,protocol,seed) cells,
+all at img_size 512. 1 job per GPU, resumable (skips cells whose metrics.json exists),
+per-job log. occupy.py auto-yields. Run detached; tail /tmp/unified512_runner.log.
+"""
+import os, sys, subprocess, time
+
+CODE = "/mnt/tidal-alsh-share2/dataset/qinshengqian/research/c3/NPJ-ACM/Code"
+DATA = "/data/temp/NPJ-ACM/Data"
+WORK = "/data/temp/NPJ-ACM/work"          # CWD; results -> WORK/results/unified512/...
+PY   = "/data/temp/miniconda3/envs/seggen/bin/python"
+LOGD = WORK + "/logs_unified512"
+PROXY = "http://10.140.15.68:3128"
+NGPU, IMG, BATCH, EPOCHS = 8, 512, 8, 300
+ARCHS = ["unet", "unetpp", "deeplabv3plus", "attention_unet"]
+CELLS = [  # (dataset, protocol, [seeds]) -- matches existing baseline structure
+    ("acdc_png", "official", [0, 1, 2]),
+    ("busi", "fold01", [0, 1, 2]),
+    ("cvc_clinicdb", "official", [0, 1, 2]),
+    ("fives", "official", [0, 1, 2]),
+    ("idridd_segmentation", "fold01", [0, 1, 2]),
+    ("kvasir_seg", "official", [0, 1, 2]),
+    ("medsegdb_isic2018", "holdout", [0, 1, 2]),
+    ("medsegdb_kits19", "fold01", [0, 1, 2]),
+    ("refuge2", "official", [0, 1, 2]),
+    ("pannuke_semantic", "fold01", [0, 1, 2]),
+    ("pannuke_semantic", "fold02", [0]),
+    ("pannuke_semantic", "fold03", [0]),
+]
+os.makedirs(LOGD, exist_ok=True)
+os.makedirs(WORK, exist_ok=True)
+
+jobs = []
+for ds, proto, seeds in CELLS:
+    for arch in ARCHS:
+        for s in seeds:
+            out = f"{WORK}/results/unified512/{ds}_{proto}/{arch}/seed{s}/metrics.json"
+            jobs.append({"ds": ds, "proto": proto, "arch": arch, "seed": s, "out": out,
+                         "tag": f"{ds}_{proto}_{arch}_s{s}"})
+
+def make_cmd(j, gpu):
+    return (
+        f"export CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES={gpu} "
+        f"OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 OPENBLAS_NUM_THREADS=8 "
+        f"https_proxy={PROXY} http_proxy={PROXY} && cd {WORK} && "
+        f"{PY} {CODE}/framework/train.py --data_root {DATA} --dataset {j['ds']} --protocol {j['proto']} "
+        f"--arch {j['arch']} --img_size {IMG} --batch_size {BATCH} --num_workers 8 --amp bf16 "
+        f"--exp_name unified512 --seed {j['seed']} --no-visualize --encoder resnet50 --encoder_weights imagenet "
+        f"--epochs {EPOCHS} && "
+        f"{PY} {CODE}/framework/test.py --data_root {DATA} --dataset {j['ds']} --protocol {j['proto']} "
+        f"--arch {j['arch']} --img_size {IMG} --exp_name unified512 --seed {j['seed']} --encoder resnet50"
+    )
+
+pending = [j for j in jobs if not os.path.isfile(j["out"])]
+print(f"[runner] total={len(jobs)} done={len(jobs)-len(pending)} pending={len(pending)}", flush=True)
+
+running = {}   # gpu -> (Popen, job, start)
+free = list(range(NGPU))
+done = ok = fail = 0
+i = 0
+while i < len(pending) or running:
+    while free and i < len(pending):
+        gpu = free.pop(0)
+        j = pending[i]; i += 1
+        lf = open(f"{LOGD}/{j['tag']}.log", "w")
+        p = subprocess.Popen(["bash", "-lc", make_cmd(j, gpu)], stdout=lf, stderr=subprocess.STDOUT)
+        running[gpu] = (p, j, time.time(), lf)
+        print(f"[launch] gpu{gpu} {j['tag']}", flush=True)
+    time.sleep(20)
+    for gpu, (p, j, st, lf) in list(running.items()):
+        if p.poll() is not None:
+            lf.close(); done += 1
+            okj = os.path.isfile(j["out"])
+            ok += okj; fail += (not okj)
+            mins = (time.time() - st) / 60
+            print(f"[finish] gpu{gpu} {j['tag']} rc={p.returncode} ok={okj} "
+                  f"{mins:.0f}min ({done}/{len(pending)} done, {fail} failed)", flush=True)
+            del running[gpu]; free.append(gpu); free.sort()
+print(f"[runner] ALL DONE. ok={ok} fail={fail} of {len(pending)}", flush=True)
+print("UNIFIED512_RUNNER_DONE", flush=True)

code/scripts/h800_setup_seggen.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Build the seggen conda env on h800 (L20Y / CUDA 12.8). torch installed FIRST (cu128)
+# so SMP/MONAI don't pull a mismatched torch. Run detached; log to /tmp/seggen_env.log.
+set -e
+export https_proxy=http://10.140.15.68:3128 http_proxy=http://10.140.15.68:3128
+CONDA=/data/temp/miniconda3
+PROXY=http://10.140.15.68:3128
+
+echo "[1] create env (python 3.11) -- conda-forge only, avoids defaults-channel ToS block"
+$CONDA/bin/conda create -y -n seggen -c conda-forge --override-channels python=3.11 pip
+
+PIP="$CONDA/envs/seggen/bin/pip"
+echo "[2] torch cu128 (host CUDA 12.8, >2.6)"
+$PIP install --proxy "$PROXY" torch torchvision --index-url https://download.pytorch.org/whl/cu128
+
+echo "[3] seg stack (torch already present -> no reinstall)"
+$PIP install --proxy "$PROXY" \
+  segmentation-models-pytorch albumentations==2.0.8 monai medpy \
+  opencv-python-headless numpy pyyaml timm einops ml-collections tqdm \
+  diffusers==0.21.4 datasets==2.14.5
+
+echo "[4] verify"
+$CONDA/envs/seggen/bin/python - <<'PY'
+import torch, segmentation_models_pytorch, monai, albumentations, timm, cv2
+print("torch", torch.__version__, "| cuda", torch.version.cuda, "| avail", torch.cuda.is_available(), "| ndev", torch.cuda.device_count())
+print("smp ok, monai ok, albumentations ok, timm ok, cv2", cv2.__version__)
+PY
+echo "SEGGEN_ENV_DONE"

code/scripts/h800_swin_transunet_eval512.py

@@ -0,0 +1,68 @@
+"""Phase-1 of unified-512 re-eval on h800: re-score SwinUNet/TransUNet at eval_size 512.
+These are res-locked (224/256) so we DON'T retrain — we load their HF-curated best-seed
+weights, run at native input, resize preds+GT to 512, write metrics.json into the
+unified512 results tree. 12 cells x {swinunet, transunet} = 24 evals, 8-GPU pool.
+"""
+import os, glob, shutil, subprocess, time
+
+CODE = "/mnt/tidal-alsh-share2/dataset/qinshengqian/research/c3/NPJ-ACM/Code"
+DATA = "/data/temp/NPJ-ACM/Data"
+WORK = "/data/temp/NPJ-ACM/work"
+PY   = "/data/temp/miniconda3/envs/seggen/bin/python"
+HFW  = WORK + "/hf_weights/weights/framework"          # <cell>/{swinunet,transunet}.pth
+RES  = WORK + "/results/unified512"                    # eval_at_res writes here (out_root=results rel to WORK)
+LOGD = WORK + "/logs_eval512"; os.makedirs(LOGD, exist_ok=True)
+PROXY = "http://10.140.15.68:3128"
+PROTOS = ["official", "holdout", "fold01", "fold02", "fold03"]
+NGPU = 8
+
+def split_cell(cell):
+    for p in PROTOS:
+        if cell.endswith("_" + p):
+            return cell[:-(len(p) + 1)], p
+    raise ValueError("bad cell " + cell)
+
+jobs = []
+for cell_dir in sorted(glob.glob(HFW + "/*")):
+    cell = os.path.basename(cell_dir)
+    ds, proto = split_cell(cell)
+    for arch in ("swinunet", "transunet"):
+        w = f"{cell_dir}/{arch}.pth"
+        if not os.path.isfile(w):
+            continue
+        out = f"{RES}/{cell}/{arch}/seed0"
+        jobs.append({"ds": ds, "proto": proto, "arch": arch, "w": w, "out": out,
+                     "tag": f"{cell}_{arch}", "mj": f"{out}/metrics.json"})
+
+pending = [j for j in jobs if not os.path.isfile(j["mj"])]
+print(f"[eval512] total={len(jobs)} done={len(jobs)-len(pending)} pending={len(pending)}", flush=True)
+
+def make_cmd(j, gpu):
+    enc = "R50-ViT-B_16" if j["arch"] == "transunet" else "resnet50"
+    # place the HF weight as best.pth where eval_at_res.py expects it
+    os.makedirs(j["out"], exist_ok=True)
+    shutil.copy(j["w"], j["out"] + "/best.pth")
+    return (
+        f"export CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES={gpu} "
+        f"OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 OPENBLAS_NUM_THREADS=8 "
+        f"https_proxy={PROXY} http_proxy={PROXY} && cd {WORK} && "
+        f"{PY} {CODE}/framework/eval_at_res.py --data_root {DATA} --dataset {j['ds']} "
+        f"--protocol {j['proto']} --arch {j['arch']} --seed 0 --eval_size 512 "
+        f"--exp_name unified512 --encoder {enc}"
+    )
+
+running = {}; free = list(range(NGPU)); i = 0; ok = fail = 0
+while i < len(pending) or running:
+    while free and i < len(pending):
+        gpu = free.pop(0); j = pending[i]; i += 1
+        lf = open(f"{LOGD}/{j['tag']}.log", "w")
+        p = subprocess.Popen(["bash", "-lc", make_cmd(j, gpu)], stdout=lf, stderr=subprocess.STDOUT)
+        running[gpu] = (p, j, lf); print(f"[launch] gpu{gpu} {j['tag']}", flush=True)
+    time.sleep(8)
+    for gpu, (p, j, lf) in list(running.items()):
+        if p.poll() is not None:
+            lf.close(); okj = os.path.isfile(j["mj"]); ok += okj; fail += (not okj)
+            print(f"[finish] gpu{gpu} {j['tag']} rc={p.returncode} ok={okj}", flush=True)
+            del running[gpu]; free.append(gpu); free.sort()
+print(f"[eval512] ALL DONE ok={ok} fail={fail}", flush=True)
+print("SWIN_TRANSUNET_EVAL512_DONE", flush=True)

code/scripts/hf_update_unified512.py

@@ -0,0 +1,26 @@
+"""Push the unified-512 deliverables to HF MaybeRichard/GenSeg-Baselines (replacing the
+old 256 report): enhanced aggregate.py (code) + unified-512 summary.* + all metrics.json
+(results/). Run with the write token + the REAL HF endpoint (local env defaults to the
+hf-mirror download mirror, which does NOT accept authenticated writes).
+  HF_ENDPOINT=https://huggingface.co T=<write-token> python3 scripts/hf_update_unified512.py
+"""
+import os
+from huggingface_hub import HfApi
+
+REPO = "MaybeRichard/GenSeg-Baselines"
+LOCAL = "/home/richard/Documents/Code/ZJU/SegGen"
+api = HfApi(token=os.environ["T"], endpoint="https://huggingface.co")
+
+print("[1] code: enhanced aggregate.py")
+api.upload_file(path_or_fileobj=f"{LOCAL}/framework/report/aggregate.py", repo_id=REPO,
+                repo_type="model", path_in_repo="code/framework/report/aggregate.py",
+                commit_message="aggregate.py: unified-512 intro + per-class + Wilcoxon")
+
+print("[2] results: unified-512 summary.* + all metrics.json -> results/")
+api.upload_folder(folder_path=f"{LOCAL}/results/unified512", repo_id=REPO, repo_type="model",
+                  path_in_repo="results",
+                  allow_patterns=["**/metrics.json", "summary.html", "summary.csv",
+                                  "summary.md", "summary.tex"],
+                  commit_message="results: unified-512 (8 methods @512, per-class + significance)")
+
+print("DONE")

code/scripts/hf_upload_gensegdataset.py

@@ -0,0 +1,23 @@
+"""Upload GenSegDataset (processed_unified mirror + dataset card) to the Hugging
+Face Hub as a PRIVATE dataset repo. Storage is content-addressed, so fold-duplicated
+images collapse to unique blobs. Resumable via upload_large_folder."""
+import sys
+from huggingface_hub import HfApi
+
+REPO = "MaybeRichard/GenSegDataset"
+DATA = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+CARD = "/home/wzhang/LSC/Dataset/Segmentation/GenSegDataset_README.md"
+
+api = HfApi()
+api.create_repo(REPO, repo_type="dataset", private=True, exist_ok=True)
+print("repo ready:", REPO, flush=True)
+
+# big data upload (resumable, dedups blobs, parallel)
+api.upload_large_folder(repo_id=REPO, repo_type="dataset", folder_path=DATA)
+print("folder uploaded", flush=True)
+
+# dataset card last so it is the final README at the repo root
+api.upload_file(path_or_fileobj=CARD, path_in_repo="README.md",
+                repo_id=REPO, repo_type="dataset",
+                commit_message="add dataset card")
+print("UPLOAD_DONE", flush=True)

code/scripts/hf_upload_tars.py

@@ -0,0 +1,31 @@
+"""Recovery upload: ship GenSegDataset as ONE tar per subset (10 files) instead of
+~110k loose PNGs, to stay under HF's 128-commit/hour limit. Resets the partially
+populated repo, uploads <subset>.tar + the dataset card."""
+import os, subprocess
+from huggingface_hub import HfApi
+
+REPO = "MaybeRichard/GenSegDataset"
+DATA = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+TARS = "/home/wzhang/LSC/Dataset/Segmentation/hf_tars"
+CARD = "/home/wzhang/LSC/Dataset/Segmentation/GenSegDataset_README.md"
+
+os.makedirs(TARS, exist_ok=True)
+for ds in sorted(os.listdir(DATA)):
+    if not os.path.isdir(os.path.join(DATA, ds)):
+        continue
+    out = os.path.join(TARS, ds + ".tar")
+    if os.path.exists(out) and os.path.getsize(out) > 0:
+        print("skip (exists):", ds, flush=True); continue
+    # -h dereferences symlinks so fold-shared images are materialized into the tar
+    subprocess.run(["tar", "-chf", out, "-C", DATA, ds], check=True)
+    print("tarred %s -> %.1f MB" % (ds, os.path.getsize(out) / 1e6), flush=True)
+
+api = HfApi()
+api.delete_repo(REPO, repo_type="dataset", missing_ok=True)
+api.create_repo(REPO, repo_type="dataset", private=True, exist_ok=True)
+print("repo reset:", REPO, flush=True)
+
+api.upload_file(path_or_fileobj=CARD, path_in_repo="README.md",
+                repo_id=REPO, repo_type="dataset", commit_message="dataset card")
+api.upload_large_folder(repo_id=REPO, repo_type="dataset", folder_path=TARS)
+print("UPLOAD_DONE", flush=True)

code/scripts/p1/backbones.py

@@ -0,0 +1,44 @@
+"""build_backbone: instantiate one of {jit, pixelgen, deco, pixeldit} pixel-space
+denoisers for mask-concat conditioning. Each returns a net callable as net(x, t, y)
+-> (N, C>=img_ch, H, W); the caller slices [:, :img_ch] (backbone-agnostic decouple).
+in_channels = img_channels + cond_channels; a single dummy class (num_classes=1).
+Unified 'Base' tier (~130-150M) for the P1 backbone bake-off (P1 = native arch under a
+common flow-matching objective; perceptual/DCT/FD losses are P2 levers, not used here)."""
+import os
+import sys
+
+_SOTA = "/home/wzhang/LSC/Code/NPJ/sota"
+
+
+def _add(path):
+    if path not in sys.path:
+        sys.path.insert(0, path)
+
+
+def build_backbone(backbone: str, model_name: str, img_size: int,
+                   in_channels: int, num_classes: int = 1):
+    bk = backbone.lower()
+    if bk == "jit":
+        _add(os.path.join(_SOTA, "JiT"))
+        from model_jit import JiT_models
+        return JiT_models[model_name](input_size=img_size, in_channels=in_channels,
+                                      num_classes=num_classes)
+    if bk == "pixelgen":
+        _add(os.path.join(_SOTA, "PixelGen", "src", "models", "transformer"))
+        import importlib
+        jit = importlib.import_module("JiT")           # PixelGen's self-contained JiT.py
+        return jit.JiT_models[model_name](input_size=img_size, in_channels=in_channels,
+                                          num_classes=num_classes)
+    if bk == "deco":
+        _add(os.path.join(_SOTA, "DeCo", "src", "models", "transformer"))
+        from dit_c2i_DeCo import PixNerDiT
+        return PixNerDiT(in_channels=in_channels, patch_size=16, num_groups=12,
+                         hidden_size=768, hidden_size_x=32, num_blocks=13,
+                         num_cond_blocks=12, num_classes=num_classes)
+    if bk == "pixeldit":
+        _add(os.path.join(_SOTA, "PixelDiT"))
+        from pixdit_core.pixeldit_c2i import PixDiT
+        return PixDiT(in_channels=in_channels, num_groups=10, hidden_size=640,
+                      pixel_hidden_size=16, patch_depth=9, pixel_depth=4,
+                      patch_size=16, num_classes=num_classes)
+    raise ValueError(f"unknown backbone: {backbone}")

code/scripts/p1/fd_lever.py

@@ -0,0 +1,99 @@
+"""FD-lever ablation on the recommended P2 base (JiT): refine p1_jit_{ds} with FD loss
+-> sample -> downstream. Compares +JiT-FD vs +JiT(native) vs real. 18 jobs on GPU0-5."""
+import os, time, json, subprocess
+
+ROOT = "/home/wzhang/LSC/Code/NPJ"
+DR = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+PY = "/opt/anaconda3/envs/seggen/bin/python"
+GPUS = [0, 1, 2, 3, 4, 5]
+os.chdir(ROOT)
+LOGD = os.path.join(ROOT, "logs", "fdlever")
+os.makedirs(LOGD, exist_ok=True)
+def log(m):
+    line = f"[{time.strftime('%F %T')}] {m}"
+    open(os.path.join(LOGD, "status.md"), "a").write(line + "\n"); print(line, flush=True)
+
+DSETS = {"isic": ("medsegdb_isic2018", "holdout", 2582), "kvasir": ("kvasir_seg", "official", 800)}
+NS = [50, 100]; SEEDS = [0, 1, 2]
+jobs = {}
+def add(jid, cmd, deps=(), done_path=None, done_min=1):
+    jobs[jid] = {"cmd": cmd, "deps": list(deps), "done_path": done_path, "done_min": done_min,
+                 "state": "pending", "tries": 0, "gpu": None}
+
+for dk, (ds, proto, tot) in DSETS.items():
+    base = f"pretrained/pixdiff/p1_jit_{dk}.pt"
+    out = f"pretrained/pixdiff/p1_jitfd_{dk}.pt"
+    cmd = (f"{PY} -m framework.synth.pixdiff.train_fd --base_ckpt {base} --data_root {DR} "
+           f"--dataset {ds} --protocol {proto} --train_fraction 1.0 --epochs 150 --batch_size 16 "
+           f"--amp bf16 --fd_weight 0.5 --out_ckpt {out} --log_interval 100")
+    add(f"genfd_{dk}", cmd, done_path=os.path.join(ROOT, out))
+    for N in NS:
+        f = N / tot
+        sd = f"{DR}/{ds}/{proto}/synth_p1_jitfd_{dk}_f{N}"
+        cmd = (f"{PY} -m framework.synth.pixdiff.sample --ckpt {out} --data_root {DR} --dataset {ds} "
+               f"--protocol {proto} --train_fraction {f} --fraction_seed 0 --n_per_mask 4 --mask_aug "
+               f"--num_steps 50 --out_dir {sd}")
+        add(f"samp_jitfd_{dk}_N{N}", cmd, deps=[f"genfd_{dk}"], done_path=os.path.join(sd, "images"), done_min=N * 4)
+        for S in SEEDS:
+            exp = f"p1_jitfd_{dk}_N{N}"
+            mp = os.path.join(ROOT, f"results/{exp}/{ds}_{proto}/unet/seed{S}/metrics.json")
+            cmd = (f"{PY} framework/train.py --data_root {DR} --dataset {ds} --protocol {proto} --arch unet "
+                   f"--encoder resnet50 --aug standard --epochs 400 --train_fraction {f} --fraction_seed 0 "
+                   f"--synth_train_dir {sd} --exp_name {exp} --amp bf16 --seed {S} "
+                   f"&& {PY} framework/test.py --data_root {DR} --dataset {ds} --protocol {proto} --arch unet "
+                   f"--encoder resnet50 --aug standard --exp_name {exp} --seed {S}")
+            add(f"seg_jitfd_{dk}_N{N}_s{S}", cmd, deps=[f"samp_jitfd_{dk}_N{N}"], done_path=mp)
+
+def is_done(j):
+    p = j["done_path"]
+    if not p or not os.path.exists(p): return False
+    if os.path.isdir(p):
+        try: return len(os.listdir(p)) >= j["done_min"]
+        except OSError: return False
+    return True
+def aggregate():
+    res = {}
+    for dk, (ds, proto, tot) in DSETS.items():
+        for N in NS:
+            exp = f"p1_jitfd_{dk}_N{N}"; ious = []; dices = []
+            for S in SEEDS:
+                mp = f"results/{exp}/{ds}_{proto}/unet/seed{S}/metrics.json"
+                if os.path.exists(mp):
+                    try:
+                        m = json.load(open(mp))["metrics"]; ious.append(m["iou_mean"]); dices.append(m["dice_mean"])
+                    except Exception: pass
+            if ious:
+                res[f"{dk}_N{N}_jitfd"] = {"iou_mean": sum(ious) / len(ious), "dice_mean": sum(dices) / len(dices),
+                                           "n_seeds": len(ious), "iou_seeds": ious}
+    json.dump(res, open(os.path.join(LOGD, "fd_results.json"), "w"), indent=2)
+
+for jid, j in jobs.items():
+    if is_done(j): j["state"] = "done"
+def deps_done(j): return all(jobs[d]["state"] == "done" for d in j["deps"])
+running = {}; free = set(GPUS); last = 0
+log(f"START {len(jobs)} jobs on {GPUS} ({sum(1 for j in jobs.values() if j['state']=='done')} pre-done)")
+while True:
+    if all(j["state"] in ("done", "failed") for j in jobs.values()): break
+    for jid, j in jobs.items():
+        if not free: break
+        if j["state"] == "pending" and deps_done(j):
+            if is_done(j): j["state"] = "done"; continue
+            g = free.pop()
+            env = dict(os.environ, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=str(g),
+                       TORCHDYNAMO_DISABLE="1", PYTHONPATH=".", OMP_NUM_THREADS="4")
+            lf = open(os.path.join(LOGD, jid + ".log"), "a")
+            p = subprocess.Popen(j["cmd"], shell=True, env=env, stdout=lf, stderr=subprocess.STDOUT, cwd=ROOT)
+            running[g] = (jid, p, lf); j["state"] = "running"; j["gpu"] = g; j["tries"] += 1
+            log(f"LAUNCH {jid} GPU{g} try{j['tries']}")
+    for g, (jid, p, lf) in list(running.items()):
+        rc = p.poll()
+        if rc is None: continue
+        lf.close(); del running[g]; free.add(g); j = jobs[jid]
+        if is_done(j): j["state"] = "done"; log(f"DONE {jid}")
+        elif j["tries"] < 2: j["state"] = "pending"; log(f"RETRY {jid} rc={rc}")
+        else: j["state"] = "failed"; log(f"FAILED {jid} rc={rc}")
+    if time.time() - last > 180:
+        cnt = {s: sum(1 for j in jobs.values() if j["state"] == s) for s in ("done", "running", "pending", "failed")}
+        log(f"SUMMARY {cnt}"); aggregate(); last = time.time()
+    time.sleep(10)
+aggregate(); log("ALL DONE"); print("FD_LEVER_DONE", flush=True)

code/scripts/p1/fd_results.json

@@ -0,0 +1,42 @@
+{
+  "isic_N50_jitfd": {
+    "iou_mean": 0.807111643913793,
+    "dice_mean": 0.8785879691414072,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8075343832113462,
+      0.8052902472456404,
+      0.8085103012843926
+    ]
+  },
+  "isic_N100_jitfd": {
+    "iou_mean": 0.8194000537587307,
+    "dice_mean": 0.8883052260740664,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.815320694079291,
+      0.8190540649488038,
+      0.8238254022480974
+    ]
+  },
+  "kvasir_N50_jitfd": {
+    "iou_mean": 0.7743735031913296,
+    "dice_mean": 0.8502822525848202,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.7818383912178852,
+      0.7792812954940015,
+      0.7620008228621021
+    ]
+  },
+  "kvasir_N100_jitfd": {
+    "iou_mean": 0.8237618615778238,
+    "dice_mean": 0.8922327796820345,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8173440030838561,
+      0.8195464262764907,
+      0.8343951553731246
+    ]
+  }
+}

code/scripts/p1/fid_and_viz.py

@@ -0,0 +1,130 @@
+"""FID (1-2k samples) per backbone x dataset + clear same-mask aligned viz.
+A) fid-sample: train_fraction=1.0, mask_aug, n_per_mask -> ~1.6-2.6k synth; FID vs real train.
+B) align-sample: f50 masks, NO aug, 1/mask -> all backbones share identical real masks -> aligned grid.
+Then pytorch_fid per pair + build [mask|real|4 backbones] grids. GPU0-5 pool."""
+import os, time, json, re, subprocess
+import numpy as np
+import matplotlib; matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from PIL import Image
+
+ROOT = "/home/wzhang/LSC/Code/NPJ"; DR = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+PY = "/opt/anaconda3/envs/seggen/bin/python"; GPUS = [0, 1, 2, 3, 4, 5]
+os.chdir(ROOT); LOGD = os.path.join(ROOT, "logs", "fidviz"); os.makedirs(LOGD, exist_ok=True)
+def log(m):
+    line = f"[{time.strftime('%F %T')}] {m}"; open(os.path.join(LOGD, "status.md"), "a").write(line + "\n"); print(line, flush=True)
+# (ds, proto, total, npm_for_fid)
+DSETS = {"isic": ("medsegdb_isic2018", "holdout", 2582, 1),
+         "kvasir": ("kvasir_seg", "official", 800, 2),
+         "busi": ("busi", "fold01", 545, 3)}
+BKS = ["jit", "pixelgen", "deco", "pixeldit"]; LAB = {"jit": "JiT", "pixelgen": "PixelGen", "deco": "DeCo", "pixeldit": "PixelDiT"}
+jobs = {}
+def add(jid, cmd, deps=(), done_path=None, done_min=1):
+    jobs[jid] = {"cmd": cmd, "deps": list(deps), "done_path": done_path, "done_min": done_min, "state": "pending", "tries": 0, "gpu": None}
+
+for bk in BKS:
+    for dk, (ds, proto, tot, npm) in DSETS.items():
+        ck = f"pretrained/pixdiff/p1_{bk}_{dk}.pt"
+        fsd = f"{DR}/{ds}/{proto}/synth_fid_{bk}_{dk}"
+        add(f"fidsamp_{bk}_{dk}",
+            f"{PY} -m framework.synth.pixdiff.sample --ckpt {ck} --data_root {DR} --dataset {ds} --protocol {proto} "
+            f"--train_fraction 1.0 --fraction_seed 0 --n_per_mask {npm} --mask_aug --num_steps 50 --out_dir {fsd}",
+            done_path=os.path.join(fsd, "images"), done_min=int(0.8 * tot * npm))
+        real = f"{DR}/{ds}/{proto}/train/images"
+        flog = os.path.join(LOGD, f"fid_{bk}_{dk}.log"); fok = os.path.join(LOGD, f"fid_{bk}_{dk}.ok")
+        add(f"fid_{bk}_{dk}",
+            f"{PY} -m pytorch_fid {real} {fsd}/images --device cuda --batch-size 64 > {flog} 2>&1 && grep -q FID {flog} && touch {fok}",
+            deps=[f"fidsamp_{bk}_{dk}"], done_path=fok)
+        f50 = 50 / tot; asd = f"{DR}/{ds}/{proto}/synth_align_{bk}_{dk}"
+        add(f"alignsamp_{bk}_{dk}",
+            f"{PY} -m framework.synth.pixdiff.sample --ckpt {ck} --data_root {DR} --dataset {ds} --protocol {proto} "
+            f"--train_fraction {f50} --fraction_seed 0 --n_per_mask 1 --num_steps 50 --out_dir {asd}",
+            done_path=os.path.join(asd, "images"), done_min=40)
+
+def is_done(j):
+    p = j["done_path"]
+    if not p or not os.path.exists(p): return False
+    if os.path.isdir(p):
+        try: return len(os.listdir(p)) >= j["done_min"]
+        except OSError: return False
+    return True
+for jid, j in jobs.items():
+    if is_done(j): j["state"] = "done"
+def deps_done(j): return all(jobs[d]["state"] == "done" for d in j["deps"])
+running = {}; free = set(GPUS); last = 0
+log(f"START {len(jobs)} jobs on {GPUS}")
+while True:
+    if all(j["state"] in ("done", "failed") for j in jobs.values()): break
+    for jid, j in jobs.items():
+        if not free: break
+        if j["state"] == "pending" and deps_done(j):
+            if is_done(j): j["state"] = "done"; continue
+            g = free.pop()
+            env = dict(os.environ, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=str(g), TORCHDYNAMO_DISABLE="1", PYTHONPATH=".", OMP_NUM_THREADS="4")
+            lf = open(os.path.join(LOGD, jid + ".log"), "a")
+            p = subprocess.Popen(j["cmd"], shell=True, env=env, stdout=lf, stderr=subprocess.STDOUT, cwd=ROOT)
+            running[g] = (jid, p, lf); j["state"] = "running"; j["gpu"] = g; j["tries"] += 1
+            log(f"LAUNCH {jid} GPU{g} try{j['tries']}")
+    for g, (jid, p, lf) in list(running.items()):
+        rc = p.poll()
+        if rc is None: continue
+        lf.close(); del running[g]; free.add(g); j = jobs[jid]
+        if is_done(j): j["state"] = "done"; log(f"DONE {jid}")
+        elif j["tries"] < 2: j["state"] = "pending"; log(f"RETRY {jid} rc={rc}")
+        else: j["state"] = "failed"; log(f"FAILED {jid} rc={rc}")
+    if time.time() - last > 180:
+        cnt = {s: sum(1 for j in jobs.values() if j["state"] == s) for s in ("done", "running", "pending", "failed")}; log(f"SUMMARY {cnt}"); last = time.time()
+    time.sleep(8)
+
+# ---- parse FID ----
+fid = {}
+for bk in BKS:
+    for dk in DSETS:
+        lg = os.path.join(LOGD, f"fid_{bk}_{dk}.log")
+        if os.path.exists(lg):
+            m = re.findall(r"FID:\s*([0-9.]+)", open(lg).read())
+            if m: fid[f"{dk}_{bk}"] = float(m[-1])
+json.dump(fid, open(os.path.join(LOGD, "fid_results.json"), "w"), indent=2)
+log(f"FID: {fid}")
+
+# ---- aligned grids ([mask | real | 4 backbones], same real mask per column) ----
+def rgb(p): return np.asarray(Image.open(p).convert("RGB").resize((256, 256)))
+def gray(p): return np.asarray(Image.open(p).convert("L").resize((256, 256)))
+def fmap(d):
+    p = os.path.join(d, "images"); m = {}
+    if os.path.isdir(p):
+        for f in sorted(os.listdir(p)):
+            if f.endswith(".png"): m.setdefault(f[:-4].split("__")[0], os.path.join(p, f))
+    return m
+for dk, (ds, proto, tot, npm) in DSETS.items():
+    base = f"{DR}/{ds}/{proto}"; ri, rm = f"{base}/train/images", f"{base}/train/masks"
+    maps = {bk: fmap(f"{base}/synth_align_{bk}_{dk}") for bk in BKS}
+    common = set(os.path.splitext(f)[0] for f in os.listdir(ri) if f.endswith(".png"))
+    for bk in BKS: common &= set(maps[bk].keys())
+    common = sorted(common); ncol = min(6, len(common))
+    if ncol == 0: continue
+    idx = [round(i * (len(common) - 1) / (ncol - 1)) for i in range(ncol)] if ncol > 1 else [0]
+    cases = [common[i] for i in idx]
+    rows = [("Conditioning mask", "mask"), ("Real image", "real")] + [(LAB[bk], bk) for bk in BKS]
+    fig, ax = plt.subplots(len(rows), ncol, figsize=(ncol * 1.9, len(rows) * 1.95))
+    for r, (labr, kind) in enumerate(rows):
+        for c, bs in enumerate(cases):
+            a = ax[r][c]
+            try:
+                mk = gray(f"{rm}/{bs}.png")
+                if kind == "mask":
+                    a.imshow(mk, cmap="gray")
+                elif kind == "real":
+                    a.imshow(rgb(f"{ri}/{bs}.png")); a.contour((mk > 127).astype(float), levels=[0.5], colors=["#19f04b"], linewidths=1.0)
+                else:
+                    a.imshow(rgb(maps[kind][bs])); a.contour((mk > 127).astype(float), levels=[0.5], colors=["#19f04b"], linewidths=1.0)
+            except Exception:
+                a.imshow(np.ones((256, 256, 3))); a.text(0.5, 0.5, "n/a", ha="center", va="center", transform=a.transAxes, fontsize=8)
+            a.set_xticks([]); a.set_yticks([])
+            for s in a.spines.values(): s.set_visible(False)
+            if c == 0: a.set_ylabel(labr, fontsize=10, rotation=90, va="center", labelpad=8, color=("#111" if r < 2 else "#1a3b8b"))
+    fig.suptitle(f"{dk.upper()} — same-mask aligned: every backbone generates the SAME real mask (row 1)\n"
+                 f"Row2=real image; rows 3-6=each backbone's mask-conditioned synthesis (green=that mask). Proves mask guidance.", fontsize=10)
+    plt.tight_layout(rect=[0.02, 0, 1, 0.94]); plt.savefig(f"/tmp/p1_aligned_{dk}.png", dpi=145, bbox_inches="tight", facecolor="white")
+    log(f"aligned grid saved /tmp/p1_aligned_{dk}.png")
+log("ALL DONE"); print("FIDVIZ_DONE", flush=True)

code/scripts/p1/fid_fixed.py

@@ -0,0 +1,37 @@
+"""Corrected FID: resize REAL train images to 256 (synth already 256) so pytorch_fid's
+collate doesn't choke on variable native sizes (Kvasir/BUSI). FID(real256, synth) per pair."""
+import os, re, json, subprocess
+from PIL import Image
+
+ROOT = "/home/wzhang/LSC/Code/NPJ"; DR = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+PY = "/opt/anaconda3/envs/seggen/bin/python"
+DSETS = {"isic": ("medsegdb_isic2018", "holdout"), "kvasir": ("kvasir_seg", "official"), "busi": ("busi", "fold01")}
+BKS = ["jit", "pixelgen", "deco", "pixeldit"]
+
+real256 = {}
+for dk, (ds, proto) in DSETS.items():
+    src = f"{DR}/{ds}/{proto}/train/images"; dst = f"/tmp/real256_{dk}"; os.makedirs(dst, exist_ok=True)
+    for f in os.listdir(src):
+        if f.lower().endswith((".png", ".jpg", ".jpeg")):
+            o = f"{dst}/{os.path.splitext(f)[0]}.png"
+            if not os.path.exists(o):
+                Image.open(f"{src}/{f}").convert("RGB").resize((256, 256)).save(o)
+    real256[dk] = dst
+    print(f"[resize] real {dk}: {len(os.listdir(dst))} imgs", flush=True)
+
+fid = {}
+for bk in BKS:
+    for dk, (ds, proto) in DSETS.items():
+        synth = f"{DR}/{ds}/{proto}/synth_fid_{bk}_{dk}/images"
+        if not os.path.isdir(synth) or len(os.listdir(synth)) < 100:
+            print(f"[skip] {dk} {bk}: synth missing/small", flush=True); continue
+        env = dict(os.environ, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES="0")
+        r = subprocess.run([PY, "-m", "pytorch_fid", real256[dk], synth, "--device", "cuda", "--batch-size", "50"],
+                           capture_output=True, text=True, env=env)
+        m = re.findall(r"FID:\s*([0-9.]+)", r.stdout + r.stderr)
+        if m:
+            fid[f"{dk}_{bk}"] = round(float(m[-1]), 2); print(f"[FID] {dk} {bk} = {m[-1]}", flush=True)
+        else:
+            print(f"[FAIL] {dk} {bk}: {(r.stderr or r.stdout)[-300:]}", flush=True)
+        json.dump(fid, open(f"{ROOT}/logs/fidviz/fid_results.json", "w"), indent=2)
+print("FID_FIXED_DONE", json.dumps(fid), flush=True)

code/scripts/p1/fid_results.json

@@ -0,0 +1,14 @@
+{
+  "isic_jit": 107.96,
+  "kvasir_jit": 109.09,
+  "busi_jit": 139.28,
+  "isic_pixelgen": 107.63,
+  "kvasir_pixelgen": 118.44,
+  "busi_pixelgen": 142.91,
+  "isic_deco": 141.84,
+  "kvasir_deco": 114.43,
+  "busi_deco": 174.36,
+  "isic_pixeldit": 109.87,
+  "kvasir_pixeldit": 112.83,
+  "busi_pixeldit": 151.82
+}

code/scripts/p1/gen_prdc.json

@@ -0,0 +1,74 @@
+{
+  "isic_jit": {
+    "precision": 0.301,
+    "recall": 0.107,
+    "density": 0.122,
+    "coverage": 0.146
+  },
+  "kvasir_jit": {
+    "precision": 0.402,
+    "recall": 0.021,
+    "density": 0.169,
+    "coverage": 0.294
+  },
+  "busi_jit": {
+    "precision": 0.271,
+    "recall": 0.174,
+    "density": 0.103,
+    "coverage": 0.332
+  },
+  "isic_pixelgen": {
+    "precision": 0.319,
+    "recall": 0.101,
+    "density": 0.122,
+    "coverage": 0.14
+  },
+  "kvasir_pixelgen": {
+    "precision": 0.388,
+    "recall": 0.009,
+    "density": 0.159,
+    "coverage": 0.242
+  },
+  "busi_pixelgen": {
+    "precision": 0.275,
+    "recall": 0.261,
+    "density": 0.093,
+    "coverage": 0.29
+  },
+  "isic_deco": {
+    "precision": 0.14,
+    "recall": 0.04,
+    "density": 0.04,
+    "coverage": 0.048
+  },
+  "kvasir_deco": {
+    "precision": 0.287,
+    "recall": 0.01,
+    "density": 0.11,
+    "coverage": 0.195
+  },
+  "busi_deco": {
+    "precision": 0.081,
+    "recall": 0.042,
+    "density": 0.023,
+    "coverage": 0.088
+  },
+  "isic_pixeldit": {
+    "precision": 0.213,
+    "recall": 0.065,
+    "density": 0.077,
+    "coverage": 0.106
+  },
+  "kvasir_pixeldit": {
+    "precision": 0.324,
+    "recall": 0.027,
+    "density": 0.123,
+    "coverage": 0.201
+  },
+  "busi_pixeldit": {
+    "precision": 0.245,
+    "recall": 0.051,
+    "density": 0.077,
+    "coverage": 0.182
+  }
+}

code/scripts/p1/gen_prdc.py

@@ -0,0 +1,58 @@
+"""Generation-side Precision/Recall (Kynkaanniemi) + Density/Coverage (Naeem) on
+InceptionV3 (pytorch-fid) features. Precision=fidelity (fake in real manifold),
+Recall=diversity/coverage (real covered by fake). No sklearn: kNN via torch.cdist."""
+import os, sys, json, random
+import numpy as np, torch
+from PIL import Image
+sys.path.insert(0, "/home/wzhang/LSC/Code/NPJ")
+from framework.synth.pixdiff.fd_loss import InceptionFeatures
+
+DR = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+DSETS = {"isic": ("medsegdb_isic2018", "holdout"), "kvasir": ("kvasir_seg", "official"), "busi": ("busi", "fold01")}
+BKS = ["jit", "pixelgen", "deco", "pixeldit"]
+dev = "cuda"; CAP = 2000; K = 5
+inc = InceptionFeatures().to(dev).eval()
+
+def feats(d, cap=CAP):
+    fs = sorted(f for f in os.listdir(d) if f.lower().endswith((".png", ".jpg", ".jpeg")))
+    if len(fs) > cap:
+        random.seed(0); fs = random.sample(fs, cap)
+    out = []
+    for i in range(0, len(fs), 64):
+        b = []
+        for f in fs[i:i + 64]:
+            im = Image.open(os.path.join(d, f)).convert("RGB").resize((256, 256))
+            b.append(torch.from_numpy(np.asarray(im)).permute(2, 0, 1).float() / 255.)
+        with torch.no_grad():
+            out.append(inc(torch.stack(b).to(dev)).cpu())
+    return torch.cat(out)
+
+def knn_radius(X, k):
+    d = torch.cdist(X, X); d.fill_diagonal_(float("inf")); return d.kthvalue(k, dim=1).values
+
+def prdc(R, F, k=K):
+    R, F = R.to(dev), F.to(dev)
+    rr = knn_radius(R, k); ff = knn_radius(F, k); drf = torch.cdist(R, F)
+    prec = (drf <= rr[:, None]).any(0).float().mean().item()
+    rec = (drf <= ff[None, :]).any(1).float().mean().item()
+    dens = ((drf <= rr[:, None]).sum(0).float().mean() / k).item()
+    cov = (drf <= rr[:, None]).any(1).float().mean().item()
+    return prec, rec, dens, cov
+
+realf = {}
+for dk, (ds, proto) in DSETS.items():
+    realf[dk] = feats(f"{DR}/{ds}/{proto}/train/images")
+    print(f"[real] {dk}: {realf[dk].shape}", flush=True)
+
+res = {}
+for bk in BKS:
+    for dk, (ds, proto) in DSETS.items():
+        sd = f"{DR}/{ds}/{proto}/synth_fid_{bk}_{dk}/images"
+        if not os.path.isdir(sd):
+            print(f"[skip] {dk} {bk}"); continue
+        F = feats(sd)
+        p, r, de, c = prdc(realf[dk], F)
+        res[f"{dk}_{bk}"] = {"precision": round(p, 3), "recall": round(r, 3), "density": round(de, 3), "coverage": round(c, 3)}
+        print(f"[PRDC] {dk} {bk}: {res[f'{dk}_{bk}']}", flush=True)
+        json.dump(res, open("/home/wzhang/LSC/Code/NPJ/logs/fidviz/gen_prdc.json", "w"), indent=2)
+print("PRDC_DONE", flush=True)

code/scripts/p1/make_jit_vs_fd.py

@@ -0,0 +1,65 @@
+"""Same-mask JiT(native) vs JiT-FD comparison: does FD-perceptual sharpen the synth?
+Samples JiT-FD on the SAME no-aug f50 masks the native JiT align-set used, builds
+[mask | real | JiT native | JiT-FD] grids for ISIC + Kvasir."""
+import os, subprocess
+import numpy as np
+import matplotlib; matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from PIL import Image
+
+ROOT = "/home/wzhang/LSC/Code/NPJ"; DR = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+PY = "/opt/anaconda3/envs/seggen/bin/python"
+DSETS = {"isic": ("medsegdb_isic2018", "holdout", 2582), "kvasir": ("kvasir_seg", "official", 800)}
+
+def sample(ckpt, ds, proto, frac, out):
+    if os.path.isdir(out + "/images") and len(os.listdir(out + "/images")) >= 40:
+        print(f"[skip-sample] {out} exists", flush=True); return
+    env = dict(os.environ, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES="0",
+               TORCHDYNAMO_DISABLE="1", PYTHONPATH=".", OMP_NUM_THREADS="4")
+    subprocess.run([PY, "-m", "framework.synth.pixdiff.sample", "--ckpt", ckpt, "--data_root", DR,
+                    "--dataset", ds, "--protocol", proto, "--train_fraction", str(frac),
+                    "--fraction_seed", "0", "--n_per_mask", "1", "--num_steps", "50", "--out_dir", out],
+                   env=env, cwd=ROOT, check=True)
+    print(f"[sampled] {out}", flush=True)
+
+def fmap(d):
+    p = os.path.join(d, "images"); m = {}
+    if os.path.isdir(p):
+        for f in sorted(os.listdir(p)):
+            if f.endswith(".png"): m.setdefault(f[:-4].split("__")[0], os.path.join(p, f))
+    return m
+def rgb(p): return np.asarray(Image.open(p).convert("RGB").resize((256, 256)))
+def gray(p): return np.asarray(Image.open(p).convert("L").resize((256, 256)))
+
+for dk, (ds, proto, tot) in DSETS.items():
+    f50 = 50 / tot
+    fd_out = f"{DR}/{ds}/{proto}/synth_alignfd_jitfd_{dk}"
+    sample(f"pretrained/pixdiff/p1_jitfd_{dk}.pt", ds, proto, f50, fd_out)
+    base = f"{DR}/{ds}/{proto}"; ri, rm = f"{base}/train/images", f"{base}/train/masks"
+    nat = fmap(f"{base}/synth_align_jit_{dk}"); fd = fmap(fd_out)
+    common = set(os.path.splitext(f)[0] for f in os.listdir(ri) if f.endswith(".png")) & set(nat) & set(fd)
+    common = sorted(common); ncol = min(6, len(common))
+    idx = [round(i * (len(common) - 1) / (ncol - 1)) for i in range(ncol)] if ncol > 1 else [0]
+    cases = [common[i] for i in idx]
+    rows = [("Conditioning mask", "mask"), ("Real", "real"), ("JiT (native, P1)", nat), ("JiT-FD (FD-感知)", fd)]
+    fig, ax = plt.subplots(len(rows), ncol, figsize=(ncol * 2.1, len(rows) * 2.15))
+    for r, (lab, src) in enumerate(rows):
+        for c, bs in enumerate(cases):
+            a = ax[r][c]
+            try:
+                mk = gray(f"{rm}/{bs}.png")
+                if src == "mask": a.imshow(mk, cmap="gray")
+                elif src == "real": a.imshow(rgb(f"{ri}/{bs}.png"))
+                else: a.imshow(rgb(src[bs]))
+                if src not in ("mask",): a.contour((mk > 127).astype(float), levels=[0.5], colors=["#19f04b"], linewidths=0.9)
+            except Exception:
+                a.imshow(np.ones((256, 256, 3))); a.text(0.5, 0.5, "n/a", ha="center", va="center", transform=a.transAxes)
+            a.set_xticks([]); a.set_yticks([])
+            for s in a.spines.values(): s.set_visible(False)
+            if c == 0: a.set_ylabel(lab, fontsize=11, rotation=90, va="center", labelpad=8,
+                                    color=("#111" if r < 2 else "#1a3b8b"), fontweight=("bold" if r == 3 else "normal"))
+    fig.suptitle(f"{dk.upper()} — 原生 JiT vs JiT-FD(同掩码):FD-感知精修是否更锐?", fontsize=12)
+    plt.tight_layout(rect=[0.03, 0, 1, 0.95])
+    out = f"/tmp/jit_vs_fd_{dk}.png"; plt.savefig(out, dpi=150, bbox_inches="tight", facecolor="white")
+    print(f"[grid] {out}", flush=True)
+print("JIT_VS_FD_DONE", flush=True)

code/scripts/p1/p1_busi_master.py

@@ -0,0 +1,166 @@
+"""P1 master orchestrator: DAG scheduler over GPU0-5 for the backbone bake-off.
+Phases: A) 8 generators (4 backbones x 2 datasets, amortized, 50k steps)
+        B) 16 sampling jobs (per gen x N in {50,100}, mask_aug n_per_mask=4)
+        C) 60 downstream seg runs (real + 4 backbones) x 2 ds x 2 N x 3 seeds
+Single GPU per job (no DDP needed: 84 independent jobs). Retry-once on failure.
+Resumable (skips done outputs). Rolling aggregate -> logs/p1master/p1_results.json."""
+import os, sys, time, json, subprocess, statistics as st
+
+ROOT = "/home/wzhang/LSC/Code/NPJ"
+DR = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+PY = "/opt/anaconda3/envs/seggen/bin/python"
+GPUS = [0, 1, 2, 3, 4, 5]
+os.chdir(ROOT)
+LOGD = os.path.join(ROOT, "logs", "p1busi")
+os.makedirs(LOGD, exist_ok=True)
+
+def log(m):
+    line = f"[{time.strftime('%F %T')}] {m}"
+    with open(os.path.join(LOGD, "status.md"), "a") as f:
+        f.write(line + "\n")
+    print(line, flush=True)
+
+DSETS = {"busi": ("busi", "fold01", 545)}
+BKS = ["jit", "pixelgen", "deco", "pixeldit"]
+NS = [50, 100]
+SEEDS = [0, 1, 2]
+
+jobs = {}
+def add(jid, cmd, deps=(), done_path=None, done_min=1):
+    jobs[jid] = {"cmd": cmd, "deps": list(deps), "done_path": done_path,
+                 "done_min": done_min, "state": "pending", "tries": 0, "gpu": None}
+
+# Phase A: generators
+for bk in BKS:
+    for dk, (ds, proto, tot) in DSETS.items():
+        out = f"pretrained/pixdiff/p1_{bk}_{dk}.pt"
+        cmd = (f"{PY} -m framework.synth.pixdiff.train --data_root {DR} --dataset {ds} "
+               f"--protocol {proto} --backbone {bk} --img_size 256 --batch_size 16 "
+               f"--epochs 100000 --max_steps 50000 --lr 1e-4 --amp bf16 "
+               f"--train_fraction 1.0 --fraction_seed 0 --out_ckpt {out} --log_interval 500")
+        add(f"gen_{bk}_{dk}", cmd, done_path=os.path.join(ROOT, out))
+
+# Phase B: sampling
+for bk in BKS:
+    for dk, (ds, proto, tot) in DSETS.items():
+        ck = f"pretrained/pixdiff/p1_{bk}_{dk}.pt"
+        for N in NS:
+            f = N / tot
+            sd = f"{DR}/{ds}/{proto}/synth_p1_{bk}_{dk}_f{N}"
+            cmd = (f"{PY} -m framework.synth.pixdiff.sample --ckpt {ck} --data_root {DR} "
+                   f"--dataset {ds} --protocol {proto} --train_fraction {f} --fraction_seed 0 "
+                   f"--n_per_mask 4 --mask_aug --num_steps 50 --out_dir {sd}")
+            add(f"samp_{bk}_{dk}_N{N}", cmd, deps=[f"gen_{bk}_{dk}"],
+                done_path=os.path.join(sd, "images"), done_min=N * 4)
+
+# Phase C: downstream
+def mpath(exp, ds, proto, S):
+    return os.path.join(ROOT, f"results/{exp}/{ds}_{proto}/unet/seed{S}/metrics.json")
+
+def seg_cmd(ds, proto, f, exp, S, synth=None):
+    base = (f"{PY} framework/train.py --data_root {DR} --dataset {ds} --protocol {proto} "
+            f"--arch unet --encoder resnet50 --aug standard --epochs 400 "
+            f"--train_fraction {f} --fraction_seed 0 --exp_name {exp} --amp bf16 --seed {S}")
+    if synth:
+        base += f" --synth_train_dir {synth}"
+    test = (f"{PY} framework/test.py --data_root {DR} --dataset {ds} --protocol {proto} "
+            f"--arch unet --encoder resnet50 --aug standard --exp_name {exp} --seed {S}")
+    return base + " && " + test
+
+for dk, (ds, proto, tot) in DSETS.items():
+    for N in NS:
+        f = N / tot
+        for S in SEEDS:
+            exp = f"p1_real_{dk}_N{N}"
+            add(f"seg_real_{dk}_N{N}_s{S}", seg_cmd(ds, proto, f, exp, S),
+                done_path=mpath(exp, ds, proto, S))
+        for bk in BKS:
+            sd = f"{DR}/{ds}/{proto}/synth_p1_{bk}_{dk}_f{N}"
+            for S in SEEDS:
+                exp = f"p1_{bk}_{dk}_N{N}"
+                add(f"seg_{bk}_{dk}_N{N}_s{S}", seg_cmd(ds, proto, f, exp, S, synth=sd),
+                    deps=[f"samp_{bk}_{dk}_N{N}"], done_path=mpath(exp, ds, proto, S))
+
+def is_done(j):
+    p = j["done_path"]
+    if not p or not os.path.exists(p):
+        return False
+    if os.path.isdir(p):
+        try:
+            return len(os.listdir(p)) >= j["done_min"]
+        except OSError:
+            return False
+    return True
+
+def aggregate():
+    res = {}
+    for dk, (ds, proto, tot) in DSETS.items():
+        for N in NS:
+            for arm in ["real"] + BKS:
+                exp = f"p1_{arm}_{dk}_N{N}"
+                ious, dices = [], []
+                for S in SEEDS:
+                    mp = mpath(exp, ds, proto, S)
+                    if os.path.exists(mp):
+                        try:
+                            m = json.load(open(mp))["metrics"]
+                            ious.append(m["iou_mean"]); dices.append(m["dice_mean"])
+                        except Exception:
+                            pass
+                if ious:
+                    res[f"{dk}_N{N}_{arm}"] = {
+                        "iou_mean": sum(ious) / len(ious), "dice_mean": sum(dices) / len(dices),
+                        "n_seeds": len(ious), "iou_seeds": ious, "dice_seeds": dices}
+    json.dump(res, open(os.path.join(LOGD, "p1_results.json"), "w"), indent=2)
+
+for jid, j in jobs.items():
+    if is_done(j):
+        j["state"] = "done"
+def deps_done(j):
+    return all(jobs[d]["state"] == "done" for d in j["deps"])
+
+running = {}
+free = set(GPUS)
+MAXTRIES = 2
+log(f"START {len(jobs)} jobs on GPUs {GPUS} ({sum(1 for j in jobs.values() if j['state']=='done')} pre-done)")
+last_summary = 0
+while True:
+    if all(j["state"] in ("done", "failed") for j in jobs.values()):
+        break
+    for jid, j in jobs.items():
+        if not free:
+            break
+        if j["state"] == "pending" and deps_done(j):
+            if is_done(j):
+                j["state"] = "done"; continue
+            g = free.pop()
+            env = dict(os.environ, CUDA_DEVICE_ORDER="PCI_BUS_ID",
+                       CUDA_VISIBLE_DEVICES=str(g), TORCHDYNAMO_DISABLE="1",
+                       PYTHONPATH=".", OMP_NUM_THREADS="4")
+            lf = open(os.path.join(LOGD, jid + ".log"), "a")
+            p = subprocess.Popen(j["cmd"], shell=True, env=env, stdout=lf,
+                                 stderr=subprocess.STDOUT, cwd=ROOT)
+            running[g] = (jid, p, lf); j["state"] = "running"; j["gpu"] = g; j["tries"] += 1
+            log(f"LAUNCH {jid} GPU{g} try{j['tries']}")
+    for g, (jid, p, lf) in list(running.items()):
+        rc = p.poll()
+        if rc is None:
+            continue
+        lf.close(); del running[g]; free.add(g)
+        j = jobs[jid]
+        if is_done(j):
+            j["state"] = "done"; log(f"DONE {jid} rc={rc}")
+        elif j["tries"] < MAXTRIES:
+            j["state"] = "pending"; log(f"RETRY {jid} rc={rc}")
+        else:
+            j["state"] = "failed"; log(f"FAILED {jid} rc={rc}")
+    if time.time() - last_summary > 300:
+        cnt = {s: sum(1 for j in jobs.values() if j["state"] == s)
+               for s in ("done", "running", "pending", "failed")}
+        log(f"SUMMARY {cnt} | running={sorted(j['gpu'] for j in jobs.values() if j['state']=='running')}")
+        aggregate(); last_summary = time.time()
+    time.sleep(10)
+
+aggregate()
+log("ALL DONE")
+print("P1_MASTER_DONE", flush=True)

code/scripts/p1/p1_busi_results.json

@@ -0,0 +1,152 @@
+{
+  "busi_N50_real": {
+    "iou_mean": 0.5264299887746967,
+    "dice_mean": 0.6150173362667782,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.5264031863116967,
+      0.5078566908181252,
+      0.5450300891942683
+    ],
+    "dice_seeds": [
+      0.6158220746914147,
+      0.5943401974078463,
+      0.6348897367010737
+    ]
+  },
+  "busi_N50_jit": {
+    "iou_mean": 0.6034184804314947,
+    "dice_mean": 0.6819841561663282,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.6151347744667092,
+      0.5876539837190357,
+      0.6074666831087394
+    ],
+    "dice_seeds": [
+      0.689949667483754,
+      0.6696459718689409,
+      0.6863568291462898
+    ]
+  },
+  "busi_N50_pixelgen": {
+    "iou_mean": 0.5931307319807805,
+    "dice_mean": 0.6689307916999434,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.5920417792314931,
+      0.5799457559831734,
+      0.6074046607276752
+    ],
+    "dice_seeds": [
+      0.666457437158763,
+      0.6579920482874959,
+      0.6823428896535711
+    ]
+  },
+  "busi_N50_deco": {
+    "iou_mean": 0.5979877144880544,
+    "dice_mean": 0.6759540945542658,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.5926858046978442,
+      0.5927788031039587,
+      0.6084985356623608
+    ],
+    "dice_seeds": [
+      0.6700259199200342,
+      0.6746954264901631,
+      0.6831409372526002
+    ]
+  },
+  "busi_N50_pixeldit": {
+    "iou_mean": 0.607193388275135,
+    "dice_mean": 0.6873956052246571,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.6123619477064693,
+      0.611641987503641,
+      0.5975762296152947
+    ],
+    "dice_seeds": [
+      0.6897889057784018,
+      0.6909085054783096,
+      0.6814894044172604
+    ]
+  },
+  "busi_N100_real": {
+    "iou_mean": 0.5901633542778459,
+    "dice_mean": 0.672577797062739,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.596288318471246,
+      0.5925297498972647,
+      0.581671994465027
+    ],
+    "dice_seeds": [
+      0.6805305551414621,
+      0.6720643475113856,
+      0.6651384885353694
+    ]
+  },
+  "busi_N100_jit": {
+    "iou_mean": 0.638692683336804,
+    "dice_mean": 0.7151948555339698,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.638767115763446,
+      0.6464336918630936,
+      0.6308772423838722
+    ],
+    "dice_seeds": [
+      0.7157337836019346,
+      0.7221307950206755,
+      0.7077199879792992
+    ]
+  },
+  "busi_N100_pixelgen": {
+    "iou_mean": 0.6466251026809454,
+    "dice_mean": 0.7228950392456911,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.6475516801145157,
+      0.6421441721654421,
+      0.6501794557628784
+    ],
+    "dice_seeds": [
+      0.7219398382769914,
+      0.7182700866504312,
+      0.7284751928096508
+    ]
+  },
+  "busi_N100_deco": {
+    "iou_mean": 0.6285762758560305,
+    "dice_mean": 0.7061591091011407,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.6474242343217572,
+      0.6371748339087213,
+      0.6011297593376128
+    ],
+    "dice_seeds": [
+      0.7268623914793524,
+      0.7158535119640104,
+      0.6757614238600594
+    ]
+  },
+  "busi_N100_pixeldit": {
+    "iou_mean": 0.6540538953088814,
+    "dice_mean": 0.7316955478068173,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.6616519138889801,
+      0.6470095283658505,
+      0.6535002436718134
+    ],
+    "dice_seeds": [
+      0.7383933408398845,
+      0.7258913137588962,
+      0.730801988821671
+    ]
+  }
+}

code/scripts/p1/p1_full_metrics.json

@@ -0,0 +1,182 @@
+{
+  "isic_N50_real": {
+    "dice": 85.16,
+    "iou": 77.49,
+    "sensitivity": 85.34,
+    "precision": 90.16
+  },
+  "isic_N50_jit": {
+    "dice": 87.74,
+    "iou": 80.52,
+    "sensitivity": 88.36,
+    "precision": 90.69
+  },
+  "isic_N50_pixelgen": {
+    "dice": 87.9,
+    "iou": 80.71,
+    "sensitivity": 88.68,
+    "precision": 90.82
+  },
+  "isic_N50_deco": {
+    "dice": 87.27,
+    "iou": 79.96,
+    "sensitivity": 87.87,
+    "precision": 90.63
+  },
+  "isic_N50_pixeldit": {
+    "dice": 87.68,
+    "iou": 80.52,
+    "sensitivity": 88.33,
+    "precision": 90.58
+  },
+  "isic_N100_real": {
+    "dice": 87.31,
+    "iou": 80.18,
+    "sensitivity": 87.78,
+    "precision": 90.47
+  },
+  "isic_N100_jit": {
+    "dice": 88.52,
+    "iou": 81.66,
+    "sensitivity": 89.46,
+    "precision": 90.72
+  },
+  "isic_N100_pixelgen": {
+    "dice": 88.95,
+    "iou": 82.13,
+    "sensitivity": 90.15,
+    "precision": 90.83
+  },
+  "isic_N100_deco": {
+    "dice": 88.15,
+    "iou": 81.19,
+    "sensitivity": 89.85,
+    "precision": 89.97
+  },
+  "isic_N100_pixeldit": {
+    "dice": 88.6,
+    "iou": 81.7,
+    "sensitivity": 89.26,
+    "precision": 91.13
+  },
+  "kvasir_N50_real": {
+    "dice": 83.75,
+    "iou": 75.35,
+    "sensitivity": 83.39,
+    "precision": 88.81
+  },
+  "kvasir_N50_jit": {
+    "dice": 86.75,
+    "iou": 79.2,
+    "sensitivity": 86.93,
+    "precision": 89.85
+  },
+  "kvasir_N50_pixelgen": {
+    "dice": 86.16,
+    "iou": 78.42,
+    "sensitivity": 86.95,
+    "precision": 89.06
+  },
+  "kvasir_N50_deco": {
+    "dice": 88.03,
+    "iou": 80.84,
+    "sensitivity": 87.64,
+    "precision": 90.75
+  },
+  "kvasir_N50_pixeldit": {
+    "dice": 85.87,
+    "iou": 78.97,
+    "sensitivity": 85.23,
+    "precision": 89.44
+  },
+  "kvasir_N100_real": {
+    "dice": 86.95,
+    "iou": 79.5,
+    "sensitivity": 87.53,
+    "precision": 90.66
+  },
+  "kvasir_N100_jit": {
+    "dice": 88.5,
+    "iou": 81.62,
+    "sensitivity": 87.71,
+    "precision": 92.71
+  },
+  "kvasir_N100_pixelgen": {
+    "dice": 86.94,
+    "iou": 80.32,
+    "sensitivity": 86.37,
+    "precision": 91.14
+  },
+  "kvasir_N100_deco": {
+    "dice": 88.14,
+    "iou": 81.22,
+    "sensitivity": 88.95,
+    "precision": 90.6
+  },
+  "kvasir_N100_pixeldit": {
+    "dice": 89.52,
+    "iou": 82.83,
+    "sensitivity": 89.42,
+    "precision": 91.78
+  },
+  "busi_N50_real": {
+    "dice": 61.5,
+    "iou": 52.64,
+    "sensitivity": 59.37,
+    "precision": 72.68
+  },
+  "busi_N50_jit": {
+    "dice": 68.2,
+    "iou": 60.34,
+    "sensitivity": 68.58,
+    "precision": 73.26
+  },
+  "busi_N50_pixelgen": {
+    "dice": 66.89,
+    "iou": 59.31,
+    "sensitivity": 65.33,
+    "precision": 77.14
+  },
+  "busi_N50_deco": {
+    "dice": 67.6,
+    "iou": 59.8,
+    "sensitivity": 66.72,
+    "precision": 75.06
+  },
+  "busi_N50_pixeldit": {
+    "dice": 68.74,
+    "iou": 60.72,
+    "sensitivity": 67.82,
+    "precision": 75.91
+  },
+  "busi_N100_real": {
+    "dice": 67.26,
+    "iou": 59.02,
+    "sensitivity": 66.01,
+    "precision": 73.61
+  },
+  "busi_N100_jit": {
+    "dice": 71.52,
+    "iou": 63.87,
+    "sensitivity": 70.99,
+    "precision": 76.41
+  },
+  "busi_N100_pixelgen": {
+    "dice": 72.29,
+    "iou": 64.66,
+    "sensitivity": 72.49,
+    "precision": 77.16
+  },
+  "busi_N100_deco": {
+    "dice": 70.62,
+    "iou": 62.86,
+    "sensitivity": 71.52,
+    "precision": 74.61
+  },
+  "busi_N100_pixeldit": {
+    "dice": 73.17,
+    "iou": 65.41,
+    "sensitivity": 72.65,
+    "precision": 79.23
+  }
+}

code/scripts/p1/p1_gen_queue.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# P1 generator-training queue: backbones x datasets, amortized (full-data), ~50k steps.
+# Args: GPU [datasets] [backbones]   e.g.  p1_gen_queue.sh 5 "isic kvasir" "jit pixelgen deco pixeldit"
+set -u
+DR=/home/wzhang/LSC/Dataset/Segmentation/processed_unified
+ROOT=/home/wzhang/LSC/Code/NPJ
+PY=/opt/anaconda3/envs/seggen/bin/python
+GPU=${1:-5}
+ORDER=${2:-"isic kvasir"}
+BKS=${3:-"jit pixelgen deco pixeldit"}
+E="CUDA_DEVICE_ORDER=PCI_BUS_ID CUDA_VISIBLE_DEVICES=$GPU TORCHDYNAMO_DISABLE=1 PYTHONPATH=. OMP_NUM_THREADS=4"
+cd "$ROOT" || exit 1
+ST=logs/p1gen_status.md
+log(){ echo "[$(date '+%F %T')] $*" >> "$ST"; }
+declare -A DSMAP=( [isic]="medsegdb_isic2018 holdout" [kvasir]="kvasir_seg official" )
+log "=== P1 GEN QUEUE START GPU-$GPU order=[$ORDER] bks=[$BKS] pid=$$ ==="
+for dk in $ORDER; do
+  set -- ${DSMAP[$dk]}; ds=$1; proto=$2
+  for bk in $BKS; do
+    out=pretrained/pixdiff/p1_${bk}_${dk}.pt
+    if [ -f "$out" ]; then log "SKIP $bk/$dk (ckpt exists)"; continue; fi
+    log "TRAIN $bk/$dk -> $out"
+    env $E $PY -m framework.synth.pixdiff.train --data_root "$DR" --dataset "$ds" --protocol "$proto" \
+        --backbone "$bk" --img_size 256 --batch_size 16 --epochs 100000 --max_steps 50000 \
+        --lr 1e-4 --amp bf16 --train_fraction 1.0 --fraction_seed 0 \
+        --out_ckpt "$out" --log_interval 200 > logs/p1gen_${bk}_${dk}.log 2>&1
+    rc=$?
+    log "  done $bk/$dk rc=$rc ckpt=$([ -f "$out" ] && echo ok || echo MISSING)"
+  done
+done
+log "=== P1 GEN QUEUE DONE GPU-$GPU ==="
+echo "P1GEN_DONE_$GPU" >> "$ST"

code/scripts/p1/p1_master.py

@@ -0,0 +1,167 @@
+"""P1 master orchestrator: DAG scheduler over GPU0-5 for the backbone bake-off.
+Phases: A) 8 generators (4 backbones x 2 datasets, amortized, 50k steps)
+        B) 16 sampling jobs (per gen x N in {50,100}, mask_aug n_per_mask=4)
+        C) 60 downstream seg runs (real + 4 backbones) x 2 ds x 2 N x 3 seeds
+Single GPU per job (no DDP needed: 84 independent jobs). Retry-once on failure.
+Resumable (skips done outputs). Rolling aggregate -> logs/p1master/p1_results.json."""
+import os, sys, time, json, subprocess, statistics as st
+
+ROOT = "/home/wzhang/LSC/Code/NPJ"
+DR = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+PY = "/opt/anaconda3/envs/seggen/bin/python"
+GPUS = [0, 1, 2, 3, 4, 5]
+os.chdir(ROOT)
+LOGD = os.path.join(ROOT, "logs", "p1master")
+os.makedirs(LOGD, exist_ok=True)
+
+def log(m):
+    line = f"[{time.strftime('%F %T')}] {m}"
+    with open(os.path.join(LOGD, "status.md"), "a") as f:
+        f.write(line + "\n")
+    print(line, flush=True)
+
+DSETS = {"isic": ("medsegdb_isic2018", "holdout", 2582),
+         "kvasir": ("kvasir_seg", "official", 800)}
+BKS = ["jit", "pixelgen", "deco", "pixeldit"]
+NS = [50, 100]
+SEEDS = [0, 1, 2]
+
+jobs = {}
+def add(jid, cmd, deps=(), done_path=None, done_min=1):
+    jobs[jid] = {"cmd": cmd, "deps": list(deps), "done_path": done_path,
+                 "done_min": done_min, "state": "pending", "tries": 0, "gpu": None}
+
+# Phase A: generators
+for bk in BKS:
+    for dk, (ds, proto, tot) in DSETS.items():
+        out = f"pretrained/pixdiff/p1_{bk}_{dk}.pt"
+        cmd = (f"{PY} -m framework.synth.pixdiff.train --data_root {DR} --dataset {ds} "
+               f"--protocol {proto} --backbone {bk} --img_size 256 --batch_size 16 "
+               f"--epochs 100000 --max_steps 50000 --lr 1e-4 --amp bf16 "
+               f"--train_fraction 1.0 --fraction_seed 0 --out_ckpt {out} --log_interval 500")
+        add(f"gen_{bk}_{dk}", cmd, done_path=os.path.join(ROOT, out))
+
+# Phase B: sampling
+for bk in BKS:
+    for dk, (ds, proto, tot) in DSETS.items():
+        ck = f"pretrained/pixdiff/p1_{bk}_{dk}.pt"
+        for N in NS:
+            f = N / tot
+            sd = f"{DR}/{ds}/{proto}/synth_p1_{bk}_{dk}_f{N}"
+            cmd = (f"{PY} -m framework.synth.pixdiff.sample --ckpt {ck} --data_root {DR} "
+                   f"--dataset {ds} --protocol {proto} --train_fraction {f} --fraction_seed 0 "
+                   f"--n_per_mask 4 --mask_aug --num_steps 50 --out_dir {sd}")
+            add(f"samp_{bk}_{dk}_N{N}", cmd, deps=[f"gen_{bk}_{dk}"],
+                done_path=os.path.join(sd, "images"), done_min=N * 4)
+
+# Phase C: downstream
+def mpath(exp, ds, proto, S):
+    return os.path.join(ROOT, f"results/{exp}/{ds}_{proto}/unet/seed{S}/metrics.json")
+
+def seg_cmd(ds, proto, f, exp, S, synth=None):
+    base = (f"{PY} framework/train.py --data_root {DR} --dataset {ds} --protocol {proto} "
+            f"--arch unet --encoder resnet50 --aug standard --epochs 400 "
+            f"--train_fraction {f} --fraction_seed 0 --exp_name {exp} --amp bf16 --seed {S}")
+    if synth:
+        base += f" --synth_train_dir {synth}"
+    test = (f"{PY} framework/test.py --data_root {DR} --dataset {ds} --protocol {proto} "
+            f"--arch unet --encoder resnet50 --aug standard --exp_name {exp} --seed {S}")
+    return base + " && " + test
+
+for dk, (ds, proto, tot) in DSETS.items():
+    for N in NS:
+        f = N / tot
+        for S in SEEDS:
+            exp = f"p1_real_{dk}_N{N}"
+            add(f"seg_real_{dk}_N{N}_s{S}", seg_cmd(ds, proto, f, exp, S),
+                done_path=mpath(exp, ds, proto, S))
+        for bk in BKS:
+            sd = f"{DR}/{ds}/{proto}/synth_p1_{bk}_{dk}_f{N}"
+            for S in SEEDS:
+                exp = f"p1_{bk}_{dk}_N{N}"
+                add(f"seg_{bk}_{dk}_N{N}_s{S}", seg_cmd(ds, proto, f, exp, S, synth=sd),
+                    deps=[f"samp_{bk}_{dk}_N{N}"], done_path=mpath(exp, ds, proto, S))
+
+def is_done(j):
+    p = j["done_path"]
+    if not p or not os.path.exists(p):
+        return False
+    if os.path.isdir(p):
+        try:
+            return len(os.listdir(p)) >= j["done_min"]
+        except OSError:
+            return False
+    return True
+
+def aggregate():
+    res = {}
+    for dk, (ds, proto, tot) in DSETS.items():
+        for N in NS:
+            for arm in ["real"] + BKS:
+                exp = f"p1_{arm}_{dk}_N{N}"
+                ious, dices = [], []
+                for S in SEEDS:
+                    mp = mpath(exp, ds, proto, S)
+                    if os.path.exists(mp):
+                        try:
+                            m = json.load(open(mp))["metrics"]
+                            ious.append(m["iou_mean"]); dices.append(m["dice_mean"])
+                        except Exception:
+                            pass
+                if ious:
+                    res[f"{dk}_N{N}_{arm}"] = {
+                        "iou_mean": sum(ious) / len(ious), "dice_mean": sum(dices) / len(dices),
+                        "n_seeds": len(ious), "iou_seeds": ious, "dice_seeds": dices}
+    json.dump(res, open(os.path.join(LOGD, "p1_results.json"), "w"), indent=2)
+
+for jid, j in jobs.items():
+    if is_done(j):
+        j["state"] = "done"
+def deps_done(j):
+    return all(jobs[d]["state"] == "done" for d in j["deps"])
+
+running = {}
+free = set(GPUS)
+MAXTRIES = 2
+log(f"START {len(jobs)} jobs on GPUs {GPUS} ({sum(1 for j in jobs.values() if j['state']=='done')} pre-done)")
+last_summary = 0
+while True:
+    if all(j["state"] in ("done", "failed") for j in jobs.values()):
+        break
+    for jid, j in jobs.items():
+        if not free:
+            break
+        if j["state"] == "pending" and deps_done(j):
+            if is_done(j):
+                j["state"] = "done"; continue
+            g = free.pop()
+            env = dict(os.environ, CUDA_DEVICE_ORDER="PCI_BUS_ID",
+                       CUDA_VISIBLE_DEVICES=str(g), TORCHDYNAMO_DISABLE="1",
+                       PYTHONPATH=".", OMP_NUM_THREADS="4")
+            lf = open(os.path.join(LOGD, jid + ".log"), "a")
+            p = subprocess.Popen(j["cmd"], shell=True, env=env, stdout=lf,
+                                 stderr=subprocess.STDOUT, cwd=ROOT)
+            running[g] = (jid, p, lf); j["state"] = "running"; j["gpu"] = g; j["tries"] += 1
+            log(f"LAUNCH {jid} GPU{g} try{j['tries']}")
+    for g, (jid, p, lf) in list(running.items()):
+        rc = p.poll()
+        if rc is None:
+            continue
+        lf.close(); del running[g]; free.add(g)
+        j = jobs[jid]
+        if is_done(j):
+            j["state"] = "done"; log(f"DONE {jid} rc={rc}")
+        elif j["tries"] < MAXTRIES:
+            j["state"] = "pending"; log(f"RETRY {jid} rc={rc}")
+        else:
+            j["state"] = "failed"; log(f"FAILED {jid} rc={rc}")
+    if time.time() - last_summary > 300:
+        cnt = {s: sum(1 for j in jobs.values() if j["state"] == s)
+               for s in ("done", "running", "pending", "failed")}
+        log(f"SUMMARY {cnt} | running={sorted(j['gpu'] for j in jobs.values() if j['state']=='running')}")
+        aggregate(); last_summary = time.time()
+    time.sleep(10)
+
+aggregate()
+log("ALL DONE")
+print("P1_MASTER_DONE", flush=True)

code/scripts/p1/p1_results.json

@@ -0,0 +1,302 @@
+{
+  "isic_N50_real": {
+    "iou_mean": 0.7749372989665764,
+    "dice_mean": 0.8516048790137111,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.7697344814191404,
+      0.772716144763783,
+      0.7823612707168056
+    ],
+    "dice_seeds": [
+      0.8480779677176173,
+      0.8503343900165478,
+      0.8564022793069677
+    ]
+  },
+  "isic_N50_jit": {
+    "iou_mean": 0.8052051026596693,
+    "dice_mean": 0.8773763597758372,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8038682316971465,
+      0.8033287650931844,
+      0.8084183111886771
+    ],
+    "dice_seeds": [
+      0.8763739862706927,
+      0.8755590664589031,
+      0.8801960265979158
+    ]
+  },
+  "isic_N50_pixelgen": {
+    "iou_mean": 0.8071234360528656,
+    "dice_mean": 0.8790093239202045,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8046804415401705,
+      0.8071308385675863,
+      0.80955902805084
+    ],
+    "dice_seeds": [
+      0.8771928252649649,
+      0.8781965482168992,
+      0.8816385982787495
+    ]
+  },
+  "isic_N50_deco": {
+    "iou_mean": 0.7996043535547449,
+    "dice_mean": 0.8726861944567714,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.7988670932254234,
+      0.8020180961470182,
+      0.797927871291793
+    ],
+    "dice_seeds": [
+      0.8724373392477919,
+      0.875006987196055,
+      0.8706142569264674
+    ]
+  },
+  "isic_N50_pixeldit": {
+    "iou_mean": 0.8051857432382189,
+    "dice_mean": 0.8768404429697196,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8076451208631789,
+      0.80299518868651,
+      0.8049169201649677
+    ],
+    "dice_seeds": [
+      0.8789570258524421,
+      0.8757209786234587,
+      0.8758433244332583
+    ]
+  },
+  "isic_N100_real": {
+    "iou_mean": 0.8018487332144956,
+    "dice_mean": 0.8731124297212501,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8018532993620694,
+      0.8030565805042732,
+      0.8006363197771443
+    ],
+    "dice_seeds": [
+      0.8742042532437185,
+      0.8744577536420762,
+      0.8706752822779558
+    ]
+  },
+  "isic_N100_jit": {
+    "iou_mean": 0.8166433570078636,
+    "dice_mean": 0.8852389536608354,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8184147975006147,
+      0.8152316048562138,
+      0.8162836686667623
+    ],
+    "dice_seeds": [
+      0.8866063380696619,
+      0.8842458679141912,
+      0.8848646549986529
+    ]
+  },
+  "isic_N100_pixelgen": {
+    "iou_mean": 0.8212823342374741,
+    "dice_mean": 0.8895036576449075,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8203996376523117,
+      0.8215948549134239,
+      0.8218525101466865
+    ],
+    "dice_seeds": [
+      0.8893210218244473,
+      0.8894422995119722,
+      0.8897476515983034
+    ]
+  },
+  "isic_N100_deco": {
+    "iou_mean": 0.8119110110180822,
+    "dice_mean": 0.8814897596168004,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.812474683629472,
+      0.81098766389058,
+      0.8122706855341947
+    ],
+    "dice_seeds": [
+      0.8834257222572127,
+      0.8798560384070768,
+      0.881187518186112
+    ]
+  },
+  "isic_N100_pixeldit": {
+    "iou_mean": 0.8169910899725092,
+    "dice_mean": 0.8860181724664612,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8168252607788484,
+      0.8168659787418466,
+      0.8172820303968324
+    ],
+    "dice_seeds": [
+      0.8855665474000166,
+      0.8859441754776876,
+      0.8865437945216795
+    ]
+  },
+  "kvasir_N50_real": {
+    "iou_mean": 0.7534671958018979,
+    "dice_mean": 0.8375380075145843,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.7422559578123432,
+      0.7636408134000111,
+      0.7545048161933395
+    ],
+    "dice_seeds": [
+      0.8244831334988446,
+      0.849740539589318,
+      0.8383903494555902
+    ]
+  },
+  "kvasir_N50_jit": {
+    "iou_mean": 0.7919630629212887,
+    "dice_mean": 0.867516315771424,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.7929972233023507,
+      0.7943493738869872,
+      0.7885425915745284
+    ],
+    "dice_seeds": [
+      0.8676604855354516,
+      0.8655865846767077,
+      0.8693018771021128
+    ]
+  },
+  "kvasir_N50_pixelgen": {
+    "iou_mean": 0.7842259846149005,
+    "dice_mean": 0.8615631488519918,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.7841354377174787,
+      0.7782063762034493,
+      0.7903361399237733
+    ],
+    "dice_seeds": [
+      0.858331250209457,
+      0.8583852594567815,
+      0.8679729368897371
+    ]
+  },
+  "kvasir_N50_deco": {
+    "iou_mean": 0.8083752894364583,
+    "dice_mean": 0.8803164279623182,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8104997825534876,
+      0.8045413607010391,
+      0.8100847250548484
+    ],
+    "dice_seeds": [
+      0.8826893496480072,
+      0.8778416284559154,
+      0.8804183057830322
+    ]
+  },
+  "kvasir_N50_pixeldit": {
+    "iou_mean": 0.7897261916004509,
+    "dice_mean": 0.8586578118039875,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.7733292625059578,
+      0.800297355081953,
+      0.7955519572134416
+    ],
+    "dice_seeds": [
+      0.8420264952707284,
+      0.8674089356274011,
+      0.8665380045138328
+    ]
+  },
+  "kvasir_N100_real": {
+    "iou_mean": 0.7949746887997339,
+    "dice_mean": 0.8695256998060409,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.7966858390583808,
+      0.7968998232753884,
+      0.7913384040654323
+    ],
+    "dice_seeds": [
+      0.87442242361909,
+      0.8720526205988572,
+      0.8621020552001751
+    ]
+  },
+  "kvasir_N100_jit": {
+    "iou_mean": 0.8162035292350188,
+    "dice_mean": 0.884963785884059,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8044583298250717,
+      0.8215198247885459,
+      0.8226324330914389
+    ],
+    "dice_seeds": [
+      0.8733364457591601,
+      0.8902805661843565,
+      0.8912743457086602
+    ]
+  },
+  "kvasir_N100_pixelgen": {
+    "iou_mean": 0.8032477917950044,
+    "dice_mean": 0.8693680901970753,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.7990136881750312,
+      0.8131907666134136,
+      0.7975389205965682
+    ],
+    "dice_seeds": [
+      0.8653448154322447,
+      0.8767741359428084,
+      0.8659853192161729
+    ]
+  },
+  "kvasir_N100_deco": {
+    "iou_mean": 0.8121859633543412,
+    "dice_mean": 0.8814163311797074,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8014463766191772,
+      0.8207859220669117,
+      0.8143255913769347
+    ],
+    "dice_seeds": [
+      0.8737877150565937,
+      0.885505301166682,
+      0.8849559773158466
+    ]
+  },
+  "kvasir_N100_pixeldit": {
+    "iou_mean": 0.8282534914466804,
+    "dice_mean": 0.8952017769853248,
+    "n_seeds": 3,
+    "iou_seeds": [
+      0.8263046284256831,
+      0.8313217018221024,
+      0.8271341440922554
+    ],
+    "dice_seeds": [
+      0.8969579025391758,
+      0.8957992526085868,
+      0.8928481758082121
+    ]
+  }
+}

code/scripts/p1/smoke_backbone.py

@@ -0,0 +1,76 @@
+"""Smoke: port DeCo (PixNerDiT) or PixelDiT (PixDiT) into the PixDiff mask-concat scaffolding.
+Backbone-agnostic decouple: build with in=img+cond, out defaults to in, take x_pred[:, :img_ch].
+Usage: python smoke_backbone.py {deco|pixeldit}"""
+import os, sys
+sys.path.insert(0, "/home/wzhang/LSC/Code/NPJ")
+import torch, torch.nn as nn
+from torch.utils.data import DataLoader
+from framework.synth.pixdiff.conditioning import build_conditioner
+from framework.synth.pixdiff.data import MaskCondGenDataset
+
+BK = sys.argv[1]
+DECO = "/home/wzhang/LSC/Code/NPJ/sota/DeCo"
+PIXELDIT = "/home/wzhang/LSC/Code/NPJ/sota/PixelDiT"
+dev = "cuda"
+DR = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+
+ds = MaskCondGenDataset(DR, "medsegdb_isic2018", "holdout", img_size=256,
+                        train_fraction=0.02, fraction_seed=0)
+cond = build_conditioner("onehot", ds.num_classes).to(dev)
+img_ch, K = ds.in_channels, cond.cond_channels
+in_tot = img_ch + K
+print(f"[{BK}] ds n={len(ds)} img_ch={img_ch} K={K} in_tot={in_tot}", flush=True)
+
+if BK == "deco":
+    sys.path.insert(0, os.path.join(DECO, "src", "models", "transformer"))
+    from dit_c2i_DeCo import PixNerDiT
+    net = PixNerDiT(in_channels=in_tot, patch_size=16, num_groups=12, hidden_size=768,
+                    hidden_size_x=32, num_blocks=13, num_cond_blocks=12, num_classes=1).to(dev)
+elif BK == "pixeldit":
+    sys.path.insert(0, PIXELDIT)
+    from pixdit_core.pixeldit_c2i import PixDiT
+    net = PixDiT(in_channels=in_tot, num_groups=12, hidden_size=768, pixel_hidden_size=16,
+                 patch_depth=12, pixel_depth=4, patch_size=16, num_classes=1).to(dev)
+else:
+    raise SystemExit("backbone must be deco|pixeldit")
+
+print(f"[{BK}] params={sum(p.numel() for p in net.parameters())/1e6:.1f}M", flush=True)
+opt = torch.optim.AdamW(net.parameters(), lr=1e-4)
+dl = DataLoader(ds, batch_size=4, shuffle=True, drop_last=True, num_workers=2)
+it = iter(dl)
+def get_batch():
+    global it
+    try: b = next(it)
+    except StopIteration: it = iter(dl); b = next(it)
+    return (b["image"], b["mask"]) if isinstance(b, dict) else (b[0], b[1])
+
+net.train()
+for step in range(20):
+    img, msk = get_batch(); img, msk = img.to(dev), msk.to(dev)
+    t = torch.sigmoid(torch.randn(img.size(0), device=dev) * 0.8 - 0.8).view(-1, 1, 1, 1)
+    e = torch.randn_like(img)
+    z = t * img + (1 - t) * e
+    v = (img - z) / (1 - t).clamp_min(5e-2)
+    c = cond(msk)
+    y = torch.zeros(img.size(0), dtype=torch.long, device=dev)
+    out = net(torch.cat([z, c], dim=1), t.flatten(), y)
+    assert out.dim() == 4 and out.shape[1] >= img_ch, f"bad out shape {tuple(out.shape)}"
+    x_pred = out[:, :img_ch]
+    v_pred = (x_pred - z) / (1 - t).clamp_min(5e-2)
+    loss = ((v - v_pred) ** 2).mean()
+    loss.backward(); opt.step(); opt.zero_grad()
+    if step % 5 == 0 or step == 19:
+        print(f"[{BK}] step {step:2d} loss {loss.item():.4f}", flush=True)
+
+net.eval()
+with torch.no_grad():
+    msk0 = msk[:2]; c0 = cond(msk0)
+    z = torch.randn(2, img_ch, 256, 256, device=dev)
+    ts = torch.linspace(0, 1, 11).tolist()
+    for i in range(10):
+        tc, dt = ts[i], ts[i + 1] - ts[i]
+        out = net(torch.cat([z, c0], dim=1), torch.full((2,), tc, device=dev),
+                  torch.zeros(2, dtype=torch.long, device=dev))[:, :img_ch]
+        z = z + (out - z) / max(1 - tc, 5e-2) * dt
+    print(f"[{BK}] sample ok shape={tuple(z.shape)} range=({z.min():.2f},{z.max():.2f})", flush=True)
+print(f"SMOKE_{BK.upper()}_PASS", flush=True)

code/scripts/p1/smoke_pixelgen.py

@@ -0,0 +1,70 @@
+"""Smoke test: port PixelGen's JiT denoiser into the PixDiff mask-concat scaffolding.
+Validates: import on server, in=img+cond/out=img build, flow-matching train step, sampling.
+Tiny: 20 train steps + 10-step sample on ~50 ISIC images. No checkpoint written."""
+import os, sys
+sys.path.insert(0, "/home/wzhang/LSC/Code/NPJ")                                   # framework.*
+PG = "/home/wzhang/LSC/Code/NPJ/sota/PixelGen"
+sys.path.insert(0, os.path.join(PG, "src", "models", "transformer"))               # JiT.py
+import torch, torch.nn as nn
+from torch.utils.data import DataLoader
+from JiT import JiT_models, FinalLayer                                             # PixelGen denoiser
+from framework.synth.pixdiff.conditioning import build_conditioner
+from framework.synth.pixdiff.data import MaskCondGenDataset
+
+dev = "cuda"
+DR = "/home/wzhang/LSC/Dataset/Segmentation/processed_unified"
+ds = MaskCondGenDataset(DR, "medsegdb_isic2018", "holdout", img_size=256,
+                        train_fraction=0.02, fraction_seed=0)
+print(f"[ds] n={len(ds)} in_ch={ds.in_channels} num_classes={ds.num_classes}", flush=True)
+cond = build_conditioner("onehot", ds.num_classes).to(dev)
+img_ch, K = ds.in_channels, cond.cond_channels
+
+net = JiT_models["JiT-B/16"](input_size=256, in_channels=img_ch + K, num_classes=1).to(dev)
+if net.out_channels != img_ch:
+    net.out_channels = img_ch
+    net.final_layer = FinalLayer(net.hidden_size, net.patch_size, img_ch).to(dev)
+    nn.init.constant_(net.final_layer.linear.weight, 0); nn.init.constant_(net.final_layer.linear.bias, 0)
+    nn.init.constant_(net.final_layer.adaLN_modulation[-1].weight, 0)
+    nn.init.constant_(net.final_layer.adaLN_modulation[-1].bias, 0)
+print(f"[net] PixelGen JiT-B/16 in={img_ch+K} out={net.out_channels} params={sum(p.numel() for p in net.parameters())/1e6:.1f}M", flush=True)
+
+opt = torch.optim.AdamW(net.parameters(), lr=1e-4)
+dl = DataLoader(ds, batch_size=4, shuffle=True, drop_last=True, num_workers=2)
+it = iter(dl)
+
+def get_batch():
+    global it
+    try: b = next(it)
+    except StopIteration:
+        it = iter(dl); b = next(it)
+    if isinstance(b, dict): return b["image"], b["mask"]
+    return b[0], b[1]
+
+net.train()
+for step in range(20):
+    img, msk = get_batch(); img, msk = img.to(dev), msk.to(dev)
+    t = torch.sigmoid(torch.randn(img.size(0), device=dev) * 0.8 - 0.8).view(-1, 1, 1, 1)
+    e = torch.randn_like(img)
+    z = t * img + (1 - t) * e
+    v = (img - z) / (1 - t).clamp_min(5e-2)
+    c = cond(msk)
+    y = torch.zeros(img.size(0), dtype=torch.long, device=dev)
+    x_pred = net(torch.cat([z, c], dim=1), t.flatten(), y)
+    v_pred = (x_pred - z) / (1 - t).clamp_min(5e-2)
+    loss = ((v - v_pred) ** 2).mean()
+    loss.backward(); opt.step(); opt.zero_grad()
+    if step % 5 == 0 or step == 19:
+        print(f"[train] step {step:2d} loss {loss.item():.4f}", flush=True)
+
+net.eval()
+with torch.no_grad():
+    msk0 = msk[:2]; c0 = cond(msk0)
+    z = torch.randn(2, img_ch, 256, 256, device=dev)
+    ts = torch.linspace(0, 1, 11).tolist()
+    for i in range(10):
+        tc, dt = ts[i], ts[i + 1] - ts[i]
+        tt = torch.full((2,), tc, device=dev)
+        xp = net(torch.cat([z, c0], dim=1), tt, torch.zeros(2, dtype=torch.long, device=dev))
+        z = z + (xp - z) / max(1 - tc, 5e-2) * dt
+    print(f"[sample] ok shape={tuple(z.shape)} range=({z.min():.2f},{z.max():.2f})", flush=True)
+print("SMOKE_PIXELGEN_PASS", flush=True)

code/scripts/p1/train_fd_patched.py

@@ -0,0 +1,179 @@
+"""FD-Loss post-training for a PixDiff generator (env: seggen, GPU 5).
+
+Starts from a base PixDiff checkpoint and continues training with:
+    total = flow_matching_loss  +  fd_weight * normalized_FD_loss
+where the FD term matches the generated x0 feature distribution to the real-image
+reference distribution (Inception space), gated to low-noise timesteps (where x0 is
+a meaningful image). This targets the blur/distribution gap the MSE objective leaves.
+
+Run from project root (…/NPJ):
+  CUDA_VISIBLE_DEVICES=5 python -m framework.synth.pixdiff.train_fd \
+    --base_ckpt pretrained/pixdiff/kvasir_seg_official_f1.0.pt \
+    --data_root /home/wzhang/LSC/Dataset/Segmentation/processed_unified \
+    --dataset kvasir_seg --protocol official \
+    --epochs 200 --lr 2e-5 --fd_weight 0.5 \
+    --out_ckpt pretrained/pixdiff/kvasir_seg_official_f1.0_fd.pt
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import time
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")))
+
+import numpy as np
+import torch
+from torch.utils.data import DataLoader
+
+from framework.synth.pixdiff.data import MaskCondGenDataset
+from framework.synth.pixdiff.conditioning import build_conditioner
+from framework.synth.pixdiff.mask_jit import MaskDenoiser
+from framework.synth.pixdiff.fd_loss import (
+    InceptionFeatures, FeatureQueue, compute_frechet_distance_loss,
+    precompute_sigma_ref_sqrt, compute_ref_stats,
+)
+
+
+def get_args():
+    p = argparse.ArgumentParser("PixDiff FD-Loss post-training")
+    p.add_argument("--base_ckpt", required=True)
+    p.add_argument("--data_root", required=True)
+    p.add_argument("--dataset", required=True)
+    p.add_argument("--protocol", required=True)
+    p.add_argument("--train_fraction", type=float, default=1.0)
+    p.add_argument("--fraction_seed", type=int, default=0)
+    p.add_argument("--epochs", type=int, default=200)
+    p.add_argument("--batch_size", type=int, default=32)
+    p.add_argument("--lr", type=float, default=2e-5)
+    p.add_argument("--num_workers", type=int, default=6)
+    p.add_argument("--amp", default="bf16", choices=["bf16", "fp16", "fp32"])
+    # FD-Loss knobs
+    p.add_argument("--fd_weight", type=float, default=0.5)
+    p.add_argument("--fd_gate_t", type=float, default=0.5, help="apply FD only when t>=this (low noise)")
+    p.add_argument("--queue_size", type=int, default=512)
+    p.add_argument("--fd_norm_eps", type=float, default=1e-2)
+    p.add_argument("--lpips_weight", type=float, default=0.0)
+    p.add_argument("--dino_weight", type=float, default=0.0)
+    p.add_argument("--percep_gate_t", type=float, default=0.5, help="apply perceptual only when t>=this")
+    p.add_argument("--ref_stats", default="", help="npz of (mu,sigma); auto path + compute if empty")
+    p.add_argument("--ema_decay", type=float, default=0.9999)
+    p.add_argument("--seed", type=int, default=0)
+    p.add_argument("--out_ckpt", required=True)
+    p.add_argument("--log_interval", type=int, default=20)
+    return p.parse_args()
+
+
+def main():
+    a = get_args()
+    torch.manual_seed(a.seed)
+    device = "cuda"
+    amp_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}.get(a.amp)
+
+    # ---- data ----
+    ds = MaskCondGenDataset(a.data_root, a.dataset, a.protocol, img_size=256,
+                            train_fraction=a.train_fraction, fraction_seed=a.fraction_seed)
+    img_ch, n_cls = ds.in_channels, ds.num_classes
+    loader = DataLoader(ds, batch_size=a.batch_size, shuffle=True, drop_last=True,
+                        num_workers=a.num_workers, pin_memory=True, persistent_workers=a.num_workers > 0)
+    print(f"[fd] {a.dataset}/{a.protocol} n={len(ds)} in_ch={img_ch} num_classes={n_cls}", flush=True)
+    if img_ch != 3:
+        print("[fd][warn] Inception expects 3ch; non-RGB dataset — FD features may be weak.", flush=True)
+
+    # ---- model from base ckpt ----
+    ckpt = torch.load(a.base_ckpt, map_location="cpu", weights_only=False)
+    cond = build_conditioner(ckpt.get("conditioner", "onehot"), n_cls)
+    model = MaskDenoiser(ckpt["model_name"], ckpt["img_size"], ckpt["img_channels"], cond,
+                         noise_scale=ckpt.get("noise_scale", 1.0), ema_decay=a.ema_decay, backbone=ckpt.get("backbone", "jit")).to(device)
+    model.load_state_dict(ckpt["state_dict"])
+    model._ema = [e.to(device) for e in ckpt["ema"]] if ckpt.get("ema") is not None else None
+    if model._ema is None:
+        model.ema_init()
+    print(f"[fd] loaded base {a.base_ckpt}", flush=True)
+
+    # ---- FD machinery ----
+    inception = InceptionFeatures().to(device).eval()
+    queue = FeatureQueue(size=a.queue_size, feat_dim=inception.feat_dim).to(device)
+    percep = None
+    if a.lpips_weight > 0 or a.dino_weight > 0:
+        from framework.synth.pixdiff.perceptual import PerceptualLoss
+        percep = PerceptualLoss(use_lpips=a.lpips_weight > 0, use_dino=a.dino_weight > 0, device=device)
+        print(f"[fd] perceptual ON lpips_w={a.lpips_weight} dino_w={a.dino_weight} gate_t={a.percep_gate_t}", flush=True)
+
+    ref_path = a.ref_stats or a.out_ckpt.replace(".pt", "_refstats.npz")
+    if os.path.isfile(ref_path):
+        rs = np.load(ref_path); mu_ref_np, sigma_ref_np = rs["mu"], rs["sigma"]
+        print(f"[fd] loaded ref stats {ref_path}", flush=True)
+    else:
+        print("[fd] computing reference stats from real train images...", flush=True)
+        ref_loader = DataLoader(MaskCondGenDataset(a.data_root, a.dataset, a.protocol, img_size=256,
+                                                   train_fraction=a.train_fraction, fraction_seed=a.fraction_seed,
+                                                   hflip=False, vflip=False),
+                                batch_size=a.batch_size, shuffle=False, num_workers=a.num_workers)
+        mu_ref_np, sigma_ref_np, nref = compute_ref_stats(ref_loader, inception, device)
+        os.makedirs(os.path.dirname(os.path.abspath(ref_path)) or ".", exist_ok=True)
+        np.savez(ref_path, mu=mu_ref_np, sigma=sigma_ref_np)
+        print(f"[fd] ref stats from {nref} imgs -> {ref_path}", flush=True)
+    mu_ref = torch.tensor(mu_ref_np, device=device, dtype=torch.float64)
+    sigma_ref = torch.tensor(sigma_ref_np, device=device, dtype=torch.float64)
+    sigma_ref_sqrt = precompute_sigma_ref_sqrt(sigma_ref)
+
+    opt = torch.optim.AdamW(model._trainable(), lr=a.lr, weight_decay=0.0)
+    os.makedirs(os.path.dirname(os.path.abspath(a.out_ckpt)) or ".", exist_ok=True)
+
+    def save():
+        torch.save({"model_name": ckpt["model_name"], "img_size": ckpt["img_size"],
+                    "img_channels": img_ch, "num_classes": n_cls,
+                    "conditioner": ckpt.get("conditioner", "onehot"),
+                    "noise_scale": ckpt.get("noise_scale", 1.0),
+                    "state_dict": model.state_dict(), "ema": model._ema, "args": vars(a)}, a.out_ckpt)
+        print(f"[fd] saved {a.out_ckpt}", flush=True)
+
+    step = 0
+    for epoch in range(a.epochs):
+        model.train(); t0 = time.time(); run_fm = run_fd = run_fdraw = 0.0
+        for batch in loader:
+            img = batch["image"].to(device, non_blocking=True)
+            mask = batch["mask"].to(device, non_blocking=True)
+            opt.zero_grad(set_to_none=True)
+            with torch.autocast("cuda", dtype=amp_dtype) if amp_dtype else _null():
+                fm_loss, x_pred, t = model(img, mask, return_pred=True)
+            # FD term on predicted clean image, gated to low noise
+            gate = t >= a.fd_gate_t
+            fd_loss = torch.zeros((), device=device); fd_raw = 0.0
+            if int(gate.sum()) >= 2:
+                xg = x_pred[gate].float()
+                feats = inception((xg.clamp(-1, 1) + 1) / 2)          # (Ng,2048), grad flows
+                if queue.is_ready():
+                    mu, sigma = queue.build_feats_stats(feats)
+                    fd = compute_frechet_distance_loss(mu_ref, sigma_ref, mu, sigma, sigma_ref_sqrt)
+                    fd_raw = float(fd); fd_loss = fd / (fd.detach() + a.fd_norm_eps)
+                queue.enqueue(feats)
+            total = fm_loss + a.fd_weight * fd_loss
+            pl_lpips = pl_dino = 0.0
+            if percep is not None:
+                pgate = t >= a.percep_gate_t
+                if int(pgate.sum()) >= 1:
+                    pld = percep(x_pred[pgate].float(), img[pgate].float())
+                    if "lpips" in pld:
+                        total = total + a.lpips_weight * pld["lpips"]; pl_lpips = float(pld["lpips"])
+                    if "dino" in pld:
+                        total = total + a.dino_weight * pld["dino"]; pl_dino = float(pld["dino"])
+            total.backward(); opt.step(); model.ema_update()
+            run_fm += float(fm_loss); run_fd += float(fd_loss); run_fdraw += fd_raw; step += 1
+            if step % a.log_interval == 0:
+                print(f"[fd] ep{epoch} step{step} fm={float(fm_loss):.4f} fd_raw={fd_raw:.1f} "
+                      f"lpips={pl_lpips:.3f} dino={pl_dino:.3f} qready={queue.is_ready()}", flush=True)
+        print(f"[fd] epoch {epoch} fm={run_fm/max(1,len(loader)):.4f} "
+              f"fd_raw={run_fdraw/max(1,len(loader)):.1f} ({time.time()-t0:.1f}s)", flush=True)
+    save()
+
+
+class _null:
+    def __enter__(self): return self
+    def __exit__(self, *a): return False
+
+
+if __name__ == "__main__":
+    main()

code/sota/Swin-Unet/README.md

@@ -0,0 +1,64 @@
+# Swin-Unet
+[ECCVW2022] The codes for the work "Swin-Unet: Unet-like Pure Transformer for Medical Image Segmentation"(https://arxiv.org/abs/2105.05537). Our paper has been accepted by ECCV 2022 MEDICAL COMPUTER VISION WORKSHOP (https://mcv-workshop.github.io/). We updated the Reproducibility. I hope this will help you to reproduce the results.
+
+## 1. Download pre-trained swin transformer model (Swin-T)
+* [Get pre-trained model in this link] (https://drive.google.com/drive/folders/1UC3XOoezeum0uck4KBVGa8osahs6rKUY?usp=sharing): Put pretrained Swin-T into folder "pretrained_ckpt/"
+
+## 2. Prepare data
+
+- The datasets we used are provided by TransUnet's authors. [Get processed data in this link] (Synapse/BTCV: https://drive.google.com/drive/folders/1ACJEoTp-uqfFJ73qS3eUObQh52nGuzCd and ACDC: https://drive.google.com/drive/folders/1KQcrci7aKsYZi1hQoZ3T3QUtcy7b--n4).
+
+## 3. Environment
+
+- Please prepare an environment with python=3.7, and then use the command "pip install -r requirements.txt" for the dependencies.
+
+## 4. Train/Test
+
+- Run the train script on synapse dataset. The batch size we used is 24. If you do not have enough GPU memory, the bacth size can be reduced to 12 or 6 to save memory.
+
+- Train
+
+```bash
+sh train.sh 
+# or 
+python train.py --dataset Synapse --cfg configs/swin_tiny_patch4_window7_224_lite.yaml --root_path your DATA_DIR --max_epochs 150 --output_dir your OUT_DIR  --img_size 224 --base_lr 0.05 --batch_size 24
+```
+
+- Test 
+
+```bash
+sh test.sh 
+# or 
+python test.py --dataset Synapse --cfg configs/swin_tiny_patch4_window7_224_lite.yaml --is_saveni --volume_path your DATA_DIR --output_dir your OUT_DIR --max_epoch 150 --base_lr 0.05 --img_size 224 --batch_size 24
+```
+
+## Reproducibility
+
+
+- Codes
+
+Our trained model is stored on the Huawei cloud. The interns do not have the right to send any files out from the internal system, so I can't share our trained model weights. Regarding how to reproduce the segmentation results presented in the paper, we discovered that different GPU types would generate different results. In our code, we carefully set the random seed, so the results should be consistent when trained multiple times on the same type of GPU. If the training does not give the same segmentation results as in the paper, it is recommended to adjust the learning rate. And, the type of GPU we used in this work is Tesla v100. Finaly, pre-training is very important for pure transformer models. In our experiments, both the encoder and decoder are initialized with pretrained weights rather than initializing the encoder with pretrained weights only.
+
+## References
+* [TransUnet](https://github.com/Beckschen/TransUNet)
+* [SwinTransformer](https://github.com/microsoft/Swin-Transformer)
+
+## Citation
+
+```bibtex
+@InProceedings{swinunet,
+author = {Hu Cao and Yueyue Wang and Joy Chen and Dongsheng Jiang and Xiaopeng Zhang and Qi Tian and Manning Wang},
+title = {Swin-Unet: Unet-like Pure Transformer for Medical Image Segmentation},
+booktitle = {Proceedings of the European Conference on Computer Vision Workshops(ECCVW)},
+year = {2022}
+}
+
+@misc{cao2021swinunet,
+      title={Swin-Unet: Unet-like Pure Transformer for Medical Image Segmentation}, 
+      author={Hu Cao and Yueyue Wang and Joy Chen and Dongsheng Jiang and Xiaopeng Zhang and Qi Tian and Manning Wang},
+      year={2021},
+      eprint={2105.05537},
+      archivePrefix={arXiv},
+      primaryClass={eess.IV}
+}
+```

code/sota/Swin-Unet/config.py

@@ -0,0 +1,229 @@
+# --------------------------------------------------------
+# Swin Transformer
+# Copyright (c) 2021 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Written by Ze Liu
+# --------------------------------------------------------'
+
+import os
+import yaml
+from yacs.config import CfgNode as CN
+
+_C = CN()
+
+# Base config files
+_C.BASE = ['']
+
+# -----------------------------------------------------------------------------
+# Data settings
+# -----------------------------------------------------------------------------
+_C.DATA = CN()
+# Batch size for a single GPU, could be overwritten by command line argument
+_C.DATA.BATCH_SIZE = 128
+# Path to dataset, could be overwritten by command line argument
+_C.DATA.DATA_PATH = ''
+# Dataset name
+_C.DATA.DATASET = 'imagenet'
+# Input image size
+_C.DATA.IMG_SIZE = 224
+# Interpolation to resize image (random, bilinear, bicubic)
+_C.DATA.INTERPOLATION = 'bicubic'
+# Use zipped dataset instead of folder dataset
+# could be overwritten by command line argument
+_C.DATA.ZIP_MODE = False
+# Cache Data in Memory, could be overwritten by command line argument
+_C.DATA.CACHE_MODE = 'part'
+# Pin CPU memory in DataLoader for more efficient (sometimes) transfer to GPU.
+_C.DATA.PIN_MEMORY = True
+# Number of data loading threads
+_C.DATA.NUM_WORKERS = 8
+
+# -----------------------------------------------------------------------------
+# Model settings
+# -----------------------------------------------------------------------------
+_C.MODEL = CN()
+# Model type
+_C.MODEL.TYPE = 'swin'
+# Model name
+_C.MODEL.NAME = 'swin_tiny_patch4_window7_224'
+# Checkpoint to resume, could be overwritten by command line argument
+_C.MODEL.PRETRAIN_CKPT = './pretrained_ckpt/swin_tiny_patch4_window7_224.pth'
+_C.MODEL.RESUME = ''
+# Number of classes, overwritten in data preparation
+_C.MODEL.NUM_CLASSES = 1000
+# Dropout rate
+_C.MODEL.DROP_RATE = 0.0
+# Drop path rate
+_C.MODEL.DROP_PATH_RATE = 0.1
+# Label Smoothing
+_C.MODEL.LABEL_SMOOTHING = 0.1
+
+# Swin Transformer parameters
+_C.MODEL.SWIN = CN()
+_C.MODEL.SWIN.PATCH_SIZE = 4
+_C.MODEL.SWIN.IN_CHANS = 3
+_C.MODEL.SWIN.EMBED_DIM = 96
+_C.MODEL.SWIN.DEPTHS = [2, 2, 6, 2]
+_C.MODEL.SWIN.DECODER_DEPTHS = [2, 2, 6, 2]
+_C.MODEL.SWIN.NUM_HEADS = [3, 6, 12, 24]
+_C.MODEL.SWIN.WINDOW_SIZE = 7
+_C.MODEL.SWIN.MLP_RATIO = 4.
+_C.MODEL.SWIN.QKV_BIAS = True
+_C.MODEL.SWIN.QK_SCALE = None
+_C.MODEL.SWIN.APE = False
+_C.MODEL.SWIN.PATCH_NORM = True
+_C.MODEL.SWIN.FINAL_UPSAMPLE= "expand_first"
+
+# -----------------------------------------------------------------------------
+# Training settings
+# -----------------------------------------------------------------------------
+_C.TRAIN = CN()
+_C.TRAIN.START_EPOCH = 0
+_C.TRAIN.EPOCHS = 300
+_C.TRAIN.WARMUP_EPOCHS = 20
+_C.TRAIN.WEIGHT_DECAY = 0.05
+_C.TRAIN.BASE_LR = 5e-4
+_C.TRAIN.WARMUP_LR = 5e-7
+_C.TRAIN.MIN_LR = 5e-6
+# Clip gradient norm
+_C.TRAIN.CLIP_GRAD = 5.0
+# Auto resume from latest checkpoint
+_C.TRAIN.AUTO_RESUME = True
+# Gradient accumulation steps
+# could be overwritten by command line argument
+_C.TRAIN.ACCUMULATION_STEPS = 0
+# Whether to use gradient checkpointing to save memory
+# could be overwritten by command line argument
+_C.TRAIN.USE_CHECKPOINT = False
+
+# LR scheduler
+_C.TRAIN.LR_SCHEDULER = CN()
+_C.TRAIN.LR_SCHEDULER.NAME = 'cosine'
+# Epoch interval to decay LR, used in StepLRScheduler
+_C.TRAIN.LR_SCHEDULER.DECAY_EPOCHS = 30
+# LR decay rate, used in StepLRScheduler
+_C.TRAIN.LR_SCHEDULER.DECAY_RATE = 0.1
+
+# Optimizer
+_C.TRAIN.OPTIMIZER = CN()
+_C.TRAIN.OPTIMIZER.NAME = 'adamw'
+# Optimizer Epsilon
+_C.TRAIN.OPTIMIZER.EPS = 1e-8
+# Optimizer Betas
+_C.TRAIN.OPTIMIZER.BETAS = (0.9, 0.999)
+# SGD momentum
+_C.TRAIN.OPTIMIZER.MOMENTUM = 0.9
+
+# -----------------------------------------------------------------------------
+# Augmentation settings
+# -----------------------------------------------------------------------------
+_C.AUG = CN()
+# Color jitter factor
+_C.AUG.COLOR_JITTER = 0.4
+# Use AutoAugment policy. "v0" or "original"
+_C.AUG.AUTO_AUGMENT = 'rand-m9-mstd0.5-inc1'
+# Random erase prob
+_C.AUG.REPROB = 0.25
+# Random erase mode
+_C.AUG.REMODE = 'pixel'
+# Random erase count
+_C.AUG.RECOUNT = 1
+# Mixup alpha, mixup enabled if > 0
+_C.AUG.MIXUP = 0.8
+# Cutmix alpha, cutmix enabled if > 0
+_C.AUG.CUTMIX = 1.0
+# Cutmix min/max ratio, overrides alpha and enables cutmix if set
+_C.AUG.CUTMIX_MINMAX = None
+# Probability of performing mixup or cutmix when either/both is enabled
+_C.AUG.MIXUP_PROB = 1.0
+# Probability of switching to cutmix when both mixup and cutmix enabled
+_C.AUG.MIXUP_SWITCH_PROB = 0.5
+# How to apply mixup/cutmix params. Per "batch", "pair", or "elem"
+_C.AUG.MIXUP_MODE = 'batch'
+
+# -----------------------------------------------------------------------------
+# Testing settings
+# -----------------------------------------------------------------------------
+_C.TEST = CN()
+# Whether to use center crop when testing
+_C.TEST.CROP = True
+
+# -----------------------------------------------------------------------------
+# Misc
+# -----------------------------------------------------------------------------
+# Mixed precision opt level, if O0, no amp is used ('O0', 'O1', 'O2')
+# overwritten by command line argument
+_C.AMP_OPT_LEVEL = ''
+# Path to output folder, overwritten by command line argument
+_C.OUTPUT = ''
+# Tag of experiment, overwritten by command line argument
+_C.TAG = 'default'
+# Frequency to save checkpoint
+_C.SAVE_FREQ = 1
+# Frequency to logging info
+_C.PRINT_FREQ = 10
+# Fixed random seed
+_C.SEED = 0
+# Perform evaluation only, overwritten by command line argument
+_C.EVAL_MODE = False
+# Test throughput only, overwritten by command line argument
+_C.THROUGHPUT_MODE = False
+# local rank for DistributedDataParallel, given by command line argument
+_C.LOCAL_RANK = 0
+
+
+def _update_config_from_file(config, cfg_file):
+    config.defrost()
+    with open(cfg_file, 'r') as f:
+        yaml_cfg = yaml.load(f, Loader=yaml.FullLoader)
+
+    for cfg in yaml_cfg.setdefault('BASE', ['']):
+        if cfg:
+            _update_config_from_file(
+                config, os.path.join(os.path.dirname(cfg_file), cfg)
+            )
+    print('=> merge config from {}'.format(cfg_file))
+    config.merge_from_file(cfg_file)
+    config.freeze()
+
+
+def update_config(config, args):
+    _update_config_from_file(config, args.cfg)
+
+    config.defrost()
+    if args.opts:
+        config.merge_from_list(args.opts)
+
+    # merge from specific arguments
+    if args.batch_size:
+        config.DATA.BATCH_SIZE = args.batch_size
+    if args.zip:
+        config.DATA.ZIP_MODE = True
+    if args.cache_mode:
+        config.DATA.CACHE_MODE = args.cache_mode
+    if args.resume:
+        config.MODEL.RESUME = args.resume
+    if args.accumulation_steps:
+        config.TRAIN.ACCUMULATION_STEPS = args.accumulation_steps
+    if args.use_checkpoint:
+        config.TRAIN.USE_CHECKPOINT = True
+    if args.amp_opt_level:
+        config.AMP_OPT_LEVEL = args.amp_opt_level
+    if args.tag:
+        config.TAG = args.tag
+    if args.eval:
+        config.EVAL_MODE = True
+    if args.throughput:
+        config.THROUGHPUT_MODE = True
+
+    config.freeze()
+
+
+def get_config(args):
+    """Get a yacs CfgNode object with default values."""
+    # Return a clone so that the defaults will not be altered
+    # This is for the "local variable" use pattern
+    config = _C.clone()
+    update_config(config, args)
+
+    return config

code/sota/Swin-Unet/configs/swin_tiny_patch4_window7_224_lite.yaml

@@ -0,0 +1,12 @@
+MODEL:
+  TYPE: swin
+  NAME: swin_tiny_patch4_window7_224
+  DROP_PATH_RATE: 0.2
+  PRETRAIN_CKPT: "./pretrained_ckpt/swin_tiny_patch4_window7_224.pth"
+  SWIN:
+    FINAL_UPSAMPLE: "expand_first"
+    EMBED_DIM: 96
+    DEPTHS: [ 2, 2, 2, 2 ]
+    DECODER_DEPTHS: [ 2, 2, 2, 1]
+    NUM_HEADS: [ 3, 6, 12, 24 ]
+    WINDOW_SIZE: 7

code/sota/Swin-Unet/datasets/README.md

@@ -0,0 +1,29 @@
+# Data Preparing
+
+1. Access to the synapse multi-organ dataset:
+   1. Sign up in the [official Synapse website](https://www.synapse.org/#!Synapse:syn3193805/wiki/) and download the dataset. Convert them to numpy format, clip the images within [-125, 275], normalize each 3D image to [0, 1], and extract 2D slices from 3D volume for training cases while keeping the 3D volume in h5 format for testing cases.
+   2.  You can also send an Email directly to jienengchen01 AT gmail.com to request the preprocessed data for reproduction.
+2. The directory structure of the whole project is as follows:
+
+```bash
+.
+├── TransUNet
+│   ├──datasets
+│   │       └── dataset_*.py
+│   ├──train.py
+│   ├──test.py
+│   └──...
+├── model
+│   └── vit_checkpoint
+│       └── imagenet21k
+│           ├── R50+ViT-B_16.npz
+│           └── *.npz
+└── data
+    └──Synapse
+        ├── test_vol_h5
+        │   ├── case0001.npy.h5
+        │   └── *.npy.h5
+        └── train_npz
+            ├── case0005_slice000.npz
+            └── *.npz
+```

code/sota/Swin-Unet/datasets/dataset_synapse.py

@@ -0,0 +1,82 @@
+import os
+import random
+
+import h5py
+import numpy as np
+import torch
+from scipy import ndimage
+from scipy.ndimage.interpolation import zoom
+from torch.utils.data import Dataset
+
+
+def random_rot_flip(image, label):
+    k = np.random.randint(0, 4)
+    image = np.rot90(image, k)
+    label = np.rot90(label, k)
+    axis = np.random.randint(0, 2)
+    image = np.flip(image, axis=axis).copy()
+    label = np.flip(label, axis=axis).copy()
+    return image, label
+
+
+def random_rotate(image, label):
+    angle = np.random.randint(-20, 20)
+    image = ndimage.rotate(image, angle, order=0, reshape=False)
+    label = ndimage.rotate(label, angle, order=0, reshape=False)
+    return image, label
+
+
+class RandomGenerator(object):
+    def __init__(self, output_size):
+        self.output_size = output_size
+
+    def __call__(self, sample):
+        image, label = sample['image'], sample['label']
+
+        if random.random() > 0.5:
+            image, label = random_rot_flip(image, label)
+        elif random.random() > 0.5:
+            image, label = random_rotate(image, label)
+        x, y = image.shape
+        if x != self.output_size[0] or y != self.output_size[1]:
+            image = zoom(image, (self.output_size[0] / x, self.output_size[1] / y), order=3)  # why not 3?
+            label = zoom(label, (self.output_size[0] / x, self.output_size[1] / y), order=0)
+        image = torch.from_numpy(image.astype(np.float32)).unsqueeze(0)
+        label = torch.from_numpy(label.astype(np.float32))
+        sample = {'image': image, 'label': label.long()}
+        return sample
+
+
+class Synapse_dataset(Dataset):
+    def __init__(self, base_dir, list_dir, split, transform=None):
+        self.transform = transform  # using transform in torch!
+        self.split = split
+        self.sample_list = open(os.path.join(list_dir, self.split + '.txt')).readlines()
+        self.data_dir = base_dir
+
+    def __len__(self):
+        return len(self.sample_list)
+
+    def __getitem__(self, idx):
+        if self.split in ["train", "val"] or self.sample_list[idx].strip('\n').split(",")[0].endswith(".npz"):
+            slice_name = self.sample_list[idx].strip('\n').split(",")[0]
+            if slice_name.endswith(".npz"):
+                data_path = os.path.join(self.data_dir, slice_name)
+            else:
+                data_path = os.path.join(self.data_dir, slice_name + '.npz')
+            data = np.load(data_path)
+            try:
+                image, label = data['image'], data['label']
+            except:
+                image, label = data['data'], data['seg']
+        else:
+            vol_name = self.sample_list[idx].strip('\n')
+            filepath = self.data_dir + "/{}.npy.h5".format(vol_name)
+            data = h5py.File(filepath)
+            image, label = data['image'][:], data['label'][:]
+
+        sample = {'image': image, 'label': label}
+        if self.transform:
+            sample = self.transform(sample)
+        sample['case_name'] = self.sample_list[idx].strip('\n')
+        return sample

code/sota/Swin-Unet/lists/lists_Synapse/all.lst

@@ -0,0 +1,30 @@
+case0031.npy.h5
+case0007.npy.h5
+case0009.npy.h5
+case0005.npy.h5
+case0026.npy.h5
+case0039.npy.h5
+case0024.npy.h5
+case0034.npy.h5
+case0033.npy.h5
+case0030.npy.h5
+case0023.npy.h5
+case0040.npy.h5
+case0010.npy.h5
+case0021.npy.h5
+case0006.npy.h5
+case0027.npy.h5
+case0028.npy.h5
+case0037.npy.h5
+case0008.npy.h5
+case0022.npy.h5
+case0038.npy.h5
+case0036.npy.h5
+case0032.npy.h5
+case0002.npy.h5
+case0029.npy.h5
+case0003.npy.h5
+case0001.npy.h5
+case0004.npy.h5
+case0025.npy.h5
+case0035.npy.h5

code/sota/Swin-Unet/lists/lists_Synapse/test_vol.txt

@@ -0,0 +1,12 @@
+case0008
+case0022
+case0038
+case0036
+case0032
+case0002
+case0029
+case0003
+case0001
+case0004
+case0025
+case0035