Add resumable SigLIP2 Giant formal training
CI / test (push) Successful in 13s

This commit is contained in:
2026-10-01 23:40:40 +08:00
parent 521dfb3d57
commit bfb96b76c7
11 changed files with 341 additions and 27 deletions
+9 -2
View File
@@ -33,7 +33,7 @@ def _write_split(
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
def build(config_path: Path, small: bool = False) -> Path:
def build(config_path: Path, small: bool = False, reuse_existing: bool = False) -> Path:
config = yaml.safe_load(config_path.read_text(encoding="utf-8"))
root = Path(config["data"]["root"])
root.mkdir(parents=True, exist_ok=True)
@@ -45,6 +45,12 @@ def build(config_path: Path, small: bool = False) -> Path:
}
if small:
sizes = {"train": 64, "validation": 16, "test_id": 16, "test_ood": 16}
if reuse_existing and all(
(root / f"{split}.jsonl").exists()
and sum(1 for _ in (root / f"{split}.jsonl").open(encoding="utf-8")) == size
for split, size in sizes.items()
):
return root
cursor = 0
for split, size in sizes.items():
_write_split(
@@ -64,8 +70,9 @@ def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml"))
parser.add_argument("--small", action="store_true")
parser.add_argument("--reuse-existing", action="store_true")
args = parser.parse_args()
print(build(args.config, small=args.small))
print(build(args.config, small=args.small, reuse_existing=args.reuse_existing))
if __name__ == "__main__":