This commit is contained in:
@@ -33,7 +33,7 @@ def _write_split(
|
||||
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def build(config_path: Path, small: bool = False) -> Path:
|
||||
def build(config_path: Path, small: bool = False, reuse_existing: bool = False) -> Path:
|
||||
config = yaml.safe_load(config_path.read_text(encoding="utf-8"))
|
||||
root = Path(config["data"]["root"])
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
@@ -45,6 +45,12 @@ def build(config_path: Path, small: bool = False) -> Path:
|
||||
}
|
||||
if small:
|
||||
sizes = {"train": 64, "validation": 16, "test_id": 16, "test_ood": 16}
|
||||
if reuse_existing and all(
|
||||
(root / f"{split}.jsonl").exists()
|
||||
and sum(1 for _ in (root / f"{split}.jsonl").open(encoding="utf-8")) == size
|
||||
for split, size in sizes.items()
|
||||
):
|
||||
return root
|
||||
cursor = 0
|
||||
for split, size in sizes.items():
|
||||
_write_split(
|
||||
@@ -64,8 +70,9 @@ def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml"))
|
||||
parser.add_argument("--small", action="store_true")
|
||||
parser.add_argument("--reuse-existing", action="store_true")
|
||||
args = parser.parse_args()
|
||||
print(build(args.config, small=args.small))
|
||||
print(build(args.config, small=args.small, reuse_existing=args.reuse_existing))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user