Add bounded 2B tensor and block similarity pipeline
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
"""Supervise the one-model job; no automatic download of subsequent models."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--root', type=pathlib.Path, required=True)
|
||||
args = p.parse_args()
|
||||
root = args.root.resolve()
|
||||
root.mkdir(exist_ok=True, parents=True)
|
||||
status_file = root / 'status.json'
|
||||
start = time.time()
|
||||
env = dict(os.environ, OPENBLAS_NUM_THREADS='1', OMP_NUM_THREADS='1')
|
||||
|
||||
def status(phase, **extra):
|
||||
status_file.write_text(json.dumps({'phase': phase, 'updated_unix': time.time(),
|
||||
'supervisor_started_unix': start, **extra}, indent=2))
|
||||
print(phase, flush=True)
|
||||
|
||||
def command(module, log, timeout, extra=()):
|
||||
with (root / log).open('w') as stream:
|
||||
subprocess.run([sys.executable, '-B', '-m', module, '--root', str(root), *extra],
|
||||
check=True, timeout=timeout, env=env, stdout=stream, stderr=subprocess.STDOUT)
|
||||
|
||||
try:
|
||||
status('waiting_for_download')
|
||||
download_wait_start = time.monotonic()
|
||||
while True:
|
||||
manifest_path = root / 'manifest.json'
|
||||
if manifest_path.exists():
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
if manifest.get('completed_unix') and all((root / 'weights' / f['name']).exists() for f in manifest['files']):
|
||||
break
|
||||
if time.monotonic() - download_wait_start > 7200:
|
||||
raise TimeoutError('Download wait exceeded 2 hours; no analysis started')
|
||||
time.sleep(10)
|
||||
status('analyzing', analysis_hard_limit_seconds=7200)
|
||||
command('pilot.analyze', 'analysis.log', 7200, ('--seconds', '6600'))
|
||||
status('plotting')
|
||||
command('pilot.plot', 'plot.log', 300)
|
||||
status('validating')
|
||||
command('pilot.validate', 'validate.log', 120)
|
||||
with (root / 'environment.txt').open('w') as f:
|
||||
subprocess.run([sys.executable, '-B', '-m', 'pip', 'freeze'], check=True, stdout=f, env=env)
|
||||
status('results_ready_weights_retained_until_archive_copy')
|
||||
except Exception as e:
|
||||
status('failed', error_type=type(e).__name__, error=str(e))
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user