name: Metadata integrity checks on: push: branches: [main, 'codex/**'] jobs: metadata: runs-on: ces-control env: ARCHIVE_SHA: ${{ github.sha }} PYTHONDONTWRITEBYTECODE: '1' steps: - name: Verify metadata storage and query commands run: | python3 -B - <<'PY' import io, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request sha = os.environ['ARCHIVE_SHA'] if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha): raise SystemExit('Invalid source commit') # Public repository archive: no owner credentials or runner SSH keys. url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz' with urllib.request.urlopen(url, timeout=60) as response: archive = response.read(16*1024*1024+1) if len(archive)>16*1024*1024: raise SystemExit('Unexpectedly large source archive') with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp: root = pathlib.Path(tmp) with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: tar.extractall(root, filter='data') candidates = list(root.glob('*/pyproject.toml')) if len(candidates)!=1: raise SystemExit('Unexpected repository archive layout') src = candidates[0].parent # Build and installation checks run only on the approved remote runner. venv = root / 'venv' subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True) python = str(venv / 'bin/python') subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8'],check=True) testenv = os.environ.copy() service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf') if service_file.is_file(): testenv.update(PGSERVICEFILE=str(service_file),PAPER_ATLAS_TEST_PG_SERVICE='foundry-research') subprocess.run([python,'-B','-m','unittest','discover','-s','tests','-t','.','-v'],cwd=src,env=testenv,check=True) wheels = root / 'wheels' subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True) wheel, = wheels.glob('paper_atlas-*.whl') subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True) outside = root / 'outside-checkout' outside.mkdir() subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','import','records','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','import','arxiv','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','backfill','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True) smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()" subprocess.run([python,'-I','-B','-c',smoke],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','query','--db','installed.sqlite','--stats'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--db','installed.sqlite','--examples','0'],cwd=outside,check=True) PY