Files
paper-atlas/tests/test_html_cache.py
T
codex-bot c97ea94b49
Metadata integrity checks / metadata (push) Failing after 2m44s
Collect ECCV 2026 public metadata with immutable resumable cache
2026-11-17 05:12:31 +08:00

91 lines
5.7 KiB
Python

from datetime import datetime, timezone
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import MagicMock
import urllib.error
from paper_atlas.io.html_cache import PublicHTMLCache, digest, NoRedirect
URL = 'https://eccv.ecva.net/virtual/2026/poster/4048'
class PublicHTMLCacheTests(unittest.TestCase):
def receipt(self, raw):
return {'url':URL,'final_url':URL,'http_status':200,'content_type':'text/html; charset=utf-8',
'bytes':len(raw),'sha256':digest(raw),'retrieved_at':datetime.now(timezone.utc).isoformat()}
def test_old_cache_reused_without_network_or_expiry(self):
with tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); raw = b'<html>metadata</html>'
body, receipt, _ = c.paths(URL); body.write_bytes(raw); receipt.write_text(json.dumps(self.receipt(raw))); os.utime(body,(0,0))
c.opener = MagicMock(); self.assertEqual(c.get(URL)[0], raw); c.opener.open.assert_not_called()
def test_pair_tampering_and_orphan_parts_fail_without_refetch(self):
for kind in ('hash','orphan','part'):
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); body, receipt, _ = c.paths(URL); body.write_bytes(b'original')
if kind == 'hash': receipt.write_text(json.dumps(self.receipt(b'different')))
if kind == 'part': body.unlink(); body.with_suffix('.html.part').write_bytes(b'partial')
c.opener = MagicMock()
with self.assertRaises(ValueError): c.get(URL)
c.opener.open.assert_not_called()
def test_http_access_failure_sticky_without_automatic_retry(self):
with tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); c.opener = MagicMock()
c.opener.open.side_effect = urllib.error.HTTPError(URL,403,'Forbidden',{},None)
with self.assertRaises(urllib.error.HTTPError): c.get(URL)
with self.assertRaisesRegex(ValueError,'Previous access'): c.get(URL)
self.assertEqual(c.opener.open.call_count,1)
def test_redirect_nonhtml_challenge_and_oversize_rejected(self):
for kind in ('redirect','type','challenge','size'):
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp,'https://eccv.ecva.net',limit=100,delay=0); c.opener=MagicMock(); r=c.opener.open.return_value.__enter__.return_value
r.status=200; r.geturl.return_value=URL; r.headers={'Content-Type':'text/html'}; r.read.return_value=b'<html>metadata</html>'
if kind=='redirect':r.geturl.return_value='https://example.org/login'
if kind=='type':r.headers={'Content-Type':'application/pdf'}
if kind=='challenge':r.read.return_value=b'<title>Just a moment</title>'
if kind=='size':r.read.return_value=b'x'*101
with self.assertRaises(ValueError):c.get(URL)
self.assertFalse(c.paths(URL)[0].exists())
self.assertIsNone(NoRedirect().redirect_request(None,None,None,None,None,None))
def test_valid_response_has_atomic_body_and_receipt(self):
with tempfile.TemporaryDirectory() as tmp:
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0);c.opener=MagicMock();r=c.opener.open.return_value.__enter__.return_value;r.status=200;r.geturl.return_value=URL;r.headers={'Content-Type':'text/html','Content-Length':'4'};r.read.return_value=b'data'
raw, receipt=c.get(URL);self.assertEqual(raw,b'data');self.assertEqual(receipt['sha256'],digest(raw));self.assertEqual(c.cached(URL),(raw,receipt));self.assertFalse(list(Path(tmp).glob('*.part')))
def test_foreign_origin_and_cache_symlink_rejected(self):
with tempfile.TemporaryDirectory() as tmp:
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0)
with self.assertRaises(ValueError):c.get('https://example.org/other')
body,_,_=c.paths(URL);target=Path(tmp)/'outside';target.write_bytes(b'other');body.symlink_to(target)
with self.assertRaises(ValueError):c.get(URL)
def test_successful_fetch_publishes_receipt_and_reuses_without_refetch(self):
with tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
response.status = 200; response.geturl.return_value = URL
raw = b'<html>public metadata</html>'
response.headers = {'Content-Type':'text/html','Content-Length':str(len(raw))}; response.read.return_value = raw
self.assertEqual(c.get(URL)[0], raw); self.assertEqual(c.get(URL)[0], raw)
self.assertEqual(c.opener.open.call_count, 1)
body, receipt, _ = c.paths(URL)
self.assertTrue(body.exists()); self.assertTrue(receipt.exists())
self.assertFalse(body.with_suffix('.html.part').exists())
self.assertFalse(receipt.with_suffix('.json.part').exists())
def test_changed_url_is_rejected_before_reading_body(self):
with tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
response.status = 200; response.geturl.return_value = 'https://example.org/login'
response.headers = {'Content-Type':'text/html'}
with self.assertRaises(ValueError): c.get(URL)
response.read.assert_not_called()