Add provenance-aware paper metadata storage and PMLR collection
Metadata integrity checks / metadata (push) Successful in 1s
Metadata integrity checks / metadata (push) Successful in 1s
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
"""Read-only command-line queries for the paper index."""
|
||||
import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--db', required=True)
|
||||
parser.add_argument('--query', help='SQLite FTS5 query over title and abstract')
|
||||
parser.add_argument('--tag', help='Filter as namespace=value, e.g. arxiv:category=cs.AI')
|
||||
parser.add_argument('--venue', help='Conference acronym')
|
||||
parser.add_argument('--year', type=int)
|
||||
parser.add_argument('--status', choices=['accepted','rejected','desk_rejected','withdrawn','under_review','unknown'])
|
||||
parser.add_argument('--limit', type=int, default=20)
|
||||
parser.add_argument('--stats', action='store_true')
|
||||
args = parser.parse_args()
|
||||
conn = sqlite3.connect(Path(args.db).resolve().as_uri()+'?mode=ro', uri=True)
|
||||
conn.row_factory = sqlite3.Row
|
||||
if args.stats:
|
||||
result = {name:conn.execute('SELECT count(*) FROM '+name).fetchone()[0] for name in ('papers','source_records','submissions','decisions','tags','match_candidates')}
|
||||
result['coverage'] = [dict(r) for r in conn.execute('SELECT status,count(*) AS sources FROM edition_sources GROUP BY status')]
|
||||
result['runs'] = [dict(r) for r in conn.execute('SELECT * FROM ingest_runs ORDER BY id DESC LIMIT 10')]
|
||||
else:
|
||||
where = []; params = []
|
||||
if args.query:
|
||||
where.append('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)'); params.append(args.query)
|
||||
if args.tag:
|
||||
namespace,value = args.tag.split('=',1)
|
||||
where.append('EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'); params.extend([namespace,value])
|
||||
if args.venue or args.year or args.status:
|
||||
clause = ['s.paper_id=p.id']
|
||||
if args.venue:
|
||||
clause.append('v.acronym=?'); params.append(args.venue)
|
||||
if args.year:
|
||||
clause.append('e.year=?'); params.append(args.year)
|
||||
if args.status:
|
||||
clause.append('EXISTS(SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=? AND NOT EXISTS(SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id AND (newer.observed_at>d.observed_at OR (newer.observed_at=d.observed_at AND newer.id>d.id))))'); params.append(args.status)
|
||||
where.append('EXISTS(SELECT 1 FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+' AND '.join(clause)+')')
|
||||
sql = 'SELECT p.* FROM papers p'+(' WHERE '+' AND '.join(where) if where else '')+' ORDER BY p.id LIMIT ?'
|
||||
params.append(max(1,min(args.limit,200)))
|
||||
result = []
|
||||
for row in conn.execute(sql,params):
|
||||
paper = dict(row)
|
||||
paper['identifiers'] = [dict(r) for r in conn.execute('SELECT scheme,value FROM identifiers WHERE paper_id=?',(row['id'],))]
|
||||
result.append(paper)
|
||||
print(json.dumps(result,ensure_ascii=False,indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user