commit a677ffadd8e5b3f22df05ba26c6acfac6afcfb1a Author: YannAhlgrim Date: Thu Jul 9 12:27:16 2026 +0000 Initial commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..90e9686 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +env/ +venv/ +.git +.cache +cache/*.json diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..84723da --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +# OpenAlex API credentials +# Get a free key at https://openalex.org/rest-api +OA_API_KEY=your_openalex_api_key_here +# OA_EMAIL=your@email.com diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9ee8499 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.env +__pycache__/ +*.pyc +*.pyo +*.pyd +.cache/ +cache/*.json +venv/ +env/ +.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7ce54cf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends gcc \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +EXPOSE 5000 + +CMD ["gunicorn", "--bind", "0.0.0.0:5000", "--workers", "2", "--timeout", "120", "--access-logfile", "-", "app:app"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..13f1916 --- /dev/null +++ b/README.md @@ -0,0 +1,95 @@ +# Linked Papers + +A lightweight, self-hosted web app for exploring academic paper reference graphs. Search for a paper and see an interactive graph of the papers it cites. Node size is proportional to citation count. + +## Features + +- Search by paper title, DOI, or arXiv ID +- Interactive force-directed graph (vis-network) +- Node size based on citation count +- Click a node to see details; double-click to re-center the graph +- File-based JSON cache (survives container restarts) +- Works offline for previously cached graphs +- Uses OpenAlex for all paper data +- Shows only outgoing references (papers the seed paper cites) +- Dockerized + +## Data Source + +- **OpenAlex API:** https://openalex.org/ + +OpenAlex is free. Anonymous usage has rate limits; if you hit them often, get a free API key and set it in a `.env` file (see below). + +## Quick Start + +```bash +# (Optional) Add your OpenAlex API key +cp .env.example .env +# edit .env with your OA_API_KEY and OA_EMAIL + +# Build and run +docker-compose up -d --build + +# Open locally +# http://127.0.0.1:5000 +``` + +## Configuration + +Edit `docker-compose.yml` to tune these environment variables: + +| Variable | Default | Description | +|----------|---------|-------------| +| `CACHE_TTL_DAYS` | 30 | Days before cached graph is refreshed | +| `SEARCH_LIMIT` | 20 | Number of search results, sorted by citation count | +| `REFERENCE_LIMIT` | 100 | Top N papers cited by the seed paper (outgoing) | +| `OA_API_KEY` | (from `.env`) | OpenAlex API key (higher rate limits) | +| `OA_EMAIL` | (from `.env`) | Email for OpenAlex polite pool (recommended) | + +### Getting an API key (optional but recommended) + +- **OpenAlex:** https://openalex.org/rest-api + +Set it in `.env` and restart: + +```bash +cp .env.example .env +# edit .env with your OA_API_KEY and OA_EMAIL +docker-compose up -d +``` + +## Cache + +Cached graphs are stored as JSON files in `./cache/`. To clear the cache: + +```bash +rm -f cache/*.json +``` + +You can also inspect the cache files directly — they are plain JSON. + +## Troubleshooting + +### 429 / "Too Many Requests" or 503 from OpenAlex + +The public IP you are running from has hit OpenAlex's rate limit. Set `OA_API_KEY` and `OA_EMAIL` for higher limits. + +### Offline mode + +If the APIs are unreachable but a cached graph exists, the app returns the cached graph automatically. + +## Useful Commands + +```bash +# View logs +docker-compose logs -f + +# Restart +docker-compose restart + +# Stop +docker-compose down + +# Rebuild after code changes +docker-compose up -d --build +``` diff --git a/app.py b/app.py new file mode 100644 index 0000000..ff888be --- /dev/null +++ b/app.py @@ -0,0 +1,332 @@ +import os +import json +import re +import time +from datetime import datetime, timedelta +from pathlib import Path + +import requests +from flask import Flask, jsonify, render_template, request + +app = Flask(__name__) + +CACHE_DIR = Path("cache") +CACHE_DIR.mkdir(exist_ok=True) +CACHE_TTL_DAYS = int(os.environ.get("CACHE_TTL_DAYS", "30")) + +OA_API = "https://api.openalex.org" + +SEARCH_LIMIT = int(os.environ.get("SEARCH_LIMIT", "20")) +REFERENCE_LIMIT = int(os.environ.get("REFERENCE_LIMIT", "100")) + +REQUEST_TIMEOUT = 30 +API_DELAY_SECONDS = 0.2 +MAX_RETRIES = 3 +RETRY_BACKOFF = 2 # seconds + +OA_EMAIL = os.environ.get("OA_EMAIL", "") +OA_API_KEY = os.environ.get("OA_API_KEY") + + +def safe_paper_id(paper_id: str) -> str: + return re.sub(r"[^a-zA-Z0-9_-]", "_", paper_id)[:128] + + +def cache_path(paper_id: str) -> Path: + return CACHE_DIR / f"{safe_paper_id(paper_id)}.json" + + +def is_cache_fresh(path: Path) -> bool: + if not path.exists(): + return False + mtime = datetime.fromtimestamp(path.stat().st_mtime) + return datetime.now() - mtime < timedelta(days=CACHE_TTL_DAYS) + + +def load_cache(paper_id: str): + path = cache_path(paper_id) + if path.exists(): + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + return None + + +def save_cache(paper_id: str, data: dict): + path = cache_path(paper_id) + payload = { + "paper_id": paper_id, + "created_at": datetime.now().isoformat(), + "data": data, + } + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False, indent=2) + + +def oa_get(url, params=None): + headers = { + "User-Agent": "connected-papers-local/1.0 (personal use)", + "Accept": "application/json", + } + params = params or {} + if OA_EMAIL: + params["mailto"] = OA_EMAIL + if OA_API_KEY: + params["api_key"] = OA_API_KEY + + last_exception = None + for attempt in range(MAX_RETRIES): + try: + r = requests.get(url, params=params, headers=headers, timeout=REQUEST_TIMEOUT) + r.raise_for_status() + return r.json() + except requests.HTTPError as e: + last_exception = e + status = getattr(e.response, "status_code", None) + if status in (429, 503) and attempt < MAX_RETRIES - 1: + time.sleep(RETRY_BACKOFF * (attempt + 1)) + continue + raise + except requests.RequestException as e: + last_exception = e + if attempt < MAX_RETRIES - 1: + time.sleep(RETRY_BACKOFF * (attempt + 1)) + continue + raise + raise last_exception + + +def oa_work_to_paper(work: dict) -> dict: + authors = [ + {"name": a["author"]["display_name"]} + for a in work.get("authorships", []) + if a.get("author") + ] + title = work.get("display_name") or work.get("title") or "" + if not title.strip(): + # fallback to DOI or arXiv ID if title is missing + ids = work.get("ids", {}) + title = ids.get("arxiv") or ids.get("doi", "").replace("https://doi.org/", "") or "Untitled" + return { + "paperId": work.get("id"), + "title": title, + "authors": authors, + "year": work.get("publication_year"), + "citationCount": work.get("cited_by_count", 0), + "referenceCount": len(work.get("referenced_works", [])), + } + + +def sort_by_citations(papers: list) -> list: + return sorted(papers, key=lambda p: p.get("citationCount") or 0, reverse=True) + + +def looks_like_doi(query: str) -> bool: + return query.startswith("10.") and "/" in query + + +def looks_like_arxiv(query: str) -> bool: + q = query.lower() + if q.startswith("arxiv:") or q.startswith("arXiv:"): + return True + import re + return bool(re.fullmatch(r"\d{4}\.\d{4,5}(v\d+)?", query)) + + +def extract_arxiv_id(query: str) -> str: + q = query.lower() + if q.startswith("arxiv:"): + return query[6:].strip() + return query.strip() + + +def resolve_arxiv(arxiv_id: str): + """Use the arXiv API to get title and authors. Returns dict or None.""" + try: + url = f"http://export.arxiv.org/api/query?id_list={arxiv_id}" + r = requests.get(url, timeout=REQUEST_TIMEOUT, allow_redirects=True) + r.raise_for_status() + import xml.etree.ElementTree as ET + root = ET.fromstring(r.text) + ns = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"} + entry = root.find("atom:entry", ns) + if entry is None: + return None + title_elem = entry.find("atom:title", ns) + title = title_elem.text.strip() if title_elem is not None and title_elem.text else "" + authors = [ + {"name": author.find("atom:name", ns).text.strip()} + for author in entry.findall("atom:author", ns) + if author.find("atom:name", ns) is not None and author.find("atom:name", ns).text + ] + return {"title": title, "authors": authors} + except Exception: + return None + + +def lookup_paper(query: str): + """Try DOI or arXiv direct lookup. Return paper dict or None.""" + if looks_like_doi(query): + try: + work = oa_get(f"{OA_API}/works/doi:{query}") + return oa_work_to_paper(work) + except requests.RequestException: + return None + + if looks_like_arxiv(query): + arxiv_id = extract_arxiv_id(query) + arxiv_info = resolve_arxiv(arxiv_id) + if arxiv_info and arxiv_info.get("title"): + # Search OpenAlex by the exact arXiv title and return candidates + url = f"{OA_API}/works" + params = {"search": arxiv_info["title"], "per-page": 10} + data = oa_get(url, params) + papers = [oa_work_to_paper(w) for w in data.get("results", [])] + return sort_by_citations(papers) + + return None + + +def search_papers(query: str): + direct = lookup_paper(query) + if direct: + if isinstance(direct, list): + return direct[:SEARCH_LIMIT] + return [direct] + + url = f"{OA_API}/works" + params = {"search": query, "per-page": SEARCH_LIMIT} + data = oa_get(url, params) + papers = [oa_work_to_paper(w) for w in data.get("results", [])] + return sort_by_citations(papers) + + +def get_paper(paper_id: str): + url = f"{OA_API}/works/{paper_id}" + return oa_work_to_paper(oa_get(url)) + + +def get_references(paper_id: str, limit: int = REFERENCE_LIMIT): + time.sleep(API_DELAY_SECONDS) + work = oa_get(f"{OA_API}/works/{paper_id}") + ref_ids = work.get("referenced_works", []) + if not ref_ids: + return [] + + short_ids = [oid.split("/")[-1] for oid in ref_ids] + refs = [] + chunk_size = 50 + fetch_limit = min(len(short_ids), limit * 3) + for i in range(0, fetch_limit, chunk_size): + chunk = short_ids[i : i + chunk_size] + ids_filter = "|".join(chunk) + url = f"{OA_API}/works" + params = {"filter": f"openalex:{ids_filter}", "per-page": chunk_size} + data = oa_get(url, params) + refs.extend([oa_work_to_paper(w) for w in data.get("results", [])]) + + return sort_by_citations(refs)[:limit] + + +def format_authors(authors: list, max_len: int = 3) -> str: + if not authors: + return "Unknown" + names = [a.get("name", "Unknown") for a in authors] + if len(names) > max_len: + return ", ".join(names[:max_len]) + " et al." + return ", ".join(names) + + +def paper_to_node(paper: dict, group: str) -> dict: + title = paper.get("title") or "Untitled" + citation_count = paper.get("citationCount") or 0 + return { + "id": paper.get("paperId"), + "label": title, + "title": f"{title}
{format_authors(paper.get('authors', []))} ({paper.get('year', 'n.d.')})
Citations: {citation_count}", + "value": citation_count, + "group": group, + "year": paper.get("year"), + "authors": format_authors(paper.get("authors", [])), + "citationCount": citation_count, + } + + +def build_graph(paper_id: str): + paper = get_paper(paper_id) + if not paper or not paper.get("paperId"): + raise ValueError("Paper not found") + + seed_id = paper["paperId"] + nodes = [paper_to_node(paper, "seed")] + edges = [] + + references = get_references(seed_id, REFERENCE_LIMIT) + + for p in references: + nodes.append(paper_to_node(p, "reference")) + edges.append({"from": seed_id, "to": p["paperId"]}) + + return { + "seed_id": seed_id, + "seed_title": paper.get("title"), + "source": "openalex", + "nodes": nodes, + "edges": edges, + "cached_at": datetime.now().isoformat(), + } + + +@app.route("/") +def index(): + return render_template("index.html") + + +@app.route("/api/search") +def api_search(): + q = request.args.get("q", "").strip() + if not q: + return jsonify({"error": "Missing query"}), 400 + try: + papers = search_papers(q) + return jsonify({"papers": papers}) + except requests.RequestException as e: + return jsonify({"error": str(e)}), 502 + + +@app.route("/api/graph/") +def api_graph(paper_id: str): + paper_id = paper_id.strip() + if not paper_id: + return jsonify({"error": "Missing paper id"}), 400 + + cached = load_cache(paper_id) + path = cache_path(paper_id) + + if cached and is_cache_fresh(path): + return jsonify(cached["data"]) + + try: + graph = build_graph(paper_id) + save_cache(paper_id, graph) + return jsonify(graph) + except requests.RequestException: + if cached: + return jsonify(cached["data"]) + return jsonify({"error": "OpenAlex is unreachable. No cached graph available."}), 502 + except Exception as e: + if cached: + return jsonify(cached["data"]) + return jsonify({"error": f"Failed to build graph: {e}"}), 500 + + +@app.route("/api/paper/") +def api_paper(paper_id: str): + try: + paper = get_paper(paper_id) + return jsonify(paper) + except requests.RequestException as e: + return jsonify({"error": str(e)}), 502 + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=5000, debug=False) diff --git a/cache/.gitkeep b/cache/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f550b7c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +version: "3.8" + +services: + connected-papers: + build: . + container_name: connected-papers + restart: unless-stopped + ports: + - "127.0.0.1:5000:5000" + volumes: + - ./cache:/app/cache + environment: + - CACHE_TTL_DAYS=30 + - SEARCH_LIMIT=20 + - REFERENCE_LIMIT=100 + - OA_API_KEY=${OA_API_KEY:-} + - OA_EMAIL=${OA_EMAIL:-} + networks: + - caddy_mesh + +networks: + caddy_mesh: + external: true diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..974fd75 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +flask==3.0.3 +requests==2.32.3 +gunicorn==22.0.0 diff --git a/static/app.js b/static/app.js new file mode 100644 index 0000000..a1bb790 --- /dev/null +++ b/static/app.js @@ -0,0 +1,186 @@ +const searchForm = document.getElementById('search-form'); +const searchInput = document.getElementById('search-input'); +const statusEl = document.getElementById('status'); +const resultsEl = document.getElementById('search-results'); +const resultsList = document.getElementById('results-list'); +const graphContainer = document.getElementById('graph-container'); +const infoPanel = document.getElementById('info-panel'); +const infoContent = document.getElementById('info-content'); + +let network = null; + +function showStatus(msg, isError = false) { + statusEl.textContent = msg; + statusEl.className = isError ? 'error' : 'loading'; +} + +function clearStatus() { + statusEl.textContent = ''; + statusEl.className = ''; +} + +function escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; +} + +function formatAuthors(authors, maxLen = 3) { + if (!authors || authors.length === 0) return 'Unknown'; + const names = authors.map(a => a.name || 'Unknown'); + if (names.length > maxLen) { + return names.slice(0, maxLen).join(', ') + ' et al.'; + } + return names.join(', '); +} + +function showResults(papers) { + resultsList.innerHTML = ''; + resultsEl.classList.remove('hidden'); + graphContainer.innerHTML = ''; + infoPanel.classList.add('hidden'); + + papers.forEach(paper => { + const li = document.createElement('li'); + const authors = formatAuthors(paper.authors); + li.innerHTML = ` + ${escapeHtml(paper.title || 'Untitled')} + ${escapeHtml(authors)} — ${paper.year || 'n.d.'} — Citations: ${paper.citationCount || 0} + `; + li.addEventListener('click', () => { + resultsEl.classList.add('hidden'); + loadGraph(paper.paperId); + }); + resultsList.appendChild(li); + }); +} + +async function search(query) { + showStatus('Searching...'); + try { + const res = await fetch(`/api/search?q=${encodeURIComponent(query)}`); + const data = await res.json(); + clearStatus(); + if (data.error) throw new Error(data.error); + if (!data.papers || data.papers.length === 0) { + showStatus('No papers found.', true); + return; + } + showResults(data.papers); + } catch (err) { + showStatus(err.message, true); + } +} + +async function loadGraph(paperId) { + showStatus('Loading reference graph...'); + try { + const res = await fetch(`/api/graph/${encodeURIComponent(paperId)}`); + const data = await res.json(); + clearStatus(); + if (data.error) throw new Error(data.error); + renderGraph(data); + } catch (err) { + showStatus(err.message, true); + } +} + +function renderGraph(graph) { + graphContainer.innerHTML = ''; + infoPanel.classList.add('hidden'); + + const container = document.createElement('div'); + container.id = 'graph'; + graphContainer.appendChild(container); + + if (graph.nodes.length <= 1) { + container.innerHTML = ` +
+

No references found in OpenAlex for this paper yet.

+

This often happens with very recent preprints — OpenAlex hasn't ingested the reference list.

+
+ `; + const nodes = new vis.DataSet(graph.nodes); + const edges = new vis.DataSet(graph.edges); + const options = { nodes: { shape: 'dot', color: '#ff6b6b' } }; + network = new vis.Network(container, { nodes, edges }, options); + return; + } + + const nodes = new vis.DataSet(graph.nodes); + const edges = new vis.DataSet(graph.edges); + + const options = { + nodes: { + shape: 'dot', + scaling: { + min: 10, + max: 60, + label: { enabled: true, min: 12, max: 22 } + }, + font: { size: 14, color: '#333' } + }, + edges: { + width: 1, + color: { color: '#888', highlight: '#000' }, + arrows: { to: { enabled: true, scaleFactor: 0.5 } }, + smooth: { type: 'continuous' } + }, + groups: { + seed: { color: '#ff6b6b' }, + reference: { color: '#45b7d1' } + }, + physics: { + stabilization: false, + barnesHut: { + gravitationalConstant: -3000, + centralGravity: 0.3, + springLength: 150, + springConstant: 0.04, + damping: 0.09 + } + }, + interaction: { + hover: true, + tooltipDelay: 200 + } + }; + + network = new vis.Network(container, { nodes, edges }, options); + + network.on('click', (params) => { + if (params.nodes.length > 0) { + const nodeId = params.nodes[0]; + const node = nodes.get(nodeId); + showNodeInfo(node); + } + }); + + network.on('doubleClick', (params) => { + if (params.nodes.length > 0) { + const nodeId = params.nodes[0]; + loadGraph(nodeId); + } + }); +} + +function showNodeInfo(node) { + infoPanel.classList.remove('hidden'); + infoContent.innerHTML = ` +

${escapeHtml(node.label)}

+

Authors: ${escapeHtml(node.authors)}

+

Year: ${node.year || 'n.d.'}

+

Citations: ${node.citationCount}

+ +

Tip: you can also double-click a node to re-center.

+ `; + document.getElementById('recenter-btn').addEventListener('click', () => { + loadGraph(node.id); + }); +} + +searchForm.addEventListener('submit', (e) => { + e.preventDefault(); + const query = searchInput.value.trim(); + if (query) search(query); +}); diff --git a/static/style.css b/static/style.css new file mode 100644 index 0000000..bb18183 --- /dev/null +++ b/static/style.css @@ -0,0 +1,185 @@ +* { + box-sizing: border-box; +} + +body { + margin: 0; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: #f5f5f5; + color: #333; +} + +#app { + position: relative; + display: flex; + flex-direction: column; + height: 100vh; +} + +header { + background: #fff; + padding: 1rem 2rem; + box-shadow: 0 2px 4px rgba(0,0,0,0.1); + display: flex; + align-items: center; + gap: 1.5rem; + flex-wrap: wrap; + z-index: 20; +} + +header h1 { + margin: 0; + font-size: 1.5rem; +} + +#search-form { + display: flex; + gap: 0.5rem; + flex: 1; + min-width: 300px; +} + +#search-input { + flex: 1; + padding: 0.5rem 0.75rem; + border: 1px solid #ddd; + border-radius: 4px; + font-size: 1rem; +} + +button { + padding: 0.5rem 1rem; + background: #0066cc; + color: #fff; + border: none; + border-radius: 4px; + cursor: pointer; + font-size: 1rem; +} + +button:hover { + background: #0052a3; +} + +#status { + padding: 0.5rem 2rem; + font-size: 0.9rem; + min-height: 2rem; +} + +#status.loading { color: #0066cc; } +#status.error { color: #cc0000; } + +#help { + background: #fff; + padding: 0.75rem 2rem; + border-bottom: 1px solid #eee; + font-size: 0.9rem; + color: #555; +} + +#help p { + margin: 0; +} + +#search-results { + background: #fff; + padding: 1rem 2rem; + border-bottom: 1px solid #ddd; + max-height: 250px; + overflow-y: auto; + z-index: 20; +} + +#search-results.hidden, +#info-panel.hidden { + display: none; +} + +#search-results h2 { + margin-top: 0; + font-size: 1rem; +} + +#results-list { + list-style: none; + padding: 0; + margin: 0; +} + +#results-list li { + padding: 0.75rem; + border-bottom: 1px solid #eee; + cursor: pointer; +} + +#results-list li:hover { + background: #f0f8ff; +} + +#results-list li strong { + display: block; + margin-bottom: 0.25rem; +} + +#results-list li span { + font-size: 0.85rem; + color: #666; +} + +main { + flex: 1; + position: relative; + overflow: hidden; + background: #fff; +} + +#graph { + width: 100%; + height: 100%; +} + +.empty-graph { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + height: 100%; + text-align: center; + color: #666; + padding: 2rem; +} + +.empty-graph p { + margin: 0.5rem 0; + max-width: 600px; +} + +#info-panel { + position: absolute; + top: 80px; + right: 20px; + width: 320px; + max-height: calc(100vh - 120px); + overflow-y: auto; + background: #fff; + border-radius: 8px; + box-shadow: 0 4px 12px rgba(0,0,0,0.15); + padding: 1rem; + z-index: 10; +} + +#info-panel h2 { + margin-top: 0; + font-size: 1.1rem; +} + +#info-content h3 { + margin-top: 0; + font-size: 1rem; +} + +#info-content p { + margin: 0.5rem 0; + font-size: 0.9rem; +} diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..1ed9f4f --- /dev/null +++ b/templates/index.html @@ -0,0 +1,45 @@ + + + + + + Connected Papers + + + + +
+
+

Connected Papers

+
+ + +
+
+ +
+ +
+

+ Search for a paper by title, DOI, or arXiv ID. Results are sorted by citation count (most cited first). + Click a result to load it in the center and see the papers it cites. Click any reference node for details, + then click Re-center (or double-click) to explore that paper's references. +

+
+ + + +
+ + +
+ + + +