343 lines
14 KiB
Python
343 lines
14 KiB
Python
"""Fetch new items from arXiv, journal RSS feeds, and Crossref.
|
|
|
|
Each item is a dict with keys:
|
|
id -- arXiv ID without version (preprints) or lower-cased DOI (journals)
|
|
doi -- journal DOI, or 10.48550/arXiv.<id> for preprints without one
|
|
arxiv_id -- arXiv ID without version, or None
|
|
version -- arXiv version ("v2"), or None
|
|
title -- paper title
|
|
authors -- list of author name strings
|
|
date -- ISO date string YYYY-MM-DD (arXiv announcement or publication)
|
|
abstract -- abstract text (may be empty); LaTeX in arXiv abstracts kept as written
|
|
source -- "arXiv", "J. Rheol.", or a feed name
|
|
link -- URL to the paper
|
|
categories -- list of arXiv categories (empty for journals)
|
|
announce_type -- arXiv: new, cross, replace, replace-cross; journals: None
|
|
fetched_at -- UTC timestamp of the fetch
|
|
|
|
Sources, as measured on 2026-09-27:
|
|
arXiv RSS rss.arxiv.org/rss/<cat>: one day's announcement, full abstracts,
|
|
announce type; empty on Saturday and Sunday (<skipDays>).
|
|
arXiv OAI oaipmh.arxiv.org: date-range backfill with full metadata. The
|
|
datestamp is the last metadata change, so filter on the v1 date.
|
|
arXiv API export.arxiv.org/api/query: returned HTTP 406 to every request.
|
|
fetch_arxiv is kept in case it comes back.
|
|
AIP RSS J. Rheol. current and open issues: full abstracts and DOIs, no
|
|
authors, and only the issue in progress.
|
|
Crossref every J. Rheol. article with authors; abstracts for nearly all.
|
|
|
|
arXiv answers bursts of requests with HTTP 406 as well, so every request goes
|
|
through http_get, which spaces and retries them.
|
|
"""
|
|
|
|
import html
|
|
import json
|
|
import re
|
|
import time
|
|
import urllib.parse
|
|
import xml.etree.ElementTree as ET
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from email.utils import parsedate_to_datetime
|
|
|
|
import httpx
|
|
|
|
from art import env
|
|
|
|
def _user_agent() -> str:
|
|
email = env.contact_email()
|
|
return f"art-fetcher/0.1 (rheology literature tool{'; ' + email if email else ''})"
|
|
|
|
|
|
def _mailto() -> str:
|
|
"""Crossref "polite pool" parameter, empty if no contact email is configured."""
|
|
email = env.contact_email()
|
|
return f"&mailto={urllib.parse.quote(email)}" if email else ""
|
|
|
|
_last_request: dict[str, float] = {}
|
|
|
|
|
|
def http_get(url: str, delay_s: float = 3.0, tries: int = 4, timeout: float = 120,
|
|
headers: dict | None = None) -> bytes:
|
|
"""GET with a minimum delay between requests to the same host, retrying 406/429/5xx."""
|
|
host = urllib.parse.urlparse(url).netloc
|
|
for attempt in range(1, tries + 1):
|
|
wait = _last_request.get(host, 0) + delay_s - time.monotonic()
|
|
if wait > 0:
|
|
time.sleep(wait)
|
|
_last_request[host] = time.monotonic()
|
|
resp = httpx.get(url, headers={"User-Agent": _user_agent(), **(headers or {})}, timeout=timeout,
|
|
follow_redirects=True)
|
|
if resp.status_code == 200:
|
|
return resp.content
|
|
if resp.status_code in (406, 429, 500, 502, 503, 504) and attempt < tries:
|
|
time.sleep(10 * attempt)
|
|
continue
|
|
resp.raise_for_status()
|
|
raise httpx.HTTPStatusError(f"HTTP {resp.status_code}", request=resp.request, response=resp)
|
|
raise RuntimeError("unreachable")
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
|
|
|
|
def _clean(text: str) -> str:
|
|
"""Strip HTML/JATS tags, unescape entities, collapse whitespace."""
|
|
text = re.sub(r"<[^>]+>", " ", text or "")
|
|
return re.sub(r"\s+", " ", html.unescape(text)).strip()
|
|
|
|
|
|
def _arxiv_item(arxiv_id: str, version: str | None, title: str, authors: list[str],
|
|
day: str, abstract: str, categories: list[str], announce_type: str | None,
|
|
doi: str | None = None) -> dict:
|
|
return {
|
|
"id": arxiv_id,
|
|
"doi": doi.lower() if doi else f"10.48550/arXiv.{arxiv_id}",
|
|
"arxiv_id": arxiv_id,
|
|
"version": version,
|
|
"title": re.sub(r"\s+", " ", title).strip(),
|
|
"authors": authors,
|
|
"date": day,
|
|
"abstract": re.sub(r"\s+", " ", abstract).strip(),
|
|
"source": "arXiv",
|
|
"link": f"https://arxiv.org/abs/{arxiv_id}",
|
|
"categories": categories,
|
|
"announce_type": announce_type,
|
|
"fetched_at": _now(),
|
|
}
|
|
|
|
|
|
# -- arXiv RSS (daily) ---------------------------------------------------------
|
|
|
|
_RSS_DESC = re.compile(r"arXiv:(\S+?)(v\d+)?\s+Announce Type:\s*(\S+)\s*Abstract:\s*(.*)", re.S)
|
|
|
|
|
|
def fetch_arxiv_rss(category: str) -> list[dict]:
|
|
"""Return the day's announcement for one arXiv category from rss.arxiv.org.
|
|
|
|
The per-item description reads "arXiv:ID vN Announce Type: new Abstract: ...".
|
|
Items carry no date of their own; the channel pubDate is the announcement date.
|
|
"""
|
|
ns = {"arxiv": "http://arxiv.org/schemas/atom",
|
|
"dc": "http://purl.org/dc/elements/1.1/"}
|
|
root = ET.fromstring(http_get(f"https://rss.arxiv.org/rss/{category}"))
|
|
channel = root.find("channel")
|
|
pub = channel.findtext("pubDate")
|
|
day = parsedate_to_datetime(pub).date().isoformat() if pub else date.today().isoformat()
|
|
items = []
|
|
for it in channel.findall("item"):
|
|
m = _RSS_DESC.match(it.findtext("description") or "")
|
|
if not m:
|
|
continue
|
|
arxiv_id, version, announce_type, abstract = m.groups()
|
|
creators = it.findtext("dc:creator", namespaces=ns) or ""
|
|
items.append(_arxiv_item(
|
|
arxiv_id, version, it.findtext("title") or "",
|
|
[a.strip() for a in creators.split(",") if a.strip()],
|
|
day, abstract, [c.text for c in it.findall("category") if c.text],
|
|
it.findtext("arxiv:announce_type", namespaces=ns) or announce_type,
|
|
doi=it.findtext("arxiv:DOI", namespaces=ns)))
|
|
return items
|
|
|
|
|
|
# -- arXiv OAI-PMH (backfill, exemplars) ------------------------------------------
|
|
|
|
# arXivRaw, not the arXiv format: the latter's <created> is not always the v1
|
|
# date (2510.01365 reports its v2 date, 2026-09-21), while arXivRaw lists
|
|
# every version with its date.
|
|
_OAI = {"o": "http://www.openarchives.org/OAI/2.0/", "a": "http://arxiv.org/OAI/arXivRaw/"}
|
|
|
|
|
|
def _split_authors(authors: str) -> list[str]:
|
|
""" "A. B, C. D and E. F (Univ)" -> ["A. B", "C. D", "E. F"] """
|
|
authors = re.sub(r"\([^)]*\)", "", re.sub(r"\s+", " ", authors))
|
|
return [a.strip() for a in re.split(r",\s*|\s+and\s+", authors) if a.strip()]
|
|
|
|
|
|
def _oai_record(rec: ET.Element, lag_days: int = 7) -> dict | None:
|
|
"""One arXivRaw record as an item dated by its OAI datestamp (about the announcement).
|
|
|
|
announce_type is "new" when v1 was submitted within `lag_days` of the
|
|
datestamp, and "replace" for an older paper with a new version or other
|
|
metadata change.
|
|
"""
|
|
meta = rec.find(".//a:arXivRaw", _OAI)
|
|
if meta is None:
|
|
return None
|
|
stamp = rec.findtext("o:header/o:datestamp", namespaces=_OAI)
|
|
versions = meta.findall("a:version", _OAI)
|
|
v1 = parsedate_to_datetime(versions[0].findtext("a:date", namespaces=_OAI)).date()
|
|
fresh = (date.fromisoformat(stamp) - v1).days <= lag_days
|
|
return _arxiv_item(
|
|
meta.findtext("a:id", namespaces=_OAI), versions[-1].get("version"),
|
|
meta.findtext("a:title", namespaces=_OAI) or "",
|
|
_split_authors(meta.findtext("a:authors", namespaces=_OAI) or ""), stamp,
|
|
meta.findtext("a:abstract", namespaces=_OAI) or "",
|
|
(meta.findtext("a:categories", namespaces=_OAI) or "").split(),
|
|
"new" if fresh else "replace",
|
|
doi=meta.findtext("a:doi", namespaces=_OAI))
|
|
|
|
|
|
def fetch_arxiv_oai(oai_set: str, since: str, until: str | None = None,
|
|
categories: list[str] | None = None, delay_s: float = 3.0,
|
|
lag_days: int = 7) -> list[dict]:
|
|
"""Return papers newly announced on or after `since` from an OAI-PMH set (e.g. "physics:cond-mat").
|
|
|
|
OAI selects by metadata datestamp, which also catches old papers with a
|
|
new version or other metadata change; those are dropped (see _oai_record).
|
|
With `categories`, keep only papers listed in at least one of them.
|
|
"""
|
|
params = {"verb": "ListRecords", "metadataPrefix": "arXivRaw", "set": oai_set, "from": since}
|
|
if until:
|
|
params["until"] = until
|
|
items = []
|
|
while True:
|
|
url = "https://oaipmh.arxiv.org/oai?" + urllib.parse.urlencode(params, safe=":")
|
|
root = ET.fromstring(http_get(url, delay_s=delay_s, timeout=300))
|
|
for rec in root.findall(".//o:record", _OAI):
|
|
it = _oai_record(rec, lag_days)
|
|
if it and it["announce_type"] == "new" and (
|
|
not categories or set(it["categories"]) & set(categories)):
|
|
items.append(it)
|
|
token = root.findtext(".//o:resumptionToken", namespaces=_OAI)
|
|
if not token or not token.strip():
|
|
return items
|
|
params = {"verb": "ListRecords", "resumptionToken": token.strip()}
|
|
|
|
|
|
def get_arxiv_record(arxiv_id: str) -> dict | None:
|
|
"""One paper's metadata from OAI-PMH GetRecord (used for exemplars)."""
|
|
url = ("https://oaipmh.arxiv.org/oai?verb=GetRecord&metadataPrefix=arXivRaw"
|
|
f"&identifier=oai:arXiv.org:{arxiv_id}")
|
|
rec = ET.fromstring(http_get(url)).find(".//o:record", _OAI)
|
|
return _oai_record(rec) if rec is not None else None
|
|
|
|
|
|
# -- arXiv API (kept for backfills; 406 as of 2026-09-27) --------------------------
|
|
|
|
def fetch_arxiv(category: str, max_results: int = 30) -> list[dict]:
|
|
"""Return recent abstracts from an arXiv category via the Atom API."""
|
|
base_url = "https://export.arxiv.org/api/query"
|
|
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
|
params = {
|
|
"search_query": f"cat:{category}",
|
|
"sortBy": "submittedDate",
|
|
"sortOrder": "descending",
|
|
"max_results": max_results,
|
|
}
|
|
root = ET.fromstring(http_get(f"{base_url}?{urllib.parse.urlencode(params)}"))
|
|
items = []
|
|
for entry in root.findall("atom:entry", ns):
|
|
m = re.match(r"(.+?)(v\d+)?$", entry.find("atom:id", ns).text.strip().split("abs/")[-1])
|
|
items.append(_arxiv_item(
|
|
m.group(1), m.group(2),
|
|
entry.find("atom:title", ns).text,
|
|
[a.find("atom:name", ns).text for a in entry.findall("atom:author", ns)],
|
|
entry.find("atom:published", ns).text.strip()[:10],
|
|
entry.find("atom:summary", ns).text,
|
|
[c.get("term") for c in entry.findall("atom:category", ns)],
|
|
None))
|
|
return items
|
|
|
|
|
|
# -- Journal RSS -------------------------------------------------------------------
|
|
|
|
def fetch_rss(feed_name: str, feed_url: str, source: str | None = None) -> list[dict]:
|
|
"""Return items from an RSS/Atom feed via feedparser."""
|
|
import feedparser
|
|
|
|
feed = feedparser.parse(http_get(feed_url, delay_s=1.0))
|
|
items = []
|
|
for entry in feed.entries:
|
|
doi = getattr(entry, "prism_doi", None) or getattr(entry, "dc_identifier", None)
|
|
link = getattr(entry, "link", "")
|
|
item_id = doi.lower() if doi else link
|
|
if not item_id:
|
|
continue
|
|
|
|
raw_abstract = ""
|
|
if hasattr(entry, "summary"):
|
|
raw_abstract = entry.summary
|
|
elif hasattr(entry, "content") and entry.content:
|
|
raw_abstract = entry.content[0].get("value", "")
|
|
|
|
day = ""
|
|
if hasattr(entry, "published_parsed") and entry.published_parsed:
|
|
day = time.strftime("%Y-%m-%d", entry.published_parsed)
|
|
|
|
authors = []
|
|
if hasattr(entry, "authors"):
|
|
authors = [a.get("name", "") for a in entry.authors if a.get("name")]
|
|
elif hasattr(entry, "author"):
|
|
authors = [entry.author]
|
|
|
|
items.append({
|
|
"id": item_id,
|
|
"doi": doi.lower() if doi else None,
|
|
"arxiv_id": None,
|
|
"version": None,
|
|
"title": _clean(getattr(entry, "title", "")),
|
|
"authors": authors,
|
|
"date": day,
|
|
"abstract": _clean(raw_abstract),
|
|
"source": source or feed_name,
|
|
"link": link,
|
|
"categories": [],
|
|
"announce_type": None,
|
|
"fetched_at": _now(),
|
|
})
|
|
return items
|
|
|
|
|
|
# -- Crossref ----------------------------------------------------------------------
|
|
|
|
def _crossref_item(w: dict, source: str) -> dict:
|
|
parts = (w.get("published-online") or w.get("published") or w.get("issued") or {}).get("date-parts", [[]])[0]
|
|
day = "-".join(f"{p:02d}" for p in parts) if len(parts) == 3 else ""
|
|
authors = [" ".join(filter(None, [a.get("given"), a.get("family")])) or a.get("name", "")
|
|
for a in w.get("author", [])]
|
|
abstract = _clean(w.get("abstract", ""))
|
|
abstract = re.sub(r"^Abstract\s+", "", abstract)
|
|
return {
|
|
"id": w["DOI"].lower(),
|
|
"doi": w["DOI"].lower(),
|
|
"arxiv_id": None,
|
|
"version": None,
|
|
"title": _clean((w.get("title") or [""])[0]),
|
|
"authors": authors,
|
|
"date": day,
|
|
"abstract": abstract,
|
|
"source": source,
|
|
"link": f"https://doi.org/{w['DOI']}",
|
|
"categories": [],
|
|
"announce_type": None,
|
|
"fetched_at": _now(),
|
|
}
|
|
|
|
|
|
def fetch_crossref(issn: str, since: str, source: str) -> list[dict]:
|
|
"""Return journal articles created in Crossref on or after `since` (YYYY-MM-DD).
|
|
|
|
Filters on created date: index date also changes whenever Crossref
|
|
re-indexes an old record (1066 J. Rheol. hits for one September).
|
|
"""
|
|
items, cursor = [], "*"
|
|
while True:
|
|
params = {"filter": f"from-created-date:{since},type:journal-article",
|
|
"rows": 200, "cursor": cursor}
|
|
url = f"https://api.crossref.org/journals/{issn}/works?" + urllib.parse.urlencode(params) + _mailto()
|
|
msg = json.loads(http_get(url, delay_s=1.0))["message"]
|
|
items.extend(_crossref_item(w, source) for w in msg["items"])
|
|
cursor = msg.get("next-cursor")
|
|
if not msg["items"] or not cursor:
|
|
return items
|
|
|
|
|
|
def get_crossref_work(doi: str) -> dict | None:
|
|
"""One DOI's metadata from Crossref (used for exemplars)."""
|
|
try:
|
|
data = http_get(f"https://api.crossref.org/works/{urllib.parse.quote(doi)}?{_mailto().lstrip('&')}",
|
|
delay_s=1.0, tries=2)
|
|
except httpx.HTTPStatusError:
|
|
return None
|
|
return _crossref_item(json.loads(data)["message"], source="Crossref")
|