Initial build out
This commit is contained in:
parent
f037c50736
commit
29215e2bd2
40 changed files with 2622 additions and 0 deletions
0
src/admin_analytics/__init__.py
Normal file
0
src/admin_analytics/__init__.py
Normal file
0
src/admin_analytics/bls/__init__.py
Normal file
0
src/admin_analytics/bls/__init__.py
Normal file
126
src/admin_analytics/cli.py
Normal file
126
src/admin_analytics/cli.py
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
from typing import Annotated
|
||||
|
||||
import typer
|
||||
|
||||
from admin_analytics.config import DEFAULT_YEAR_RANGE
|
||||
from admin_analytics.db.connection import get_connection
|
||||
from admin_analytics.db.schema import ensure_schema
|
||||
|
||||
app = typer.Typer(help="University of Delaware administrative analytics")
|
||||
ingest_app = typer.Typer(help="Ingest data from external sources")
|
||||
app.add_typer(ingest_app, name="ingest")
|
||||
|
||||
|
||||
def _parse_year_range(year_range: str) -> range:
|
||||
"""Parse '2005-2024' into a range object."""
|
||||
parts = year_range.split("-")
|
||||
if len(parts) != 2:
|
||||
raise typer.BadParameter("Year range must be in format YYYY-YYYY")
|
||||
start, end = int(parts[0]), int(parts[1])
|
||||
return range(start, end + 1)
|
||||
|
||||
|
||||
@ingest_app.command()
|
||||
def ipeds(
|
||||
year_range: Annotated[
|
||||
str, typer.Option(help="Year range, e.g. 2005-2024")
|
||||
] = f"{DEFAULT_YEAR_RANGE.start}-{DEFAULT_YEAR_RANGE.stop - 1}",
|
||||
component: Annotated[
|
||||
str, typer.Option(help="Component: all, hd, finance, staff, enrollment")
|
||||
] = "all",
|
||||
force: Annotated[
|
||||
bool, typer.Option("--force", help="Re-download even if files exist")
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Ingest IPEDS data for the University of Delaware."""
|
||||
years = _parse_year_range(year_range)
|
||||
conn = get_connection()
|
||||
ensure_schema(conn)
|
||||
|
||||
from admin_analytics.ipeds.download import download_all
|
||||
from admin_analytics.ipeds.institution import load_institutions
|
||||
from admin_analytics.ipeds.finance import load_finance
|
||||
from admin_analytics.ipeds.staff import load_staff
|
||||
from admin_analytics.ipeds.enrollment import load_enrollment
|
||||
|
||||
components = (
|
||||
["hd", "finance", "staff", "enrollment"]
|
||||
if component == "all"
|
||||
else [component]
|
||||
)
|
||||
|
||||
# Finance needs both F1A and F2 downloads (UD reports under FASB/F2)
|
||||
download_components = list(components)
|
||||
if "finance" in download_components:
|
||||
download_components.append("finance_f2")
|
||||
|
||||
typer.echo(f"Downloading IPEDS data for {years.start}-{years.stop - 1}...")
|
||||
download_all(years, download_components, force=force)
|
||||
|
||||
if "hd" in components:
|
||||
typer.echo("Loading institutional directory (HD)...")
|
||||
load_institutions(conn, years)
|
||||
|
||||
if "finance" in components:
|
||||
typer.echo("Loading finance data (F1A)...")
|
||||
load_finance(conn, years)
|
||||
|
||||
if "staff" in components:
|
||||
typer.echo("Loading staff data (S)...")
|
||||
load_staff(conn, years)
|
||||
|
||||
if "enrollment" in components:
|
||||
typer.echo("Loading enrollment data (EF)...")
|
||||
load_enrollment(conn, years)
|
||||
|
||||
typer.echo("IPEDS ingestion complete.")
|
||||
conn.close()
|
||||
|
||||
|
||||
@ingest_app.command()
|
||||
def irs990(
|
||||
year_range: Annotated[
|
||||
str, typer.Option(help="Year range for index files, e.g. 2019-2024")
|
||||
] = "2019-2024",
|
||||
force: Annotated[
|
||||
bool, typer.Option("--force", help="Re-download even if files exist")
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Ingest IRS 990 data for UD and UD Research Foundation."""
|
||||
years = _parse_year_range(year_range)
|
||||
conn = get_connection()
|
||||
ensure_schema(conn)
|
||||
|
||||
from admin_analytics.irs990.download import download_all_filings
|
||||
from admin_analytics.irs990.loader import load_all
|
||||
|
||||
typer.echo(f"Downloading 990 filings for {years.start}-{years.stop - 1}...")
|
||||
download_all_filings(years, force=force)
|
||||
|
||||
typer.echo("Loading 990 data into database...")
|
||||
totals = load_all(conn, years)
|
||||
typer.echo(
|
||||
f"IRS 990 ingestion complete: {totals['filings']} filings, "
|
||||
f"{totals['part_vii']} Part VII rows, {totals['schedule_j']} Schedule J rows."
|
||||
)
|
||||
conn.close()
|
||||
|
||||
|
||||
@ingest_app.command()
|
||||
def cpi() -> None:
|
||||
"""Ingest BLS CPI-U data. (Not yet implemented.)"""
|
||||
typer.echo("CPI-U ingestion is not yet implemented (Sprint 3).")
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
@ingest_app.command(name="all")
|
||||
def ingest_all(
|
||||
year_range: Annotated[
|
||||
str, typer.Option(help="Year range, e.g. 2005-2024")
|
||||
] = f"{DEFAULT_YEAR_RANGE.start}-{DEFAULT_YEAR_RANGE.stop - 1}",
|
||||
force: Annotated[
|
||||
bool, typer.Option("--force", help="Re-download even if files exist")
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Ingest all data sources."""
|
||||
ipeds(year_range=year_range, component="all", force=force)
|
||||
50
src/admin_analytics/config.py
Normal file
50
src/admin_analytics/config.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
from pathlib import Path
|
||||
|
||||
# University of Delaware IPEDS UNITID
|
||||
UD_UNITID = 130943
|
||||
|
||||
# Paths
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
DATA_DIR = PROJECT_ROOT / "data" / "raw"
|
||||
DB_PATH = PROJECT_ROOT / "data" / "admin_analytics.duckdb"
|
||||
|
||||
# IPEDS
|
||||
IPEDS_BASE_URL = "https://nces.ed.gov/ipeds/datacenter/data"
|
||||
IPEDS_DATA_DIR = DATA_DIR / "ipeds"
|
||||
|
||||
DEFAULT_YEAR_RANGE = range(2005, 2025) # 2005-2024 inclusive
|
||||
|
||||
# IRS 990
|
||||
UD_EINS = [516000297, 516017306] # UD + UD Research Foundation
|
||||
IRS990_BASE_URL = "https://apps.irs.gov/pub/epostcard/990/xml"
|
||||
IRS990_DATA_DIR = DATA_DIR / "irs990"
|
||||
|
||||
|
||||
def ipeds_filename(component: str, year: int) -> str:
|
||||
"""Return the IPEDS filename (without .zip) for a component and year.
|
||||
|
||||
IPEDS uses different naming conventions per survey:
|
||||
HD: HD{YYYY} e.g. HD2023
|
||||
F1A: F{YY-1}{YY}_F1A e.g. F2223_F1A (fiscal year)
|
||||
EF: EF{YYYY}A e.g. EF2023A
|
||||
S: S{YYYY}_OC e.g. S2023_OC (occupational categories)
|
||||
SAL: SAL{YYYY}_IS e.g. SAL2023_IS
|
||||
"""
|
||||
if component == "hd":
|
||||
return f"HD{year}"
|
||||
elif component == "finance":
|
||||
yy_prev = (year - 1) % 100
|
||||
yy = year % 100
|
||||
return f"F{yy_prev:02d}{yy:02d}_F1A"
|
||||
elif component == "finance_f2":
|
||||
yy_prev = (year - 1) % 100
|
||||
yy = year % 100
|
||||
return f"F{yy_prev:02d}{yy:02d}_F2"
|
||||
elif component == "enrollment":
|
||||
return f"EF{year}A"
|
||||
elif component == "staff":
|
||||
return f"S{year}_OC"
|
||||
elif component == "salaries":
|
||||
return f"SAL{year}_IS"
|
||||
else:
|
||||
raise ValueError(f"Unknown IPEDS component: {component}")
|
||||
0
src/admin_analytics/db/__init__.py
Normal file
0
src/admin_analytics/db/__init__.py
Normal file
12
src/admin_analytics/db/connection.py
Normal file
12
src/admin_analytics/db/connection.py
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
|
||||
from admin_analytics.config import DB_PATH
|
||||
|
||||
|
||||
def get_connection(path: Path | None = None) -> duckdb.DuckDBPyConnection:
|
||||
"""Return a DuckDB connection, creating the parent directory if needed."""
|
||||
path = path or DB_PATH
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
return duckdb.connect(str(path))
|
||||
122
src/admin_analytics/db/schema.py
Normal file
122
src/admin_analytics/db/schema.py
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
import duckdb
|
||||
|
||||
TABLES = {
|
||||
"raw_institution": """
|
||||
CREATE TABLE IF NOT EXISTS raw_institution (
|
||||
unitid INTEGER NOT NULL,
|
||||
year INTEGER NOT NULL,
|
||||
ein VARCHAR,
|
||||
institution_name VARCHAR,
|
||||
city VARCHAR,
|
||||
state VARCHAR,
|
||||
sector INTEGER,
|
||||
control INTEGER,
|
||||
carnegie_class INTEGER,
|
||||
enrollment_total INTEGER,
|
||||
PRIMARY KEY (unitid, year)
|
||||
)
|
||||
""",
|
||||
"raw_ipeds_finance": """
|
||||
CREATE TABLE IF NOT EXISTS raw_ipeds_finance (
|
||||
unitid INTEGER NOT NULL,
|
||||
year INTEGER NOT NULL,
|
||||
reporting_standard VARCHAR,
|
||||
total_expenses BIGINT,
|
||||
instruction_expenses BIGINT,
|
||||
research_expenses BIGINT,
|
||||
public_service_expenses BIGINT,
|
||||
academic_support_expenses BIGINT,
|
||||
student_services_expenses BIGINT,
|
||||
institutional_support_expenses BIGINT,
|
||||
auxiliary_expenses BIGINT,
|
||||
hospital_expenses BIGINT,
|
||||
other_expenses BIGINT,
|
||||
salaries_wages BIGINT,
|
||||
benefits BIGINT,
|
||||
PRIMARY KEY (unitid, year)
|
||||
)
|
||||
""",
|
||||
"raw_ipeds_staff": """
|
||||
CREATE TABLE IF NOT EXISTS raw_ipeds_staff (
|
||||
unitid INTEGER NOT NULL,
|
||||
year INTEGER NOT NULL,
|
||||
total_staff INTEGER,
|
||||
faculty_total INTEGER,
|
||||
management_total INTEGER,
|
||||
PRIMARY KEY (unitid, year)
|
||||
)
|
||||
""",
|
||||
"raw_ipeds_enrollment": """
|
||||
CREATE TABLE IF NOT EXISTS raw_ipeds_enrollment (
|
||||
unitid INTEGER NOT NULL,
|
||||
year INTEGER NOT NULL,
|
||||
total_enrollment INTEGER,
|
||||
PRIMARY KEY (unitid, year)
|
||||
)
|
||||
""",
|
||||
"raw_990_filing": """
|
||||
CREATE TABLE IF NOT EXISTS raw_990_filing (
|
||||
object_id VARCHAR PRIMARY KEY,
|
||||
ein VARCHAR,
|
||||
tax_year INTEGER,
|
||||
organization_name VARCHAR,
|
||||
return_type VARCHAR,
|
||||
filing_date DATE,
|
||||
total_revenue BIGINT,
|
||||
total_expenses BIGINT,
|
||||
total_assets BIGINT
|
||||
)
|
||||
""",
|
||||
"raw_990_schedule_j": """
|
||||
CREATE SEQUENCE IF NOT EXISTS seq_990_schedule_j START 1;
|
||||
CREATE TABLE IF NOT EXISTS raw_990_schedule_j (
|
||||
id INTEGER PRIMARY KEY DEFAULT nextval('seq_990_schedule_j'),
|
||||
object_id VARCHAR,
|
||||
ein VARCHAR,
|
||||
tax_year INTEGER,
|
||||
person_name VARCHAR,
|
||||
title VARCHAR,
|
||||
base_compensation BIGINT,
|
||||
bonus_compensation BIGINT,
|
||||
other_compensation BIGINT,
|
||||
deferred_compensation BIGINT,
|
||||
nontaxable_benefits BIGINT,
|
||||
total_compensation BIGINT,
|
||||
compensation_from_related BIGINT
|
||||
)
|
||||
""",
|
||||
"raw_990_part_vii": """
|
||||
CREATE SEQUENCE IF NOT EXISTS seq_990_part_vii START 1;
|
||||
CREATE TABLE IF NOT EXISTS raw_990_part_vii (
|
||||
id INTEGER PRIMARY KEY DEFAULT nextval('seq_990_part_vii'),
|
||||
object_id VARCHAR,
|
||||
ein VARCHAR,
|
||||
tax_year INTEGER,
|
||||
person_name VARCHAR,
|
||||
title VARCHAR,
|
||||
avg_hours_per_week DOUBLE,
|
||||
reportable_comp_from_org BIGINT,
|
||||
reportable_comp_from_related BIGINT,
|
||||
other_compensation BIGINT
|
||||
)
|
||||
""",
|
||||
"raw_cpi_u": """
|
||||
CREATE TABLE IF NOT EXISTS raw_cpi_u (
|
||||
year INTEGER NOT NULL,
|
||||
month INTEGER NOT NULL,
|
||||
value DOUBLE,
|
||||
series_id VARCHAR,
|
||||
PRIMARY KEY (year, month)
|
||||
)
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
def ensure_schema(conn: duckdb.DuckDBPyConnection) -> None:
|
||||
"""Create all raw tables if they don't exist."""
|
||||
for ddl in TABLES.values():
|
||||
# Some DDL blocks contain multiple statements (e.g. CREATE SEQUENCE + CREATE TABLE)
|
||||
for stmt in ddl.split(";"):
|
||||
stmt = stmt.strip()
|
||||
if stmt:
|
||||
conn.execute(stmt)
|
||||
0
src/admin_analytics/ipeds/__init__.py
Normal file
0
src/admin_analytics/ipeds/__init__.py
Normal file
108
src/admin_analytics/ipeds/dictionary.py
Normal file
108
src/admin_analytics/ipeds/dictionary.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
from pathlib import Path
|
||||
|
||||
import openpyxl
|
||||
import xlrd
|
||||
|
||||
|
||||
def parse_dictionary(dict_path: Path) -> dict[str, str]:
|
||||
"""Parse an IPEDS dictionary Excel file, returning {varname: vartitle}.
|
||||
|
||||
Handles both .xls (xlrd) and .xlsx (openpyxl) formats.
|
||||
Looks for a sheet named 'varlist' (case-insensitive).
|
||||
"""
|
||||
suffix = dict_path.suffix.lower()
|
||||
if suffix == ".xls":
|
||||
return _parse_xls(dict_path)
|
||||
elif suffix == ".xlsx":
|
||||
return _parse_xlsx(dict_path)
|
||||
else:
|
||||
raise ValueError(f"Unsupported dictionary format: {suffix}")
|
||||
|
||||
|
||||
def _parse_xls(path: Path) -> dict[str, str]:
|
||||
wb = xlrd.open_workbook(str(path))
|
||||
sheet = _find_varlist_sheet_xls(wb)
|
||||
if sheet is None:
|
||||
return {}
|
||||
# Detect column positions from header row
|
||||
header = [str(v).strip().lower() for v in sheet.row_values(0)]
|
||||
varname_idx = header.index("varname") if "varname" in header else 1
|
||||
vartitle_idx = header.index("vartitle") if "vartitle" in header else len(header) - 1
|
||||
mapping = {}
|
||||
for row_idx in range(1, sheet.nrows):
|
||||
row = sheet.row_values(row_idx)
|
||||
if len(row) > max(varname_idx, vartitle_idx):
|
||||
varname = str(row[varname_idx]).strip().upper()
|
||||
vartitle = str(row[vartitle_idx]).strip()
|
||||
if varname:
|
||||
mapping[varname] = vartitle
|
||||
return mapping
|
||||
|
||||
|
||||
def _parse_xlsx(path: Path) -> dict[str, str]:
|
||||
wb = openpyxl.load_workbook(str(path), read_only=True, data_only=True)
|
||||
sheet = _find_varlist_sheet_xlsx(wb)
|
||||
if sheet is None:
|
||||
wb.close()
|
||||
return {}
|
||||
mapping = {}
|
||||
varname_idx = 1 # default
|
||||
vartitle_idx = -1 # default: last column
|
||||
first = True
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
if first:
|
||||
first = False
|
||||
header = [str(v).strip().lower() if v else "" for v in row]
|
||||
if "varname" in header:
|
||||
varname_idx = header.index("varname")
|
||||
if "vartitle" in header:
|
||||
vartitle_idx = header.index("vartitle")
|
||||
continue
|
||||
if row and len(row) > varname_idx:
|
||||
val = row[varname_idx]
|
||||
if val:
|
||||
varname = str(val).strip().upper()
|
||||
vartitle = str(row[vartitle_idx]).strip() if row[vartitle_idx] else ""
|
||||
if varname:
|
||||
mapping[varname] = vartitle
|
||||
wb.close()
|
||||
return mapping
|
||||
|
||||
|
||||
def _find_varlist_sheet_xls(wb: xlrd.Book):
|
||||
for name in wb.sheet_names():
|
||||
if name.lower() == "varlist":
|
||||
return wb.sheet_by_name(name)
|
||||
return None
|
||||
|
||||
|
||||
def _find_varlist_sheet_xlsx(wb: openpyxl.Workbook):
|
||||
for name in wb.sheetnames:
|
||||
if name.lower() == "varlist":
|
||||
return wb[name]
|
||||
return None
|
||||
|
||||
|
||||
def find_dict_file(dict_dir: Path) -> Path | None:
|
||||
"""Find the dictionary Excel file in an extracted dictionary directory."""
|
||||
for ext in (".xlsx", ".xls"):
|
||||
files = list(dict_dir.glob(f"*{ext}"))
|
||||
if files:
|
||||
return files[0]
|
||||
return None
|
||||
|
||||
|
||||
def get_variable_mapping(component: str, year: int) -> dict[str, str]:
|
||||
"""Load the variable mapping for a component/year pair.
|
||||
|
||||
Returns an empty dict if the dictionary file is not available.
|
||||
"""
|
||||
from admin_analytics.config import IPEDS_DATA_DIR
|
||||
|
||||
dict_dir = IPEDS_DATA_DIR / component / str(year) / "dict"
|
||||
if not dict_dir.exists():
|
||||
return {}
|
||||
dict_file = find_dict_file(dict_dir)
|
||||
if dict_file is None:
|
||||
return {}
|
||||
return parse_dictionary(dict_file)
|
||||
75
src/admin_analytics/ipeds/download.py
Normal file
75
src/admin_analytics/ipeds/download.py
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import io
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from admin_analytics import config
|
||||
|
||||
|
||||
def _component_url(component: str, year: int) -> str:
|
||||
"""Build the URL for an IPEDS data ZIP file."""
|
||||
filename = config.ipeds_filename(component, year)
|
||||
return f"{config.IPEDS_BASE_URL}/{filename}.zip"
|
||||
|
||||
|
||||
def _dict_url(component: str, year: int) -> str:
|
||||
"""Build the URL for an IPEDS dictionary ZIP file."""
|
||||
filename = config.ipeds_filename(component, year)
|
||||
return f"{config.IPEDS_BASE_URL}/{filename}_Dict.zip"
|
||||
|
||||
|
||||
def _dest_dir(component: str, year: int) -> Path:
|
||||
"""Return the local directory for a component/year pair."""
|
||||
return config.IPEDS_DATA_DIR / component / str(year)
|
||||
|
||||
|
||||
def _download_and_extract(url: str, dest: Path) -> Path:
|
||||
"""Download a ZIP from url, extract to dest, return dest."""
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
with httpx.Client(follow_redirects=True, timeout=120.0) as client:
|
||||
resp = client.get(url)
|
||||
resp.raise_for_status()
|
||||
with zipfile.ZipFile(io.BytesIO(resp.content)) as zf:
|
||||
zf.extractall(dest)
|
||||
return dest
|
||||
|
||||
|
||||
def download_component(
|
||||
component: str, year: int, force: bool = False
|
||||
) -> Path:
|
||||
"""Download an IPEDS data file. Returns the extraction directory."""
|
||||
dest = _dest_dir(component, year)
|
||||
if not force and dest.exists() and any(dest.iterdir()):
|
||||
return dest
|
||||
url = _component_url(component, year)
|
||||
return _download_and_extract(url, dest)
|
||||
|
||||
|
||||
def download_dictionary(
|
||||
component: str, year: int, force: bool = False
|
||||
) -> Path:
|
||||
"""Download an IPEDS dictionary file. Returns the extraction directory."""
|
||||
dest = _dest_dir(component, year) / "dict"
|
||||
if not force and dest.exists() and any(dest.iterdir()):
|
||||
return dest
|
||||
url = _dict_url(component, year)
|
||||
return _download_and_extract(url, dest)
|
||||
|
||||
|
||||
def download_all(
|
||||
year_range: range,
|
||||
components: list[str],
|
||||
force: bool = False,
|
||||
) -> None:
|
||||
"""Download all requested components for all years."""
|
||||
for component in components:
|
||||
for year in year_range:
|
||||
try:
|
||||
download_component(component, year, force=force)
|
||||
except httpx.HTTPStatusError as e:
|
||||
print(f" Warning: {component}/{year} data not available ({e.response.status_code})")
|
||||
try:
|
||||
download_dictionary(component, year, force=force)
|
||||
except httpx.HTTPStatusError:
|
||||
pass # Dictionaries aren't always available
|
||||
87
src/admin_analytics/ipeds/enrollment.py
Normal file
87
src/admin_analytics/ipeds/enrollment.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import polars as pl
|
||||
|
||||
from admin_analytics import config
|
||||
from admin_analytics.config import UD_UNITID
|
||||
|
||||
CANONICAL_COLUMNS = ["unitid", "year", "total_enrollment"]
|
||||
|
||||
|
||||
def _find_csv(component_dir: Path) -> Path | None:
|
||||
csvs = [f for f in component_dir.glob("*.csv") if "dict" not in f.stem.lower()]
|
||||
return csvs[0] if csvs else None
|
||||
|
||||
|
||||
def load_enrollment(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
year_range: range,
|
||||
unitid_filter: int | None = UD_UNITID,
|
||||
) -> int:
|
||||
"""Load IPEDS enrollment data into raw_ipeds_enrollment.
|
||||
|
||||
EF files have multiple rows per institution (breakdowns by level, race, etc.).
|
||||
EFALEVEL=1 is the grand total row. If EFALEVEL is not present, we aggregate
|
||||
by taking the max EFTOTLT/EFYTOTLT per UNITID.
|
||||
"""
|
||||
total = 0
|
||||
for year in year_range:
|
||||
data_dir = config.IPEDS_DATA_DIR / "enrollment" / str(year)
|
||||
csv_path = _find_csv(data_dir)
|
||||
if csv_path is None:
|
||||
print(f" No EF CSV found for {year}, skipping")
|
||||
continue
|
||||
|
||||
df = pl.read_csv(
|
||||
csv_path,
|
||||
infer_schema_length=0,
|
||||
encoding="utf8-lossy",
|
||||
)
|
||||
|
||||
upper_cols = {c.upper(): c for c in df.columns}
|
||||
|
||||
# Find the enrollment total column
|
||||
enroll_col = None
|
||||
for candidate in ["EFTOTLT", "EFYTOTLT"]:
|
||||
if candidate in upper_cols:
|
||||
enroll_col = upper_cols[candidate]
|
||||
break
|
||||
if enroll_col is None:
|
||||
print(f" No enrollment total column found for {year}, skipping")
|
||||
continue
|
||||
|
||||
unitid_col = upper_cols.get("UNITID", "UNITID")
|
||||
|
||||
# Filter to grand total row (EFALEVEL=1) if that column exists
|
||||
efalevel_col = upper_cols.get("EFALEVEL")
|
||||
if efalevel_col:
|
||||
df = df.filter(pl.col(efalevel_col).str.strip_chars() == "1")
|
||||
|
||||
# Select, rename, cast
|
||||
df = df.select(
|
||||
pl.col(unitid_col).cast(pl.Int64).alias("unitid"),
|
||||
pl.lit(year).cast(pl.Int64).alias("year"),
|
||||
pl.col(enroll_col).cast(pl.Int64, strict=False).alias("total_enrollment"),
|
||||
)
|
||||
|
||||
# If still multiple rows per UNITID (shouldn't be after EFALEVEL filter), take max
|
||||
df = df.group_by("unitid").agg(
|
||||
pl.col("year").first(),
|
||||
pl.col("total_enrollment").max(),
|
||||
)
|
||||
|
||||
if unitid_filter is not None:
|
||||
df = df.filter(pl.col("unitid") == unitid_filter)
|
||||
|
||||
if df.height == 0:
|
||||
continue
|
||||
|
||||
df = df.select(CANONICAL_COLUMNS)
|
||||
conn.execute("DELETE FROM raw_ipeds_enrollment WHERE year = ?", [year])
|
||||
conn.register("_tmp_enrollment", df.to_arrow())
|
||||
conn.execute("INSERT INTO raw_ipeds_enrollment SELECT * FROM _tmp_enrollment")
|
||||
conn.unregister("_tmp_enrollment")
|
||||
total += df.height
|
||||
|
||||
return total
|
||||
142
src/admin_analytics/ipeds/finance.py
Normal file
142
src/admin_analytics/ipeds/finance.py
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import polars as pl
|
||||
|
||||
from admin_analytics import config
|
||||
from admin_analytics.config import UD_UNITID
|
||||
|
||||
# F1A (GASB/public) column mappings — first match wins per canonical column.
|
||||
F1A_COLUMN_VARIANTS = {
|
||||
"unitid": ["UNITID"],
|
||||
"total_expenses": ["F1C191", "F1D02"],
|
||||
"instruction_expenses": ["F1C011"],
|
||||
"research_expenses": ["F1C021"],
|
||||
"public_service_expenses": ["F1C031"],
|
||||
"academic_support_expenses": ["F1C051"],
|
||||
"student_services_expenses": ["F1C061"],
|
||||
"institutional_support_expenses": ["F1C071"],
|
||||
"auxiliary_expenses": ["F1C111"],
|
||||
"hospital_expenses": ["F1C121"],
|
||||
"other_expenses": ["F1C141"],
|
||||
"salaries_wages": ["F1C192"],
|
||||
"benefits": ["F1C193"],
|
||||
}
|
||||
|
||||
# F2 (FASB/private-style) column mappings — UD reports here despite being public.
|
||||
F2_COLUMN_VARIANTS = {
|
||||
"unitid": ["UNITID"],
|
||||
"total_expenses": ["F2E131"],
|
||||
"instruction_expenses": ["F2E011"],
|
||||
"research_expenses": ["F2E021"],
|
||||
"public_service_expenses": ["F2E031"],
|
||||
"academic_support_expenses": ["F2E041"],
|
||||
"student_services_expenses": ["F2E051"],
|
||||
"institutional_support_expenses": ["F2E061"],
|
||||
"auxiliary_expenses": ["F2E071"],
|
||||
"hospital_expenses": ["F2E081"],
|
||||
"other_expenses": ["F2E121"],
|
||||
"salaries_wages": ["F2E132"],
|
||||
"benefits": ["F2E133"],
|
||||
}
|
||||
|
||||
CANONICAL_COLUMNS = [
|
||||
"unitid", "year", "reporting_standard", "total_expenses",
|
||||
"instruction_expenses", "research_expenses", "public_service_expenses",
|
||||
"academic_support_expenses", "student_services_expenses",
|
||||
"institutional_support_expenses", "auxiliary_expenses", "hospital_expenses",
|
||||
"other_expenses", "salaries_wages", "benefits",
|
||||
]
|
||||
|
||||
|
||||
def _find_csv(component_dir: Path) -> Path | None:
|
||||
csvs = [f for f in component_dir.glob("*.csv") if "dict" not in f.stem.lower()]
|
||||
return csvs[0] if csvs else None
|
||||
|
||||
|
||||
def _resolve_columns(df: pl.DataFrame, variants: dict) -> dict[str, str]:
|
||||
"""For each canonical name, find the first matching column."""
|
||||
upper_cols = {c.upper(): c for c in df.columns}
|
||||
resolved = {}
|
||||
for canonical, candidates in variants.items():
|
||||
for var in candidates:
|
||||
if var in upper_cols:
|
||||
resolved[canonical] = upper_cols[var]
|
||||
break
|
||||
return resolved
|
||||
|
||||
|
||||
def _load_file(
|
||||
csv_path: Path,
|
||||
year: int,
|
||||
variants: dict,
|
||||
reporting_standard: str,
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
unitid_filter: int | None,
|
||||
) -> int:
|
||||
"""Load a single finance CSV into raw_ipeds_finance."""
|
||||
df = pl.read_csv(csv_path, infer_schema_length=0, encoding="utf8-lossy")
|
||||
col_map = _resolve_columns(df, variants)
|
||||
|
||||
if "unitid" not in col_map:
|
||||
return 0
|
||||
|
||||
# Build dataframe with canonical columns
|
||||
result = pl.DataFrame({
|
||||
canonical: df[actual] for canonical, actual in col_map.items()
|
||||
})
|
||||
result = result.with_columns(
|
||||
pl.lit(year).alias("year"),
|
||||
pl.lit(reporting_standard).alias("reporting_standard"),
|
||||
)
|
||||
|
||||
# Cast numeric columns
|
||||
for col in CANONICAL_COLUMNS:
|
||||
if col not in result.columns:
|
||||
result = result.with_columns(pl.lit(None).alias(col))
|
||||
elif col not in ("reporting_standard",):
|
||||
result = result.with_columns(pl.col(col).cast(pl.Int64, strict=False))
|
||||
|
||||
if unitid_filter is not None:
|
||||
result = result.filter(pl.col("unitid") == unitid_filter)
|
||||
|
||||
if result.height == 0:
|
||||
return 0
|
||||
|
||||
result = result.select(CANONICAL_COLUMNS)
|
||||
conn.register("_tmp_finance", result.to_arrow())
|
||||
conn.execute("INSERT INTO raw_ipeds_finance SELECT * FROM _tmp_finance")
|
||||
conn.unregister("_tmp_finance")
|
||||
return result.height
|
||||
|
||||
|
||||
def load_finance(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
year_range: range,
|
||||
unitid_filter: int | None = UD_UNITID,
|
||||
) -> int:
|
||||
"""Load IPEDS finance data into raw_ipeds_finance.
|
||||
|
||||
Tries both F1A (GASB) and F2 (FASB) files since some public institutions
|
||||
like UD report under FASB.
|
||||
"""
|
||||
total = 0
|
||||
for year in year_range:
|
||||
conn.execute("DELETE FROM raw_ipeds_finance WHERE year = ?", [year])
|
||||
|
||||
# Try F1A (GASB)
|
||||
f1a_dir = config.IPEDS_DATA_DIR / "finance" / str(year)
|
||||
csv_path = _find_csv(f1a_dir)
|
||||
if csv_path:
|
||||
total += _load_file(csv_path, year, F1A_COLUMN_VARIANTS, "GASB", conn, unitid_filter)
|
||||
|
||||
# Try F2 (FASB) — needed for institutions like UD
|
||||
f2_dir = config.IPEDS_DATA_DIR / "finance_f2" / str(year)
|
||||
csv_path = _find_csv(f2_dir)
|
||||
if csv_path:
|
||||
total += _load_file(csv_path, year, F2_COLUMN_VARIANTS, "FASB", conn, unitid_filter)
|
||||
|
||||
if total == 0 and not _find_csv(f1a_dir) and not _find_csv(f2_dir):
|
||||
print(f" No finance CSV found for {year}, skipping")
|
||||
|
||||
return total
|
||||
107
src/admin_analytics/ipeds/institution.py
Normal file
107
src/admin_analytics/ipeds/institution.py
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import polars as pl
|
||||
|
||||
from admin_analytics import config
|
||||
from admin_analytics.config import UD_UNITID
|
||||
|
||||
# HD column mappings: canonical name → list of IPEDS varnames (priority order).
|
||||
# The first match found in the file's columns wins.
|
||||
HD_COLUMN_VARIANTS = {
|
||||
"unitid": ["UNITID"],
|
||||
"institution_name": ["INSTNM"],
|
||||
"city": ["CITY"],
|
||||
"state": ["STABBR"],
|
||||
"sector": ["SECTOR"],
|
||||
"control": ["CONTROL"],
|
||||
"ein": ["EIN"],
|
||||
"carnegie_class": ["C21BASIC", "C18BASIC", "C15BASIC", "CCBASIC", "CARNEGIE"],
|
||||
"enrollment_total": ["EFYTOTLT"],
|
||||
}
|
||||
|
||||
CANONICAL_COLUMNS = [
|
||||
"unitid", "year", "ein", "institution_name", "city", "state",
|
||||
"sector", "control", "carnegie_class", "enrollment_total",
|
||||
]
|
||||
|
||||
|
||||
def _find_csv(component_dir: Path) -> Path | None:
|
||||
"""Find the main data CSV in an extracted IPEDS directory."""
|
||||
csvs = [f for f in component_dir.glob("*.csv") if "dict" not in f.stem.lower()]
|
||||
return csvs[0] if csvs else None
|
||||
|
||||
|
||||
def _resolve_columns(df: pl.DataFrame) -> dict[str, str]:
|
||||
"""For each canonical name, find the first matching IPEDS column."""
|
||||
upper_cols = {c.upper(): c for c in df.columns}
|
||||
resolved = {}
|
||||
for canonical, variants in HD_COLUMN_VARIANTS.items():
|
||||
for var in variants:
|
||||
if var in upper_cols:
|
||||
resolved[canonical] = upper_cols[var]
|
||||
break
|
||||
return resolved
|
||||
|
||||
|
||||
def _map_columns(df: pl.DataFrame, year: int) -> pl.DataFrame:
|
||||
"""Rename IPEDS columns to canonical names and add year."""
|
||||
col_map = _resolve_columns(df)
|
||||
# Select and rename only the resolved columns
|
||||
select_exprs = [pl.col(actual).alias(canonical) for canonical, actual in col_map.items()]
|
||||
df = pl.DataFrame({canonical: df[actual] for canonical, actual in col_map.items()})
|
||||
df = df.with_columns(pl.lit(year).alias("year"))
|
||||
|
||||
# Ensure all canonical columns exist
|
||||
for col in CANONICAL_COLUMNS:
|
||||
if col not in df.columns:
|
||||
df = df.with_columns(pl.lit(None).alias(col))
|
||||
|
||||
return df.select(CANONICAL_COLUMNS)
|
||||
|
||||
|
||||
def load_institutions(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
year_range: range,
|
||||
unitid_filter: int | None = UD_UNITID,
|
||||
) -> int:
|
||||
"""Load IPEDS HD data into raw_institution. Returns rows loaded."""
|
||||
total = 0
|
||||
for year in year_range:
|
||||
data_dir = config.IPEDS_DATA_DIR / "hd" / str(year)
|
||||
csv_path = _find_csv(data_dir)
|
||||
if csv_path is None:
|
||||
print(f" No HD CSV found for {year}, skipping")
|
||||
continue
|
||||
|
||||
df = pl.read_csv(
|
||||
csv_path,
|
||||
infer_schema_length=0, # read everything as string first
|
||||
encoding="utf8-lossy",
|
||||
)
|
||||
|
||||
df = _map_columns(df, year)
|
||||
|
||||
# Cast types
|
||||
df = df.with_columns(
|
||||
pl.col("unitid").cast(pl.Int64),
|
||||
pl.col("year").cast(pl.Int64),
|
||||
pl.col("sector").cast(pl.Int64, strict=False),
|
||||
pl.col("control").cast(pl.Int64, strict=False),
|
||||
pl.col("carnegie_class").cast(pl.Int64, strict=False),
|
||||
pl.col("enrollment_total").cast(pl.Int64, strict=False),
|
||||
)
|
||||
|
||||
if unitid_filter is not None:
|
||||
df = df.filter(pl.col("unitid") == unitid_filter)
|
||||
|
||||
if df.height == 0:
|
||||
continue
|
||||
|
||||
conn.execute("DELETE FROM raw_institution WHERE year = ?", [year])
|
||||
conn.register("_tmp_institution", df.to_arrow())
|
||||
conn.execute("INSERT INTO raw_institution SELECT * FROM _tmp_institution")
|
||||
conn.unregister("_tmp_institution")
|
||||
total += df.height
|
||||
|
||||
return total
|
||||
98
src/admin_analytics/ipeds/staff.py
Normal file
98
src/admin_analytics/ipeds/staff.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import polars as pl
|
||||
|
||||
from admin_analytics import config
|
||||
from admin_analytics.config import UD_UNITID
|
||||
|
||||
CANONICAL_COLUMNS = ["unitid", "year", "total_staff", "faculty_total", "management_total"]
|
||||
|
||||
|
||||
def _find_csv(component_dir: Path) -> Path | None:
|
||||
csvs = [f for f in component_dir.glob("*.csv") if "dict" not in f.stem.lower()]
|
||||
return csvs[0] if csvs else None
|
||||
|
||||
|
||||
def load_staff(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
year_range: range,
|
||||
unitid_filter: int | None = UD_UNITID,
|
||||
) -> int:
|
||||
"""Load IPEDS Fall Staff data into raw_ipeds_staff.
|
||||
|
||||
The S_OC file has multiple rows per institution (by occupational category).
|
||||
We extract:
|
||||
- total_staff: FTPT=2, OCCUPCAT=100 (full-time total, all occupations)
|
||||
- management_total: FTPT=2, OCCUPCAT=200 (full-time management)
|
||||
- faculty_total: FTPT=2, OCCUPCAT=250 (full-time instruction/research)
|
||||
"""
|
||||
total = 0
|
||||
for year in year_range:
|
||||
data_dir = config.IPEDS_DATA_DIR / "staff" / str(year)
|
||||
csv_path = _find_csv(data_dir)
|
||||
if csv_path is None:
|
||||
print(f" No Staff CSV found for {year}, skipping")
|
||||
continue
|
||||
|
||||
df = pl.read_csv(
|
||||
csv_path,
|
||||
infer_schema_length=0,
|
||||
encoding="utf8-lossy",
|
||||
)
|
||||
|
||||
upper_cols = {c.upper(): c for c in df.columns}
|
||||
unitid_col = upper_cols.get("UNITID", "UNITID")
|
||||
hrtotlt_col = upper_cols.get("HRTOTLT")
|
||||
occupcat_col = upper_cols.get("OCCUPCAT")
|
||||
ftpt_col = upper_cols.get("FTPT")
|
||||
stafftot_col = upper_cols.get("STAFFTOT") # older format: single row
|
||||
|
||||
if hrtotlt_col and occupcat_col and ftpt_col:
|
||||
# Modern multi-row format: filter by FTPT and OCCUPCAT
|
||||
df = df.with_columns(
|
||||
pl.col(ftpt_col).str.strip_chars().alias("_ftpt"),
|
||||
pl.col(occupcat_col).str.strip_chars().alias("_occupcat"),
|
||||
pl.col(unitid_col).cast(pl.Int64).alias("unitid"),
|
||||
pl.col(hrtotlt_col).cast(pl.Int64, strict=False).alias("_count"),
|
||||
)
|
||||
|
||||
# Full-time totals only (FTPT=2)
|
||||
ft = df.filter(pl.col("_ftpt") == "2")
|
||||
|
||||
# Pivot occupational categories into columns per UNITID
|
||||
total_staff = ft.filter(pl.col("_occupcat") == "100").select("unitid", pl.col("_count").alias("total_staff"))
|
||||
management = ft.filter(pl.col("_occupcat") == "200").select("unitid", pl.col("_count").alias("management_total"))
|
||||
faculty = ft.filter(pl.col("_occupcat") == "250").select("unitid", pl.col("_count").alias("faculty_total"))
|
||||
|
||||
result = total_staff.join(management, on="unitid", how="left").join(faculty, on="unitid", how="left")
|
||||
elif stafftot_col:
|
||||
# Older single-row format
|
||||
result = df.select(
|
||||
pl.col(unitid_col).cast(pl.Int64).alias("unitid"),
|
||||
pl.col(stafftot_col).cast(pl.Int64, strict=False).alias("total_staff"),
|
||||
)
|
||||
result = result.with_columns(
|
||||
pl.lit(None).cast(pl.Int64).alias("faculty_total"),
|
||||
pl.lit(None).cast(pl.Int64).alias("management_total"),
|
||||
)
|
||||
else:
|
||||
print(f" Unrecognized staff file format for {year}, skipping")
|
||||
continue
|
||||
|
||||
result = result.with_columns(pl.lit(year).cast(pl.Int64).alias("year"))
|
||||
|
||||
if unitid_filter is not None:
|
||||
result = result.filter(pl.col("unitid") == unitid_filter)
|
||||
|
||||
if result.height == 0:
|
||||
continue
|
||||
|
||||
result = result.select(CANONICAL_COLUMNS)
|
||||
conn.execute("DELETE FROM raw_ipeds_staff WHERE year = ?", [year])
|
||||
conn.register("_tmp_staff", result.to_arrow())
|
||||
conn.execute("INSERT INTO raw_ipeds_staff SELECT * FROM _tmp_staff")
|
||||
conn.unregister("_tmp_staff")
|
||||
total += result.height
|
||||
|
||||
return total
|
||||
0
src/admin_analytics/irs990/__init__.py
Normal file
0
src/admin_analytics/irs990/__init__.py
Normal file
194
src/admin_analytics/irs990/download.py
Normal file
194
src/admin_analytics/irs990/download.py
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
"""IRS 990 e-file download: index filtering and XML extraction from bulk ZIPs."""
|
||||
|
||||
import io
|
||||
import re
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import polars as pl
|
||||
|
||||
from admin_analytics import config
|
||||
|
||||
|
||||
def download_index(year: int, force: bool = False) -> Path:
|
||||
"""Download the IRS 990 e-file index CSV for a given year."""
|
||||
dest_dir = config.IRS990_DATA_DIR / str(year)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest = dest_dir / f"index_{year}.csv"
|
||||
if not force and dest.exists():
|
||||
return dest
|
||||
url = f"{config.IRS990_BASE_URL}/{year}/index_{year}.csv"
|
||||
with httpx.Client(follow_redirects=True, timeout=120.0) as client:
|
||||
resp = client.get(url)
|
||||
resp.raise_for_status()
|
||||
dest.write_bytes(resp.content)
|
||||
return dest
|
||||
|
||||
|
||||
def filter_index(
|
||||
index_path: Path,
|
||||
eins: list[int],
|
||||
return_types: list[str] | None = None,
|
||||
) -> pl.DataFrame:
|
||||
"""Filter the index CSV to specific EINs and return types.
|
||||
|
||||
Args:
|
||||
index_path: Path to the index CSV.
|
||||
eins: EINs to filter for.
|
||||
return_types: e.g. ["990", "990PF"]. Default: ["990"] (skip 990T, 990EZ).
|
||||
"""
|
||||
if return_types is None:
|
||||
return_types = ["990"]
|
||||
|
||||
df = pl.read_csv(index_path, infer_schema_length=0, encoding="utf8-lossy")
|
||||
# Normalize column names to uppercase
|
||||
df = df.rename({c: c.strip().upper() for c in df.columns})
|
||||
|
||||
ein_strs = [str(e) for e in eins]
|
||||
df = df.filter(
|
||||
pl.col("EIN").is_in(ein_strs)
|
||||
& pl.col("RETURN_TYPE").is_in(return_types)
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
def get_batch_list(year: int) -> list[str]:
|
||||
"""Get the list of available TEOS XML batch ZIP IDs for a year.
|
||||
|
||||
Tries common naming patterns: {YEAR}_TEOS_XML_{MM}{L} where
|
||||
MM is 01-12 and L is A-D.
|
||||
"""
|
||||
batches = []
|
||||
with httpx.Client(follow_redirects=True, timeout=30.0) as client:
|
||||
for month in range(1, 13):
|
||||
for letter in "ABCD":
|
||||
batch_id = f"{year}_TEOS_XML_{month:02d}{letter}"
|
||||
url = f"{config.IRS990_BASE_URL}/{year}/{batch_id}.zip"
|
||||
try:
|
||||
resp = client.head(url)
|
||||
if resp.status_code == 200:
|
||||
batches.append(batch_id)
|
||||
except httpx.HTTPError:
|
||||
continue
|
||||
return batches
|
||||
|
||||
|
||||
def _extract_xml_from_zip(
|
||||
zip_url: str, object_id: str, dest: Path
|
||||
) -> bool:
|
||||
"""Download a batch ZIP and extract a specific XML file.
|
||||
|
||||
Streams the ZIP to a temp file, extracts the target, deletes the temp.
|
||||
Returns True if the file was found and extracted.
|
||||
"""
|
||||
target_name = f"{object_id}_public.xml"
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp:
|
||||
tmp_path = Path(tmp.name)
|
||||
|
||||
try:
|
||||
with httpx.Client(follow_redirects=True, timeout=600.0) as client:
|
||||
with client.stream("GET", zip_url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(tmp_path, "wb") as f:
|
||||
for chunk in resp.iter_bytes(chunk_size=1024 * 1024):
|
||||
f.write(chunk)
|
||||
|
||||
with zipfile.ZipFile(tmp_path) as zf:
|
||||
names = zf.namelist()
|
||||
# Look for our target file (case-insensitive)
|
||||
match = None
|
||||
for name in names:
|
||||
if name.lower() == target_name.lower():
|
||||
match = name
|
||||
break
|
||||
if match is None:
|
||||
return False
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(match) as src, open(dest, "wb") as dst:
|
||||
dst.write(src.read())
|
||||
return True
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def download_filing_xml(
|
||||
year: int,
|
||||
object_id: str,
|
||||
batch_id: str | None = None,
|
||||
force: bool = False,
|
||||
) -> Path | None:
|
||||
"""Download a specific 990 XML filing from the IRS bulk data.
|
||||
|
||||
Args:
|
||||
year: The index year (not necessarily the tax year).
|
||||
object_id: The OBJECT_ID from the index CSV.
|
||||
batch_id: The XML_BATCH_ID if known (2024+ indexes have this).
|
||||
force: Re-download even if already exists.
|
||||
|
||||
Returns:
|
||||
Path to the extracted XML file, or None if not found.
|
||||
"""
|
||||
dest = config.IRS990_DATA_DIR / str(year) / f"{object_id}.xml"
|
||||
if not force and dest.exists():
|
||||
return dest
|
||||
|
||||
if batch_id:
|
||||
# We know which ZIP to look in
|
||||
url = f"{config.IRS990_BASE_URL}/{year}/{batch_id}.zip"
|
||||
if _extract_xml_from_zip(url, object_id, dest):
|
||||
return dest
|
||||
return None
|
||||
|
||||
# No batch ID — try each batch until we find our file
|
||||
batches = get_batch_list(year)
|
||||
for bid in batches:
|
||||
url = f"{config.IRS990_BASE_URL}/{year}/{bid}.zip"
|
||||
print(f" Searching batch {bid}...")
|
||||
if _extract_xml_from_zip(url, object_id, dest):
|
||||
print(f" Found in {bid}")
|
||||
return dest
|
||||
|
||||
print(f" Warning: filing {object_id} not found in any batch for {year}")
|
||||
return None
|
||||
|
||||
|
||||
def download_all_filings(
|
||||
year_range: range,
|
||||
eins: list[int] | None = None,
|
||||
force: bool = False,
|
||||
) -> list[Path]:
|
||||
"""Download all 990 XML filings for the target EINs across years.
|
||||
|
||||
Returns list of paths to downloaded XML files.
|
||||
"""
|
||||
eins = eins or config.UD_EINS
|
||||
xml_paths = []
|
||||
|
||||
for year in year_range:
|
||||
print(f" Downloading index for {year}...")
|
||||
try:
|
||||
index_path = download_index(year, force=force)
|
||||
except httpx.HTTPStatusError as e:
|
||||
print(f" Index not available for {year} ({e.response.status_code})")
|
||||
continue
|
||||
|
||||
filings = filter_index(index_path, eins)
|
||||
if filings.height == 0:
|
||||
print(f" No filings found for target EINs in {year}")
|
||||
continue
|
||||
|
||||
print(f" Found {filings.height} filing(s) in {year}")
|
||||
|
||||
for row in filings.iter_rows(named=True):
|
||||
object_id = row["OBJECT_ID"]
|
||||
batch_id = row.get("XML_BATCH_ID")
|
||||
taxpayer = row.get("TAXPAYER_NAME", "")
|
||||
print(f" Downloading {taxpayer} ({object_id})...")
|
||||
path = download_filing_xml(year, object_id, batch_id, force=force)
|
||||
if path:
|
||||
xml_paths.append(path)
|
||||
|
||||
return xml_paths
|
||||
162
src/admin_analytics/irs990/loader.py
Normal file
162
src/admin_analytics/irs990/loader.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
"""Load parsed IRS 990 data into DuckDB."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
|
||||
from admin_analytics import config
|
||||
from admin_analytics.irs990.parser import parse_xml, parse_filing, parse_part_vii, parse_schedule_j
|
||||
|
||||
|
||||
def load_filing(conn: duckdb.DuckDBPyConnection, xml_path: Path, object_id: str) -> bool:
|
||||
"""Parse a 990 XML and insert the filing header into raw_990_filing.
|
||||
|
||||
Returns True if a row was inserted.
|
||||
"""
|
||||
root = parse_xml(xml_path)
|
||||
filing = parse_filing(root)
|
||||
|
||||
if not filing["ein"] or not filing["tax_year"]:
|
||||
return False
|
||||
|
||||
conn.execute(
|
||||
"DELETE FROM raw_990_filing WHERE object_id = ?", [object_id]
|
||||
)
|
||||
conn.execute(
|
||||
"""INSERT INTO raw_990_filing
|
||||
(object_id, ein, tax_year, organization_name, return_type,
|
||||
total_revenue, total_expenses, total_assets)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
[
|
||||
object_id,
|
||||
filing["ein"],
|
||||
filing["tax_year"],
|
||||
filing["organization_name"],
|
||||
filing["return_type"],
|
||||
filing["total_revenue"],
|
||||
filing["total_expenses"],
|
||||
filing["total_assets"],
|
||||
],
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def load_part_vii(conn: duckdb.DuckDBPyConnection, xml_path: Path, object_id: str) -> int:
|
||||
"""Parse Part VII compensation and insert into raw_990_part_vii.
|
||||
|
||||
Returns number of rows inserted.
|
||||
"""
|
||||
root = parse_xml(xml_path)
|
||||
filing = parse_filing(root)
|
||||
people = parse_part_vii(root)
|
||||
|
||||
if not people:
|
||||
return 0
|
||||
|
||||
ein = filing["ein"]
|
||||
tax_year = filing["tax_year"]
|
||||
|
||||
conn.execute(
|
||||
"DELETE FROM raw_990_part_vii WHERE object_id = ?", [object_id]
|
||||
)
|
||||
|
||||
count = 0
|
||||
for person in people:
|
||||
if not person["person_name"]:
|
||||
continue
|
||||
conn.execute(
|
||||
"""INSERT INTO raw_990_part_vii
|
||||
(object_id, ein, tax_year, person_name, title,
|
||||
avg_hours_per_week, reportable_comp_from_org,
|
||||
reportable_comp_from_related, other_compensation)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
[
|
||||
object_id,
|
||||
ein,
|
||||
tax_year,
|
||||
person["person_name"],
|
||||
person["title"],
|
||||
person["avg_hours_per_week"],
|
||||
person["reportable_comp_from_org"],
|
||||
person["reportable_comp_from_related"],
|
||||
person["other_compensation"],
|
||||
],
|
||||
)
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def load_schedule_j(conn: duckdb.DuckDBPyConnection, xml_path: Path, object_id: str) -> int:
|
||||
"""Parse Schedule J compensation and insert into raw_990_schedule_j.
|
||||
|
||||
Returns number of rows inserted.
|
||||
"""
|
||||
root = parse_xml(xml_path)
|
||||
filing = parse_filing(root)
|
||||
people = parse_schedule_j(root)
|
||||
|
||||
if not people:
|
||||
return 0
|
||||
|
||||
ein = filing["ein"]
|
||||
tax_year = filing["tax_year"]
|
||||
|
||||
conn.execute(
|
||||
"DELETE FROM raw_990_schedule_j WHERE object_id = ?", [object_id]
|
||||
)
|
||||
|
||||
count = 0
|
||||
for person in people:
|
||||
if not person["person_name"]:
|
||||
continue
|
||||
conn.execute(
|
||||
"""INSERT INTO raw_990_schedule_j
|
||||
(object_id, ein, tax_year, person_name, title,
|
||||
base_compensation, bonus_compensation, other_compensation,
|
||||
deferred_compensation, nontaxable_benefits,
|
||||
total_compensation, compensation_from_related)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
[
|
||||
object_id,
|
||||
ein,
|
||||
tax_year,
|
||||
person["person_name"],
|
||||
person["title"],
|
||||
person["base_compensation"],
|
||||
person["bonus_compensation"],
|
||||
person["other_compensation"],
|
||||
person["deferred_compensation"],
|
||||
person["nontaxable_benefits"],
|
||||
person["total_compensation"],
|
||||
person["compensation_from_related"],
|
||||
],
|
||||
)
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def load_all(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
year_range: range,
|
||||
) -> dict[str, int]:
|
||||
"""Load all downloaded 990 XML filings into DuckDB.
|
||||
|
||||
Returns counts of rows loaded per table.
|
||||
"""
|
||||
totals = {"filings": 0, "part_vii": 0, "schedule_j": 0}
|
||||
|
||||
for year in year_range:
|
||||
year_dir = config.IRS990_DATA_DIR / str(year)
|
||||
if not year_dir.exists():
|
||||
continue
|
||||
|
||||
for xml_path in sorted(year_dir.glob("*.xml")):
|
||||
object_id = xml_path.stem
|
||||
if load_filing(conn, xml_path, object_id):
|
||||
totals["filings"] += 1
|
||||
totals["part_vii"] += load_part_vii(conn, xml_path, object_id)
|
||||
totals["schedule_j"] += load_schedule_j(conn, xml_path, object_id)
|
||||
|
||||
return totals
|
||||
259
src/admin_analytics/irs990/parser.py
Normal file
259
src/admin_analytics/irs990/parser.py
Normal file
|
|
@ -0,0 +1,259 @@
|
|||
"""Parse IRS 990 XML filings: header, Part VII, and Schedule J."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from lxml import etree
|
||||
|
||||
|
||||
def _strip_ns(tree: etree._Element) -> etree._Element:
|
||||
"""Strip all namespaces from an XML tree for simpler XPath queries."""
|
||||
for el in tree.iter():
|
||||
if isinstance(el.tag, str) and "{" in el.tag:
|
||||
el.tag = el.tag.split("}", 1)[1]
|
||||
for attr_name in list(el.attrib):
|
||||
if "{" in attr_name:
|
||||
new_name = attr_name.split("}", 1)[1]
|
||||
el.attrib[new_name] = el.attrib.pop(attr_name)
|
||||
return tree
|
||||
|
||||
|
||||
def _text(el: etree._Element | None, path: str) -> str | None:
|
||||
"""Get text of a child element by relative XPath, or None."""
|
||||
if el is None:
|
||||
return None
|
||||
child = el.find(path)
|
||||
return child.text.strip() if child is not None and child.text else None
|
||||
|
||||
|
||||
def _int(el: etree._Element | None, path: str) -> int | None:
|
||||
"""Get integer value of a child element."""
|
||||
text = _text(el, path)
|
||||
if text is None:
|
||||
return None
|
||||
try:
|
||||
return int(text)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _float(el: etree._Element | None, path: str) -> float | None:
|
||||
"""Get float value of a child element."""
|
||||
text = _text(el, path)
|
||||
if text is None:
|
||||
return None
|
||||
try:
|
||||
return float(text)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _find_first(root: etree._Element, xpaths: list[str]) -> list:
|
||||
"""Try multiple XPaths, return the first one that produces results."""
|
||||
for xpath in xpaths:
|
||||
results = root.findall(xpath)
|
||||
if results:
|
||||
return results
|
||||
return []
|
||||
|
||||
|
||||
def _find_first_text(el: etree._Element, paths: list[str]) -> str | None:
|
||||
"""Try multiple child element names, return the first one with text."""
|
||||
for path in paths:
|
||||
val = _text(el, path)
|
||||
if val is not None:
|
||||
return val
|
||||
return None
|
||||
|
||||
|
||||
def _find_first_int(el: etree._Element, paths: list[str]) -> int | None:
|
||||
"""Try multiple child element names, return the first integer found."""
|
||||
for path in paths:
|
||||
val = _int(el, path)
|
||||
if val is not None:
|
||||
return val
|
||||
return None
|
||||
|
||||
|
||||
def parse_xml(xml_path: Path) -> etree._Element:
|
||||
"""Parse an XML file and strip namespaces."""
|
||||
tree = etree.parse(str(xml_path))
|
||||
return _strip_ns(tree.getroot())
|
||||
|
||||
|
||||
def parse_filing(root: etree._Element) -> dict:
|
||||
"""Extract filing header fields from a parsed 990 XML."""
|
||||
header = root.find(".//ReturnHeader")
|
||||
|
||||
ein = _find_first_text(header, [
|
||||
"Filer/EIN",
|
||||
"Filer/EIN",
|
||||
]) if header is not None else None
|
||||
|
||||
tax_year = _find_first_text(header, [
|
||||
"TaxYr",
|
||||
"TaxYear",
|
||||
]) if header is not None else None
|
||||
|
||||
# If no TaxYr element, extract from TaxPeriodEndDt (YYYY-MM-DD -> YYYY)
|
||||
if tax_year is None and header is not None:
|
||||
tax_period = _find_first_text(header, [
|
||||
"TaxPeriodEndDt",
|
||||
"TaxPrdEndDt",
|
||||
])
|
||||
if tax_period:
|
||||
tax_year = tax_period[:4]
|
||||
|
||||
org_name = None
|
||||
if header is not None:
|
||||
org_name = _find_first_text(header, [
|
||||
"Filer/BusinessName/BusinessNameLine1Txt",
|
||||
"Filer/BusinessName/BusinessNameLine1",
|
||||
"Filer/Name/BusinessNameLine1",
|
||||
])
|
||||
|
||||
return_type = _find_first_text(header, [
|
||||
"ReturnTypeCd",
|
||||
"ReturnType",
|
||||
]) if header is not None else None
|
||||
|
||||
# Get financial summary from the main form
|
||||
irs990 = root.find(".//IRS990")
|
||||
total_revenue = _find_first_int(irs990, [
|
||||
"CYTotalRevenueAmt",
|
||||
"TotalRevenueCurrentYear",
|
||||
"TotalRevenueAmt",
|
||||
]) if irs990 is not None else None
|
||||
|
||||
total_expenses = _find_first_int(irs990, [
|
||||
"CYTotalExpensesAmt",
|
||||
"TotalExpensesCurrentYear",
|
||||
"TotalFunctionalExpensesAmt",
|
||||
]) if irs990 is not None else None
|
||||
|
||||
total_assets = _find_first_int(root, [
|
||||
".//IRS990/TotalAssetsEOYAmt",
|
||||
".//IRS990/TotalAssetsEndOfYear",
|
||||
".//TotalAssetsEOYAmt",
|
||||
])
|
||||
|
||||
return {
|
||||
"ein": ein,
|
||||
"tax_year": int(tax_year) if tax_year else None,
|
||||
"organization_name": org_name,
|
||||
"return_type": return_type,
|
||||
"total_revenue": total_revenue,
|
||||
"total_expenses": total_expenses,
|
||||
"total_assets": total_assets,
|
||||
}
|
||||
|
||||
|
||||
def parse_part_vii(root: etree._Element) -> list[dict]:
|
||||
"""Extract Part VII (compensation summary) from a parsed 990 XML.
|
||||
|
||||
Returns one dict per listed person with compensation details.
|
||||
"""
|
||||
groups = _find_first(root, [
|
||||
".//IRS990/Form990PartVIISectionAGrp",
|
||||
".//IRS990/Form990PartVIISectionA",
|
||||
])
|
||||
|
||||
results = []
|
||||
for grp in groups:
|
||||
person_name = _find_first_text(grp, [
|
||||
"PersonNm",
|
||||
"NamePerson",
|
||||
"PersonName",
|
||||
"NameBusiness/BusinessNameLine1Txt",
|
||||
])
|
||||
title = _find_first_text(grp, [
|
||||
"TitleTxt",
|
||||
"Title",
|
||||
])
|
||||
hours = _find_first(grp, ["AverageHoursPerWeekRt", "AverageHoursPerWeek"])
|
||||
avg_hours = None
|
||||
if hours:
|
||||
try:
|
||||
avg_hours = float(hours[0].text)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
results.append({
|
||||
"person_name": person_name,
|
||||
"title": title,
|
||||
"avg_hours_per_week": avg_hours,
|
||||
"reportable_comp_from_org": _find_first_int(grp, [
|
||||
"ReportableCompFromOrgAmt",
|
||||
"ReportableCompFromOrganization",
|
||||
]),
|
||||
"reportable_comp_from_related": _find_first_int(grp, [
|
||||
"ReportableCompFromRltdOrgAmt",
|
||||
"ReportableCompFromRelatedOrgs",
|
||||
]),
|
||||
"other_compensation": _find_first_int(grp, [
|
||||
"OtherCompensationAmt",
|
||||
"OtherCompensation",
|
||||
]),
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def parse_schedule_j(root: etree._Element) -> list[dict]:
|
||||
"""Extract Schedule J (detailed compensation) from a parsed 990 XML.
|
||||
|
||||
Returns one dict per person with itemized compensation breakdown.
|
||||
"""
|
||||
groups = _find_first(root, [
|
||||
".//IRS990ScheduleJ/RltdOrgOfficerTrstKeyEmplGrp",
|
||||
".//IRS990ScheduleJ/Form990ScheduleJPartII",
|
||||
".//IRS990ScheduleJ/SupplementalInformationDetail",
|
||||
])
|
||||
|
||||
results = []
|
||||
for grp in groups:
|
||||
person_name = _find_first_text(grp, [
|
||||
"PersonNm",
|
||||
"NamePerson",
|
||||
"PersonName",
|
||||
"NameBusiness/BusinessNameLine1Txt",
|
||||
])
|
||||
title = _find_first_text(grp, [
|
||||
"TitleTxt",
|
||||
"Title",
|
||||
])
|
||||
|
||||
results.append({
|
||||
"person_name": person_name,
|
||||
"title": title,
|
||||
"base_compensation": _find_first_int(grp, [
|
||||
"BaseCompensationFilingOrgAmt",
|
||||
"BaseCompensation",
|
||||
]),
|
||||
"bonus_compensation": _find_first_int(grp, [
|
||||
"BonusFilingOrgAmt",
|
||||
"Bonus",
|
||||
]),
|
||||
"other_compensation": _find_first_int(grp, [
|
||||
"OtherCompensationFilingOrgAmt",
|
||||
"OtherCompensation",
|
||||
]),
|
||||
"deferred_compensation": _find_first_int(grp, [
|
||||
"DeferredCompensationFlngOrgAmt",
|
||||
"DeferredCompensation",
|
||||
]),
|
||||
"nontaxable_benefits": _find_first_int(grp, [
|
||||
"NontaxableBenefitsFilingOrgAmt",
|
||||
"NontaxableBenefits",
|
||||
]),
|
||||
"total_compensation": _find_first_int(grp, [
|
||||
"TotalCompensationFilingOrgAmt",
|
||||
"TotalCompensation",
|
||||
]),
|
||||
"compensation_from_related": _find_first_int(grp, [
|
||||
"CompensationFromOtherSrcsAmt",
|
||||
"CompensationFromOtherSources",
|
||||
"CompReportedInPriorFormFilingOrgAmt",
|
||||
]),
|
||||
})
|
||||
|
||||
return results
|
||||
43
src/admin_analytics/irs990/titles.py
Normal file
43
src/admin_analytics/irs990/titles.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
"""Normalize 990 titles to canonical roles."""
|
||||
|
||||
import re
|
||||
|
||||
# Canonical roles and their matching patterns.
|
||||
# Order matters — first match wins. More specific patterns before general ones.
|
||||
TITLE_PATTERNS: list[tuple[str, re.Pattern]] = [
|
||||
# VP patterns must come before PRESIDENT to avoid false matches
|
||||
("PROVOST", re.compile(r"\bprovost\b", re.I)),
|
||||
("VP_FINANCE", re.compile(r"(?:\bv\.?p\.?\b|\bvice\s+president\b).*\b(?:financ|budget|business|admin)|\b(?:financ|budget|business|admin).*(?:\bv\.?p\.?\b|\bvice\s+president\b)", re.I)),
|
||||
("VP_RESEARCH", re.compile(r"(?:\bv\.?p\.?\b|\bvice\s+president\b).*\bresearch|\bresearch.*(?:\bv\.?p\.?\b|\bvice\s+president\b)", re.I)),
|
||||
("VP_STUDENT_AFFAIRS", re.compile(r"(?:\bv\.?p\.?\b|\bvice\s+president\b).*\bstudent|\bstudent.*(?:\bv\.?p\.?\b|\bvice\s+president\b)", re.I)),
|
||||
("VP_ADVANCEMENT", re.compile(r"(?:\bv\.?p\.?\b|\bvice\s+president\b).*\b(?:advancement|development|giving|fundrais)|\b(?:advancement|development|giving|fundrais).*(?:\bv\.?p\.?\b|\bvice\s+president\b)", re.I)),
|
||||
("VP_OTHER", re.compile(r"\bv\.?p\.?\b|\bvice\s+president\b", re.I)),
|
||||
("CFO", re.compile(r"\b(cfo|chief\s+financial)\b", re.I)),
|
||||
("CIO", re.compile(r"\b(cio|chief\s+information)\b", re.I)),
|
||||
("COO", re.compile(r"\b(coo|chief\s+operating)\b", re.I)),
|
||||
("GENERAL_COUNSEL", re.compile(r"\b(general\s+counsel|chief\s+legal)\b", re.I)),
|
||||
("PRESIDENT", re.compile(r"\bpresident\b", re.I)),
|
||||
("DEAN", re.compile(r"\bdean\b", re.I)),
|
||||
("TREASURER", re.compile(r"\btreasurer\b", re.I)),
|
||||
("SECRETARY", re.compile(r"\bsecretary\b", re.I)),
|
||||
("TRUSTEE", re.compile(r"\btrustee\b", re.I)),
|
||||
("DIRECTOR", re.compile(r"\bdirector\b", re.I)),
|
||||
]
|
||||
|
||||
|
||||
def normalize_title(raw_title: str | None) -> str | None:
|
||||
"""Map a raw 990 title string to a canonical role.
|
||||
|
||||
Returns the canonical role string, or "OTHER" for unmatched titles,
|
||||
or None if the input is None/empty.
|
||||
"""
|
||||
if not raw_title or not raw_title.strip():
|
||||
return None
|
||||
|
||||
title = raw_title.strip()
|
||||
|
||||
for role, pattern in TITLE_PATTERNS:
|
||||
if pattern.search(title):
|
||||
return role
|
||||
|
||||
return "OTHER"
|
||||
0
src/admin_analytics/scraper/__init__.py
Normal file
0
src/admin_analytics/scraper/__init__.py
Normal file
Loading…
Add table
Add a link
Reference in a new issue