Initial build out

This commit is contained in:
emfurst 2026-03-30 07:15:14 -04:00
commit 29215e2bd2
40 changed files with 2622 additions and 0 deletions

View file

View file

126
src/admin_analytics/cli.py Normal file
View file

@ -0,0 +1,126 @@
from typing import Annotated
import typer
from admin_analytics.config import DEFAULT_YEAR_RANGE
from admin_analytics.db.connection import get_connection
from admin_analytics.db.schema import ensure_schema
app = typer.Typer(help="University of Delaware administrative analytics")
ingest_app = typer.Typer(help="Ingest data from external sources")
app.add_typer(ingest_app, name="ingest")
def _parse_year_range(year_range: str) -> range:
"""Parse '2005-2024' into a range object."""
parts = year_range.split("-")
if len(parts) != 2:
raise typer.BadParameter("Year range must be in format YYYY-YYYY")
start, end = int(parts[0]), int(parts[1])
return range(start, end + 1)
@ingest_app.command()
def ipeds(
year_range: Annotated[
str, typer.Option(help="Year range, e.g. 2005-2024")
] = f"{DEFAULT_YEAR_RANGE.start}-{DEFAULT_YEAR_RANGE.stop - 1}",
component: Annotated[
str, typer.Option(help="Component: all, hd, finance, staff, enrollment")
] = "all",
force: Annotated[
bool, typer.Option("--force", help="Re-download even if files exist")
] = False,
) -> None:
"""Ingest IPEDS data for the University of Delaware."""
years = _parse_year_range(year_range)
conn = get_connection()
ensure_schema(conn)
from admin_analytics.ipeds.download import download_all
from admin_analytics.ipeds.institution import load_institutions
from admin_analytics.ipeds.finance import load_finance
from admin_analytics.ipeds.staff import load_staff
from admin_analytics.ipeds.enrollment import load_enrollment
components = (
["hd", "finance", "staff", "enrollment"]
if component == "all"
else [component]
)
# Finance needs both F1A and F2 downloads (UD reports under FASB/F2)
download_components = list(components)
if "finance" in download_components:
download_components.append("finance_f2")
typer.echo(f"Downloading IPEDS data for {years.start}-{years.stop - 1}...")
download_all(years, download_components, force=force)
if "hd" in components:
typer.echo("Loading institutional directory (HD)...")
load_institutions(conn, years)
if "finance" in components:
typer.echo("Loading finance data (F1A)...")
load_finance(conn, years)
if "staff" in components:
typer.echo("Loading staff data (S)...")
load_staff(conn, years)
if "enrollment" in components:
typer.echo("Loading enrollment data (EF)...")
load_enrollment(conn, years)
typer.echo("IPEDS ingestion complete.")
conn.close()
@ingest_app.command()
def irs990(
year_range: Annotated[
str, typer.Option(help="Year range for index files, e.g. 2019-2024")
] = "2019-2024",
force: Annotated[
bool, typer.Option("--force", help="Re-download even if files exist")
] = False,
) -> None:
"""Ingest IRS 990 data for UD and UD Research Foundation."""
years = _parse_year_range(year_range)
conn = get_connection()
ensure_schema(conn)
from admin_analytics.irs990.download import download_all_filings
from admin_analytics.irs990.loader import load_all
typer.echo(f"Downloading 990 filings for {years.start}-{years.stop - 1}...")
download_all_filings(years, force=force)
typer.echo("Loading 990 data into database...")
totals = load_all(conn, years)
typer.echo(
f"IRS 990 ingestion complete: {totals['filings']} filings, "
f"{totals['part_vii']} Part VII rows, {totals['schedule_j']} Schedule J rows."
)
conn.close()
@ingest_app.command()
def cpi() -> None:
"""Ingest BLS CPI-U data. (Not yet implemented.)"""
typer.echo("CPI-U ingestion is not yet implemented (Sprint 3).")
raise typer.Exit(1)
@ingest_app.command(name="all")
def ingest_all(
year_range: Annotated[
str, typer.Option(help="Year range, e.g. 2005-2024")
] = f"{DEFAULT_YEAR_RANGE.start}-{DEFAULT_YEAR_RANGE.stop - 1}",
force: Annotated[
bool, typer.Option("--force", help="Re-download even if files exist")
] = False,
) -> None:
"""Ingest all data sources."""
ipeds(year_range=year_range, component="all", force=force)

View file

@ -0,0 +1,50 @@
from pathlib import Path
# University of Delaware IPEDS UNITID
UD_UNITID = 130943
# Paths
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
DATA_DIR = PROJECT_ROOT / "data" / "raw"
DB_PATH = PROJECT_ROOT / "data" / "admin_analytics.duckdb"
# IPEDS
IPEDS_BASE_URL = "https://nces.ed.gov/ipeds/datacenter/data"
IPEDS_DATA_DIR = DATA_DIR / "ipeds"
DEFAULT_YEAR_RANGE = range(2005, 2025) # 2005-2024 inclusive
# IRS 990
UD_EINS = [516000297, 516017306] # UD + UD Research Foundation
IRS990_BASE_URL = "https://apps.irs.gov/pub/epostcard/990/xml"
IRS990_DATA_DIR = DATA_DIR / "irs990"
def ipeds_filename(component: str, year: int) -> str:
"""Return the IPEDS filename (without .zip) for a component and year.
IPEDS uses different naming conventions per survey:
HD: HD{YYYY} e.g. HD2023
F1A: F{YY-1}{YY}_F1A e.g. F2223_F1A (fiscal year)
EF: EF{YYYY}A e.g. EF2023A
S: S{YYYY}_OC e.g. S2023_OC (occupational categories)
SAL: SAL{YYYY}_IS e.g. SAL2023_IS
"""
if component == "hd":
return f"HD{year}"
elif component == "finance":
yy_prev = (year - 1) % 100
yy = year % 100
return f"F{yy_prev:02d}{yy:02d}_F1A"
elif component == "finance_f2":
yy_prev = (year - 1) % 100
yy = year % 100
return f"F{yy_prev:02d}{yy:02d}_F2"
elif component == "enrollment":
return f"EF{year}A"
elif component == "staff":
return f"S{year}_OC"
elif component == "salaries":
return f"SAL{year}_IS"
else:
raise ValueError(f"Unknown IPEDS component: {component}")

View file

View file

@ -0,0 +1,12 @@
from pathlib import Path
import duckdb
from admin_analytics.config import DB_PATH
def get_connection(path: Path | None = None) -> duckdb.DuckDBPyConnection:
"""Return a DuckDB connection, creating the parent directory if needed."""
path = path or DB_PATH
path.parent.mkdir(parents=True, exist_ok=True)
return duckdb.connect(str(path))

View file

@ -0,0 +1,122 @@
import duckdb
TABLES = {
"raw_institution": """
CREATE TABLE IF NOT EXISTS raw_institution (
unitid INTEGER NOT NULL,
year INTEGER NOT NULL,
ein VARCHAR,
institution_name VARCHAR,
city VARCHAR,
state VARCHAR,
sector INTEGER,
control INTEGER,
carnegie_class INTEGER,
enrollment_total INTEGER,
PRIMARY KEY (unitid, year)
)
""",
"raw_ipeds_finance": """
CREATE TABLE IF NOT EXISTS raw_ipeds_finance (
unitid INTEGER NOT NULL,
year INTEGER NOT NULL,
reporting_standard VARCHAR,
total_expenses BIGINT,
instruction_expenses BIGINT,
research_expenses BIGINT,
public_service_expenses BIGINT,
academic_support_expenses BIGINT,
student_services_expenses BIGINT,
institutional_support_expenses BIGINT,
auxiliary_expenses BIGINT,
hospital_expenses BIGINT,
other_expenses BIGINT,
salaries_wages BIGINT,
benefits BIGINT,
PRIMARY KEY (unitid, year)
)
""",
"raw_ipeds_staff": """
CREATE TABLE IF NOT EXISTS raw_ipeds_staff (
unitid INTEGER NOT NULL,
year INTEGER NOT NULL,
total_staff INTEGER,
faculty_total INTEGER,
management_total INTEGER,
PRIMARY KEY (unitid, year)
)
""",
"raw_ipeds_enrollment": """
CREATE TABLE IF NOT EXISTS raw_ipeds_enrollment (
unitid INTEGER NOT NULL,
year INTEGER NOT NULL,
total_enrollment INTEGER,
PRIMARY KEY (unitid, year)
)
""",
"raw_990_filing": """
CREATE TABLE IF NOT EXISTS raw_990_filing (
object_id VARCHAR PRIMARY KEY,
ein VARCHAR,
tax_year INTEGER,
organization_name VARCHAR,
return_type VARCHAR,
filing_date DATE,
total_revenue BIGINT,
total_expenses BIGINT,
total_assets BIGINT
)
""",
"raw_990_schedule_j": """
CREATE SEQUENCE IF NOT EXISTS seq_990_schedule_j START 1;
CREATE TABLE IF NOT EXISTS raw_990_schedule_j (
id INTEGER PRIMARY KEY DEFAULT nextval('seq_990_schedule_j'),
object_id VARCHAR,
ein VARCHAR,
tax_year INTEGER,
person_name VARCHAR,
title VARCHAR,
base_compensation BIGINT,
bonus_compensation BIGINT,
other_compensation BIGINT,
deferred_compensation BIGINT,
nontaxable_benefits BIGINT,
total_compensation BIGINT,
compensation_from_related BIGINT
)
""",
"raw_990_part_vii": """
CREATE SEQUENCE IF NOT EXISTS seq_990_part_vii START 1;
CREATE TABLE IF NOT EXISTS raw_990_part_vii (
id INTEGER PRIMARY KEY DEFAULT nextval('seq_990_part_vii'),
object_id VARCHAR,
ein VARCHAR,
tax_year INTEGER,
person_name VARCHAR,
title VARCHAR,
avg_hours_per_week DOUBLE,
reportable_comp_from_org BIGINT,
reportable_comp_from_related BIGINT,
other_compensation BIGINT
)
""",
"raw_cpi_u": """
CREATE TABLE IF NOT EXISTS raw_cpi_u (
year INTEGER NOT NULL,
month INTEGER NOT NULL,
value DOUBLE,
series_id VARCHAR,
PRIMARY KEY (year, month)
)
""",
}
def ensure_schema(conn: duckdb.DuckDBPyConnection) -> None:
"""Create all raw tables if they don't exist."""
for ddl in TABLES.values():
# Some DDL blocks contain multiple statements (e.g. CREATE SEQUENCE + CREATE TABLE)
for stmt in ddl.split(";"):
stmt = stmt.strip()
if stmt:
conn.execute(stmt)

View file

View file

@ -0,0 +1,108 @@
from pathlib import Path
import openpyxl
import xlrd
def parse_dictionary(dict_path: Path) -> dict[str, str]:
"""Parse an IPEDS dictionary Excel file, returning {varname: vartitle}.
Handles both .xls (xlrd) and .xlsx (openpyxl) formats.
Looks for a sheet named 'varlist' (case-insensitive).
"""
suffix = dict_path.suffix.lower()
if suffix == ".xls":
return _parse_xls(dict_path)
elif suffix == ".xlsx":
return _parse_xlsx(dict_path)
else:
raise ValueError(f"Unsupported dictionary format: {suffix}")
def _parse_xls(path: Path) -> dict[str, str]:
wb = xlrd.open_workbook(str(path))
sheet = _find_varlist_sheet_xls(wb)
if sheet is None:
return {}
# Detect column positions from header row
header = [str(v).strip().lower() for v in sheet.row_values(0)]
varname_idx = header.index("varname") if "varname" in header else 1
vartitle_idx = header.index("vartitle") if "vartitle" in header else len(header) - 1
mapping = {}
for row_idx in range(1, sheet.nrows):
row = sheet.row_values(row_idx)
if len(row) > max(varname_idx, vartitle_idx):
varname = str(row[varname_idx]).strip().upper()
vartitle = str(row[vartitle_idx]).strip()
if varname:
mapping[varname] = vartitle
return mapping
def _parse_xlsx(path: Path) -> dict[str, str]:
wb = openpyxl.load_workbook(str(path), read_only=True, data_only=True)
sheet = _find_varlist_sheet_xlsx(wb)
if sheet is None:
wb.close()
return {}
mapping = {}
varname_idx = 1 # default
vartitle_idx = -1 # default: last column
first = True
for row in sheet.iter_rows(values_only=True):
if first:
first = False
header = [str(v).strip().lower() if v else "" for v in row]
if "varname" in header:
varname_idx = header.index("varname")
if "vartitle" in header:
vartitle_idx = header.index("vartitle")
continue
if row and len(row) > varname_idx:
val = row[varname_idx]
if val:
varname = str(val).strip().upper()
vartitle = str(row[vartitle_idx]).strip() if row[vartitle_idx] else ""
if varname:
mapping[varname] = vartitle
wb.close()
return mapping
def _find_varlist_sheet_xls(wb: xlrd.Book):
for name in wb.sheet_names():
if name.lower() == "varlist":
return wb.sheet_by_name(name)
return None
def _find_varlist_sheet_xlsx(wb: openpyxl.Workbook):
for name in wb.sheetnames:
if name.lower() == "varlist":
return wb[name]
return None
def find_dict_file(dict_dir: Path) -> Path | None:
"""Find the dictionary Excel file in an extracted dictionary directory."""
for ext in (".xlsx", ".xls"):
files = list(dict_dir.glob(f"*{ext}"))
if files:
return files[0]
return None
def get_variable_mapping(component: str, year: int) -> dict[str, str]:
"""Load the variable mapping for a component/year pair.
Returns an empty dict if the dictionary file is not available.
"""
from admin_analytics.config import IPEDS_DATA_DIR
dict_dir = IPEDS_DATA_DIR / component / str(year) / "dict"
if not dict_dir.exists():
return {}
dict_file = find_dict_file(dict_dir)
if dict_file is None:
return {}
return parse_dictionary(dict_file)

View file

@ -0,0 +1,75 @@
import io
import zipfile
from pathlib import Path
import httpx
from admin_analytics import config
def _component_url(component: str, year: int) -> str:
"""Build the URL for an IPEDS data ZIP file."""
filename = config.ipeds_filename(component, year)
return f"{config.IPEDS_BASE_URL}/{filename}.zip"
def _dict_url(component: str, year: int) -> str:
"""Build the URL for an IPEDS dictionary ZIP file."""
filename = config.ipeds_filename(component, year)
return f"{config.IPEDS_BASE_URL}/{filename}_Dict.zip"
def _dest_dir(component: str, year: int) -> Path:
"""Return the local directory for a component/year pair."""
return config.IPEDS_DATA_DIR / component / str(year)
def _download_and_extract(url: str, dest: Path) -> Path:
"""Download a ZIP from url, extract to dest, return dest."""
dest.mkdir(parents=True, exist_ok=True)
with httpx.Client(follow_redirects=True, timeout=120.0) as client:
resp = client.get(url)
resp.raise_for_status()
with zipfile.ZipFile(io.BytesIO(resp.content)) as zf:
zf.extractall(dest)
return dest
def download_component(
component: str, year: int, force: bool = False
) -> Path:
"""Download an IPEDS data file. Returns the extraction directory."""
dest = _dest_dir(component, year)
if not force and dest.exists() and any(dest.iterdir()):
return dest
url = _component_url(component, year)
return _download_and_extract(url, dest)
def download_dictionary(
component: str, year: int, force: bool = False
) -> Path:
"""Download an IPEDS dictionary file. Returns the extraction directory."""
dest = _dest_dir(component, year) / "dict"
if not force and dest.exists() and any(dest.iterdir()):
return dest
url = _dict_url(component, year)
return _download_and_extract(url, dest)
def download_all(
year_range: range,
components: list[str],
force: bool = False,
) -> None:
"""Download all requested components for all years."""
for component in components:
for year in year_range:
try:
download_component(component, year, force=force)
except httpx.HTTPStatusError as e:
print(f" Warning: {component}/{year} data not available ({e.response.status_code})")
try:
download_dictionary(component, year, force=force)
except httpx.HTTPStatusError:
pass # Dictionaries aren't always available

View file

@ -0,0 +1,87 @@
from pathlib import Path
import duckdb
import polars as pl
from admin_analytics import config
from admin_analytics.config import UD_UNITID
CANONICAL_COLUMNS = ["unitid", "year", "total_enrollment"]
def _find_csv(component_dir: Path) -> Path | None:
csvs = [f for f in component_dir.glob("*.csv") if "dict" not in f.stem.lower()]
return csvs[0] if csvs else None
def load_enrollment(
conn: duckdb.DuckDBPyConnection,
year_range: range,
unitid_filter: int | None = UD_UNITID,
) -> int:
"""Load IPEDS enrollment data into raw_ipeds_enrollment.
EF files have multiple rows per institution (breakdowns by level, race, etc.).
EFALEVEL=1 is the grand total row. If EFALEVEL is not present, we aggregate
by taking the max EFTOTLT/EFYTOTLT per UNITID.
"""
total = 0
for year in year_range:
data_dir = config.IPEDS_DATA_DIR / "enrollment" / str(year)
csv_path = _find_csv(data_dir)
if csv_path is None:
print(f" No EF CSV found for {year}, skipping")
continue
df = pl.read_csv(
csv_path,
infer_schema_length=0,
encoding="utf8-lossy",
)
upper_cols = {c.upper(): c for c in df.columns}
# Find the enrollment total column
enroll_col = None
for candidate in ["EFTOTLT", "EFYTOTLT"]:
if candidate in upper_cols:
enroll_col = upper_cols[candidate]
break
if enroll_col is None:
print(f" No enrollment total column found for {year}, skipping")
continue
unitid_col = upper_cols.get("UNITID", "UNITID")
# Filter to grand total row (EFALEVEL=1) if that column exists
efalevel_col = upper_cols.get("EFALEVEL")
if efalevel_col:
df = df.filter(pl.col(efalevel_col).str.strip_chars() == "1")
# Select, rename, cast
df = df.select(
pl.col(unitid_col).cast(pl.Int64).alias("unitid"),
pl.lit(year).cast(pl.Int64).alias("year"),
pl.col(enroll_col).cast(pl.Int64, strict=False).alias("total_enrollment"),
)
# If still multiple rows per UNITID (shouldn't be after EFALEVEL filter), take max
df = df.group_by("unitid").agg(
pl.col("year").first(),
pl.col("total_enrollment").max(),
)
if unitid_filter is not None:
df = df.filter(pl.col("unitid") == unitid_filter)
if df.height == 0:
continue
df = df.select(CANONICAL_COLUMNS)
conn.execute("DELETE FROM raw_ipeds_enrollment WHERE year = ?", [year])
conn.register("_tmp_enrollment", df.to_arrow())
conn.execute("INSERT INTO raw_ipeds_enrollment SELECT * FROM _tmp_enrollment")
conn.unregister("_tmp_enrollment")
total += df.height
return total

View file

@ -0,0 +1,142 @@
from pathlib import Path
import duckdb
import polars as pl
from admin_analytics import config
from admin_analytics.config import UD_UNITID
# F1A (GASB/public) column mappings — first match wins per canonical column.
F1A_COLUMN_VARIANTS = {
"unitid": ["UNITID"],
"total_expenses": ["F1C191", "F1D02"],
"instruction_expenses": ["F1C011"],
"research_expenses": ["F1C021"],
"public_service_expenses": ["F1C031"],
"academic_support_expenses": ["F1C051"],
"student_services_expenses": ["F1C061"],
"institutional_support_expenses": ["F1C071"],
"auxiliary_expenses": ["F1C111"],
"hospital_expenses": ["F1C121"],
"other_expenses": ["F1C141"],
"salaries_wages": ["F1C192"],
"benefits": ["F1C193"],
}
# F2 (FASB/private-style) column mappings — UD reports here despite being public.
F2_COLUMN_VARIANTS = {
"unitid": ["UNITID"],
"total_expenses": ["F2E131"],
"instruction_expenses": ["F2E011"],
"research_expenses": ["F2E021"],
"public_service_expenses": ["F2E031"],
"academic_support_expenses": ["F2E041"],
"student_services_expenses": ["F2E051"],
"institutional_support_expenses": ["F2E061"],
"auxiliary_expenses": ["F2E071"],
"hospital_expenses": ["F2E081"],
"other_expenses": ["F2E121"],
"salaries_wages": ["F2E132"],
"benefits": ["F2E133"],
}
CANONICAL_COLUMNS = [
"unitid", "year", "reporting_standard", "total_expenses",
"instruction_expenses", "research_expenses", "public_service_expenses",
"academic_support_expenses", "student_services_expenses",
"institutional_support_expenses", "auxiliary_expenses", "hospital_expenses",
"other_expenses", "salaries_wages", "benefits",
]
def _find_csv(component_dir: Path) -> Path | None:
csvs = [f for f in component_dir.glob("*.csv") if "dict" not in f.stem.lower()]
return csvs[0] if csvs else None
def _resolve_columns(df: pl.DataFrame, variants: dict) -> dict[str, str]:
"""For each canonical name, find the first matching column."""
upper_cols = {c.upper(): c for c in df.columns}
resolved = {}
for canonical, candidates in variants.items():
for var in candidates:
if var in upper_cols:
resolved[canonical] = upper_cols[var]
break
return resolved
def _load_file(
csv_path: Path,
year: int,
variants: dict,
reporting_standard: str,
conn: duckdb.DuckDBPyConnection,
unitid_filter: int | None,
) -> int:
"""Load a single finance CSV into raw_ipeds_finance."""
df = pl.read_csv(csv_path, infer_schema_length=0, encoding="utf8-lossy")
col_map = _resolve_columns(df, variants)
if "unitid" not in col_map:
return 0
# Build dataframe with canonical columns
result = pl.DataFrame({
canonical: df[actual] for canonical, actual in col_map.items()
})
result = result.with_columns(
pl.lit(year).alias("year"),
pl.lit(reporting_standard).alias("reporting_standard"),
)
# Cast numeric columns
for col in CANONICAL_COLUMNS:
if col not in result.columns:
result = result.with_columns(pl.lit(None).alias(col))
elif col not in ("reporting_standard",):
result = result.with_columns(pl.col(col).cast(pl.Int64, strict=False))
if unitid_filter is not None:
result = result.filter(pl.col("unitid") == unitid_filter)
if result.height == 0:
return 0
result = result.select(CANONICAL_COLUMNS)
conn.register("_tmp_finance", result.to_arrow())
conn.execute("INSERT INTO raw_ipeds_finance SELECT * FROM _tmp_finance")
conn.unregister("_tmp_finance")
return result.height
def load_finance(
conn: duckdb.DuckDBPyConnection,
year_range: range,
unitid_filter: int | None = UD_UNITID,
) -> int:
"""Load IPEDS finance data into raw_ipeds_finance.
Tries both F1A (GASB) and F2 (FASB) files since some public institutions
like UD report under FASB.
"""
total = 0
for year in year_range:
conn.execute("DELETE FROM raw_ipeds_finance WHERE year = ?", [year])
# Try F1A (GASB)
f1a_dir = config.IPEDS_DATA_DIR / "finance" / str(year)
csv_path = _find_csv(f1a_dir)
if csv_path:
total += _load_file(csv_path, year, F1A_COLUMN_VARIANTS, "GASB", conn, unitid_filter)
# Try F2 (FASB) — needed for institutions like UD
f2_dir = config.IPEDS_DATA_DIR / "finance_f2" / str(year)
csv_path = _find_csv(f2_dir)
if csv_path:
total += _load_file(csv_path, year, F2_COLUMN_VARIANTS, "FASB", conn, unitid_filter)
if total == 0 and not _find_csv(f1a_dir) and not _find_csv(f2_dir):
print(f" No finance CSV found for {year}, skipping")
return total

View file

@ -0,0 +1,107 @@
from pathlib import Path
import duckdb
import polars as pl
from admin_analytics import config
from admin_analytics.config import UD_UNITID
# HD column mappings: canonical name → list of IPEDS varnames (priority order).
# The first match found in the file's columns wins.
HD_COLUMN_VARIANTS = {
"unitid": ["UNITID"],
"institution_name": ["INSTNM"],
"city": ["CITY"],
"state": ["STABBR"],
"sector": ["SECTOR"],
"control": ["CONTROL"],
"ein": ["EIN"],
"carnegie_class": ["C21BASIC", "C18BASIC", "C15BASIC", "CCBASIC", "CARNEGIE"],
"enrollment_total": ["EFYTOTLT"],
}
CANONICAL_COLUMNS = [
"unitid", "year", "ein", "institution_name", "city", "state",
"sector", "control", "carnegie_class", "enrollment_total",
]
def _find_csv(component_dir: Path) -> Path | None:
"""Find the main data CSV in an extracted IPEDS directory."""
csvs = [f for f in component_dir.glob("*.csv") if "dict" not in f.stem.lower()]
return csvs[0] if csvs else None
def _resolve_columns(df: pl.DataFrame) -> dict[str, str]:
"""For each canonical name, find the first matching IPEDS column."""
upper_cols = {c.upper(): c for c in df.columns}
resolved = {}
for canonical, variants in HD_COLUMN_VARIANTS.items():
for var in variants:
if var in upper_cols:
resolved[canonical] = upper_cols[var]
break
return resolved
def _map_columns(df: pl.DataFrame, year: int) -> pl.DataFrame:
"""Rename IPEDS columns to canonical names and add year."""
col_map = _resolve_columns(df)
# Select and rename only the resolved columns
select_exprs = [pl.col(actual).alias(canonical) for canonical, actual in col_map.items()]
df = pl.DataFrame({canonical: df[actual] for canonical, actual in col_map.items()})
df = df.with_columns(pl.lit(year).alias("year"))
# Ensure all canonical columns exist
for col in CANONICAL_COLUMNS:
if col not in df.columns:
df = df.with_columns(pl.lit(None).alias(col))
return df.select(CANONICAL_COLUMNS)
def load_institutions(
conn: duckdb.DuckDBPyConnection,
year_range: range,
unitid_filter: int | None = UD_UNITID,
) -> int:
"""Load IPEDS HD data into raw_institution. Returns rows loaded."""
total = 0
for year in year_range:
data_dir = config.IPEDS_DATA_DIR / "hd" / str(year)
csv_path = _find_csv(data_dir)
if csv_path is None:
print(f" No HD CSV found for {year}, skipping")
continue
df = pl.read_csv(
csv_path,
infer_schema_length=0, # read everything as string first
encoding="utf8-lossy",
)
df = _map_columns(df, year)
# Cast types
df = df.with_columns(
pl.col("unitid").cast(pl.Int64),
pl.col("year").cast(pl.Int64),
pl.col("sector").cast(pl.Int64, strict=False),
pl.col("control").cast(pl.Int64, strict=False),
pl.col("carnegie_class").cast(pl.Int64, strict=False),
pl.col("enrollment_total").cast(pl.Int64, strict=False),
)
if unitid_filter is not None:
df = df.filter(pl.col("unitid") == unitid_filter)
if df.height == 0:
continue
conn.execute("DELETE FROM raw_institution WHERE year = ?", [year])
conn.register("_tmp_institution", df.to_arrow())
conn.execute("INSERT INTO raw_institution SELECT * FROM _tmp_institution")
conn.unregister("_tmp_institution")
total += df.height
return total

View file

@ -0,0 +1,98 @@
from pathlib import Path
import duckdb
import polars as pl
from admin_analytics import config
from admin_analytics.config import UD_UNITID
CANONICAL_COLUMNS = ["unitid", "year", "total_staff", "faculty_total", "management_total"]
def _find_csv(component_dir: Path) -> Path | None:
csvs = [f for f in component_dir.glob("*.csv") if "dict" not in f.stem.lower()]
return csvs[0] if csvs else None
def load_staff(
conn: duckdb.DuckDBPyConnection,
year_range: range,
unitid_filter: int | None = UD_UNITID,
) -> int:
"""Load IPEDS Fall Staff data into raw_ipeds_staff.
The S_OC file has multiple rows per institution (by occupational category).
We extract:
- total_staff: FTPT=2, OCCUPCAT=100 (full-time total, all occupations)
- management_total: FTPT=2, OCCUPCAT=200 (full-time management)
- faculty_total: FTPT=2, OCCUPCAT=250 (full-time instruction/research)
"""
total = 0
for year in year_range:
data_dir = config.IPEDS_DATA_DIR / "staff" / str(year)
csv_path = _find_csv(data_dir)
if csv_path is None:
print(f" No Staff CSV found for {year}, skipping")
continue
df = pl.read_csv(
csv_path,
infer_schema_length=0,
encoding="utf8-lossy",
)
upper_cols = {c.upper(): c for c in df.columns}
unitid_col = upper_cols.get("UNITID", "UNITID")
hrtotlt_col = upper_cols.get("HRTOTLT")
occupcat_col = upper_cols.get("OCCUPCAT")
ftpt_col = upper_cols.get("FTPT")
stafftot_col = upper_cols.get("STAFFTOT") # older format: single row
if hrtotlt_col and occupcat_col and ftpt_col:
# Modern multi-row format: filter by FTPT and OCCUPCAT
df = df.with_columns(
pl.col(ftpt_col).str.strip_chars().alias("_ftpt"),
pl.col(occupcat_col).str.strip_chars().alias("_occupcat"),
pl.col(unitid_col).cast(pl.Int64).alias("unitid"),
pl.col(hrtotlt_col).cast(pl.Int64, strict=False).alias("_count"),
)
# Full-time totals only (FTPT=2)
ft = df.filter(pl.col("_ftpt") == "2")
# Pivot occupational categories into columns per UNITID
total_staff = ft.filter(pl.col("_occupcat") == "100").select("unitid", pl.col("_count").alias("total_staff"))
management = ft.filter(pl.col("_occupcat") == "200").select("unitid", pl.col("_count").alias("management_total"))
faculty = ft.filter(pl.col("_occupcat") == "250").select("unitid", pl.col("_count").alias("faculty_total"))
result = total_staff.join(management, on="unitid", how="left").join(faculty, on="unitid", how="left")
elif stafftot_col:
# Older single-row format
result = df.select(
pl.col(unitid_col).cast(pl.Int64).alias("unitid"),
pl.col(stafftot_col).cast(pl.Int64, strict=False).alias("total_staff"),
)
result = result.with_columns(
pl.lit(None).cast(pl.Int64).alias("faculty_total"),
pl.lit(None).cast(pl.Int64).alias("management_total"),
)
else:
print(f" Unrecognized staff file format for {year}, skipping")
continue
result = result.with_columns(pl.lit(year).cast(pl.Int64).alias("year"))
if unitid_filter is not None:
result = result.filter(pl.col("unitid") == unitid_filter)
if result.height == 0:
continue
result = result.select(CANONICAL_COLUMNS)
conn.execute("DELETE FROM raw_ipeds_staff WHERE year = ?", [year])
conn.register("_tmp_staff", result.to_arrow())
conn.execute("INSERT INTO raw_ipeds_staff SELECT * FROM _tmp_staff")
conn.unregister("_tmp_staff")
total += result.height
return total

View file

View file

@ -0,0 +1,194 @@
"""IRS 990 e-file download: index filtering and XML extraction from bulk ZIPs."""
import io
import re
import tempfile
import zipfile
from pathlib import Path
import httpx
import polars as pl
from admin_analytics import config
def download_index(year: int, force: bool = False) -> Path:
"""Download the IRS 990 e-file index CSV for a given year."""
dest_dir = config.IRS990_DATA_DIR / str(year)
dest_dir.mkdir(parents=True, exist_ok=True)
dest = dest_dir / f"index_{year}.csv"
if not force and dest.exists():
return dest
url = f"{config.IRS990_BASE_URL}/{year}/index_{year}.csv"
with httpx.Client(follow_redirects=True, timeout=120.0) as client:
resp = client.get(url)
resp.raise_for_status()
dest.write_bytes(resp.content)
return dest
def filter_index(
index_path: Path,
eins: list[int],
return_types: list[str] | None = None,
) -> pl.DataFrame:
"""Filter the index CSV to specific EINs and return types.
Args:
index_path: Path to the index CSV.
eins: EINs to filter for.
return_types: e.g. ["990", "990PF"]. Default: ["990"] (skip 990T, 990EZ).
"""
if return_types is None:
return_types = ["990"]
df = pl.read_csv(index_path, infer_schema_length=0, encoding="utf8-lossy")
# Normalize column names to uppercase
df = df.rename({c: c.strip().upper() for c in df.columns})
ein_strs = [str(e) for e in eins]
df = df.filter(
pl.col("EIN").is_in(ein_strs)
& pl.col("RETURN_TYPE").is_in(return_types)
)
return df
def get_batch_list(year: int) -> list[str]:
"""Get the list of available TEOS XML batch ZIP IDs for a year.
Tries common naming patterns: {YEAR}_TEOS_XML_{MM}{L} where
MM is 01-12 and L is A-D.
"""
batches = []
with httpx.Client(follow_redirects=True, timeout=30.0) as client:
for month in range(1, 13):
for letter in "ABCD":
batch_id = f"{year}_TEOS_XML_{month:02d}{letter}"
url = f"{config.IRS990_BASE_URL}/{year}/{batch_id}.zip"
try:
resp = client.head(url)
if resp.status_code == 200:
batches.append(batch_id)
except httpx.HTTPError:
continue
return batches
def _extract_xml_from_zip(
zip_url: str, object_id: str, dest: Path
) -> bool:
"""Download a batch ZIP and extract a specific XML file.
Streams the ZIP to a temp file, extracts the target, deletes the temp.
Returns True if the file was found and extracted.
"""
target_name = f"{object_id}_public.xml"
with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp:
tmp_path = Path(tmp.name)
try:
with httpx.Client(follow_redirects=True, timeout=600.0) as client:
with client.stream("GET", zip_url) as resp:
resp.raise_for_status()
with open(tmp_path, "wb") as f:
for chunk in resp.iter_bytes(chunk_size=1024 * 1024):
f.write(chunk)
with zipfile.ZipFile(tmp_path) as zf:
names = zf.namelist()
# Look for our target file (case-insensitive)
match = None
for name in names:
if name.lower() == target_name.lower():
match = name
break
if match is None:
return False
dest.parent.mkdir(parents=True, exist_ok=True)
with zf.open(match) as src, open(dest, "wb") as dst:
dst.write(src.read())
return True
finally:
tmp_path.unlink(missing_ok=True)
def download_filing_xml(
year: int,
object_id: str,
batch_id: str | None = None,
force: bool = False,
) -> Path | None:
"""Download a specific 990 XML filing from the IRS bulk data.
Args:
year: The index year (not necessarily the tax year).
object_id: The OBJECT_ID from the index CSV.
batch_id: The XML_BATCH_ID if known (2024+ indexes have this).
force: Re-download even if already exists.
Returns:
Path to the extracted XML file, or None if not found.
"""
dest = config.IRS990_DATA_DIR / str(year) / f"{object_id}.xml"
if not force and dest.exists():
return dest
if batch_id:
# We know which ZIP to look in
url = f"{config.IRS990_BASE_URL}/{year}/{batch_id}.zip"
if _extract_xml_from_zip(url, object_id, dest):
return dest
return None
# No batch ID — try each batch until we find our file
batches = get_batch_list(year)
for bid in batches:
url = f"{config.IRS990_BASE_URL}/{year}/{bid}.zip"
print(f" Searching batch {bid}...")
if _extract_xml_from_zip(url, object_id, dest):
print(f" Found in {bid}")
return dest
print(f" Warning: filing {object_id} not found in any batch for {year}")
return None
def download_all_filings(
year_range: range,
eins: list[int] | None = None,
force: bool = False,
) -> list[Path]:
"""Download all 990 XML filings for the target EINs across years.
Returns list of paths to downloaded XML files.
"""
eins = eins or config.UD_EINS
xml_paths = []
for year in year_range:
print(f" Downloading index for {year}...")
try:
index_path = download_index(year, force=force)
except httpx.HTTPStatusError as e:
print(f" Index not available for {year} ({e.response.status_code})")
continue
filings = filter_index(index_path, eins)
if filings.height == 0:
print(f" No filings found for target EINs in {year}")
continue
print(f" Found {filings.height} filing(s) in {year}")
for row in filings.iter_rows(named=True):
object_id = row["OBJECT_ID"]
batch_id = row.get("XML_BATCH_ID")
taxpayer = row.get("TAXPAYER_NAME", "")
print(f" Downloading {taxpayer} ({object_id})...")
path = download_filing_xml(year, object_id, batch_id, force=force)
if path:
xml_paths.append(path)
return xml_paths

View file

@ -0,0 +1,162 @@
"""Load parsed IRS 990 data into DuckDB."""
from pathlib import Path
import duckdb
from admin_analytics import config
from admin_analytics.irs990.parser import parse_xml, parse_filing, parse_part_vii, parse_schedule_j
def load_filing(conn: duckdb.DuckDBPyConnection, xml_path: Path, object_id: str) -> bool:
"""Parse a 990 XML and insert the filing header into raw_990_filing.
Returns True if a row was inserted.
"""
root = parse_xml(xml_path)
filing = parse_filing(root)
if not filing["ein"] or not filing["tax_year"]:
return False
conn.execute(
"DELETE FROM raw_990_filing WHERE object_id = ?", [object_id]
)
conn.execute(
"""INSERT INTO raw_990_filing
(object_id, ein, tax_year, organization_name, return_type,
total_revenue, total_expenses, total_assets)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
[
object_id,
filing["ein"],
filing["tax_year"],
filing["organization_name"],
filing["return_type"],
filing["total_revenue"],
filing["total_expenses"],
filing["total_assets"],
],
)
return True
def load_part_vii(conn: duckdb.DuckDBPyConnection, xml_path: Path, object_id: str) -> int:
"""Parse Part VII compensation and insert into raw_990_part_vii.
Returns number of rows inserted.
"""
root = parse_xml(xml_path)
filing = parse_filing(root)
people = parse_part_vii(root)
if not people:
return 0
ein = filing["ein"]
tax_year = filing["tax_year"]
conn.execute(
"DELETE FROM raw_990_part_vii WHERE object_id = ?", [object_id]
)
count = 0
for person in people:
if not person["person_name"]:
continue
conn.execute(
"""INSERT INTO raw_990_part_vii
(object_id, ein, tax_year, person_name, title,
avg_hours_per_week, reportable_comp_from_org,
reportable_comp_from_related, other_compensation)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
[
object_id,
ein,
tax_year,
person["person_name"],
person["title"],
person["avg_hours_per_week"],
person["reportable_comp_from_org"],
person["reportable_comp_from_related"],
person["other_compensation"],
],
)
count += 1
return count
def load_schedule_j(conn: duckdb.DuckDBPyConnection, xml_path: Path, object_id: str) -> int:
"""Parse Schedule J compensation and insert into raw_990_schedule_j.
Returns number of rows inserted.
"""
root = parse_xml(xml_path)
filing = parse_filing(root)
people = parse_schedule_j(root)
if not people:
return 0
ein = filing["ein"]
tax_year = filing["tax_year"]
conn.execute(
"DELETE FROM raw_990_schedule_j WHERE object_id = ?", [object_id]
)
count = 0
for person in people:
if not person["person_name"]:
continue
conn.execute(
"""INSERT INTO raw_990_schedule_j
(object_id, ein, tax_year, person_name, title,
base_compensation, bonus_compensation, other_compensation,
deferred_compensation, nontaxable_benefits,
total_compensation, compensation_from_related)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
[
object_id,
ein,
tax_year,
person["person_name"],
person["title"],
person["base_compensation"],
person["bonus_compensation"],
person["other_compensation"],
person["deferred_compensation"],
person["nontaxable_benefits"],
person["total_compensation"],
person["compensation_from_related"],
],
)
count += 1
return count
def load_all(
conn: duckdb.DuckDBPyConnection,
year_range: range,
) -> dict[str, int]:
"""Load all downloaded 990 XML filings into DuckDB.
Returns counts of rows loaded per table.
"""
totals = {"filings": 0, "part_vii": 0, "schedule_j": 0}
for year in year_range:
year_dir = config.IRS990_DATA_DIR / str(year)
if not year_dir.exists():
continue
for xml_path in sorted(year_dir.glob("*.xml")):
object_id = xml_path.stem
if load_filing(conn, xml_path, object_id):
totals["filings"] += 1
totals["part_vii"] += load_part_vii(conn, xml_path, object_id)
totals["schedule_j"] += load_schedule_j(conn, xml_path, object_id)
return totals

View file

@ -0,0 +1,259 @@
"""Parse IRS 990 XML filings: header, Part VII, and Schedule J."""
from pathlib import Path
from lxml import etree
def _strip_ns(tree: etree._Element) -> etree._Element:
"""Strip all namespaces from an XML tree for simpler XPath queries."""
for el in tree.iter():
if isinstance(el.tag, str) and "{" in el.tag:
el.tag = el.tag.split("}", 1)[1]
for attr_name in list(el.attrib):
if "{" in attr_name:
new_name = attr_name.split("}", 1)[1]
el.attrib[new_name] = el.attrib.pop(attr_name)
return tree
def _text(el: etree._Element | None, path: str) -> str | None:
"""Get text of a child element by relative XPath, or None."""
if el is None:
return None
child = el.find(path)
return child.text.strip() if child is not None and child.text else None
def _int(el: etree._Element | None, path: str) -> int | None:
"""Get integer value of a child element."""
text = _text(el, path)
if text is None:
return None
try:
return int(text)
except ValueError:
return None
def _float(el: etree._Element | None, path: str) -> float | None:
"""Get float value of a child element."""
text = _text(el, path)
if text is None:
return None
try:
return float(text)
except ValueError:
return None
def _find_first(root: etree._Element, xpaths: list[str]) -> list:
"""Try multiple XPaths, return the first one that produces results."""
for xpath in xpaths:
results = root.findall(xpath)
if results:
return results
return []
def _find_first_text(el: etree._Element, paths: list[str]) -> str | None:
"""Try multiple child element names, return the first one with text."""
for path in paths:
val = _text(el, path)
if val is not None:
return val
return None
def _find_first_int(el: etree._Element, paths: list[str]) -> int | None:
"""Try multiple child element names, return the first integer found."""
for path in paths:
val = _int(el, path)
if val is not None:
return val
return None
def parse_xml(xml_path: Path) -> etree._Element:
"""Parse an XML file and strip namespaces."""
tree = etree.parse(str(xml_path))
return _strip_ns(tree.getroot())
def parse_filing(root: etree._Element) -> dict:
"""Extract filing header fields from a parsed 990 XML."""
header = root.find(".//ReturnHeader")
ein = _find_first_text(header, [
"Filer/EIN",
"Filer/EIN",
]) if header is not None else None
tax_year = _find_first_text(header, [
"TaxYr",
"TaxYear",
]) if header is not None else None
# If no TaxYr element, extract from TaxPeriodEndDt (YYYY-MM-DD -> YYYY)
if tax_year is None and header is not None:
tax_period = _find_first_text(header, [
"TaxPeriodEndDt",
"TaxPrdEndDt",
])
if tax_period:
tax_year = tax_period[:4]
org_name = None
if header is not None:
org_name = _find_first_text(header, [
"Filer/BusinessName/BusinessNameLine1Txt",
"Filer/BusinessName/BusinessNameLine1",
"Filer/Name/BusinessNameLine1",
])
return_type = _find_first_text(header, [
"ReturnTypeCd",
"ReturnType",
]) if header is not None else None
# Get financial summary from the main form
irs990 = root.find(".//IRS990")
total_revenue = _find_first_int(irs990, [
"CYTotalRevenueAmt",
"TotalRevenueCurrentYear",
"TotalRevenueAmt",
]) if irs990 is not None else None
total_expenses = _find_first_int(irs990, [
"CYTotalExpensesAmt",
"TotalExpensesCurrentYear",
"TotalFunctionalExpensesAmt",
]) if irs990 is not None else None
total_assets = _find_first_int(root, [
".//IRS990/TotalAssetsEOYAmt",
".//IRS990/TotalAssetsEndOfYear",
".//TotalAssetsEOYAmt",
])
return {
"ein": ein,
"tax_year": int(tax_year) if tax_year else None,
"organization_name": org_name,
"return_type": return_type,
"total_revenue": total_revenue,
"total_expenses": total_expenses,
"total_assets": total_assets,
}
def parse_part_vii(root: etree._Element) -> list[dict]:
"""Extract Part VII (compensation summary) from a parsed 990 XML.
Returns one dict per listed person with compensation details.
"""
groups = _find_first(root, [
".//IRS990/Form990PartVIISectionAGrp",
".//IRS990/Form990PartVIISectionA",
])
results = []
for grp in groups:
person_name = _find_first_text(grp, [
"PersonNm",
"NamePerson",
"PersonName",
"NameBusiness/BusinessNameLine1Txt",
])
title = _find_first_text(grp, [
"TitleTxt",
"Title",
])
hours = _find_first(grp, ["AverageHoursPerWeekRt", "AverageHoursPerWeek"])
avg_hours = None
if hours:
try:
avg_hours = float(hours[0].text)
except (ValueError, TypeError):
pass
results.append({
"person_name": person_name,
"title": title,
"avg_hours_per_week": avg_hours,
"reportable_comp_from_org": _find_first_int(grp, [
"ReportableCompFromOrgAmt",
"ReportableCompFromOrganization",
]),
"reportable_comp_from_related": _find_first_int(grp, [
"ReportableCompFromRltdOrgAmt",
"ReportableCompFromRelatedOrgs",
]),
"other_compensation": _find_first_int(grp, [
"OtherCompensationAmt",
"OtherCompensation",
]),
})
return results
def parse_schedule_j(root: etree._Element) -> list[dict]:
"""Extract Schedule J (detailed compensation) from a parsed 990 XML.
Returns one dict per person with itemized compensation breakdown.
"""
groups = _find_first(root, [
".//IRS990ScheduleJ/RltdOrgOfficerTrstKeyEmplGrp",
".//IRS990ScheduleJ/Form990ScheduleJPartII",
".//IRS990ScheduleJ/SupplementalInformationDetail",
])
results = []
for grp in groups:
person_name = _find_first_text(grp, [
"PersonNm",
"NamePerson",
"PersonName",
"NameBusiness/BusinessNameLine1Txt",
])
title = _find_first_text(grp, [
"TitleTxt",
"Title",
])
results.append({
"person_name": person_name,
"title": title,
"base_compensation": _find_first_int(grp, [
"BaseCompensationFilingOrgAmt",
"BaseCompensation",
]),
"bonus_compensation": _find_first_int(grp, [
"BonusFilingOrgAmt",
"Bonus",
]),
"other_compensation": _find_first_int(grp, [
"OtherCompensationFilingOrgAmt",
"OtherCompensation",
]),
"deferred_compensation": _find_first_int(grp, [
"DeferredCompensationFlngOrgAmt",
"DeferredCompensation",
]),
"nontaxable_benefits": _find_first_int(grp, [
"NontaxableBenefitsFilingOrgAmt",
"NontaxableBenefits",
]),
"total_compensation": _find_first_int(grp, [
"TotalCompensationFilingOrgAmt",
"TotalCompensation",
]),
"compensation_from_related": _find_first_int(grp, [
"CompensationFromOtherSrcsAmt",
"CompensationFromOtherSources",
"CompReportedInPriorFormFilingOrgAmt",
]),
})
return results

View file

@ -0,0 +1,43 @@
"""Normalize 990 titles to canonical roles."""
import re
# Canonical roles and their matching patterns.
# Order matters — first match wins. More specific patterns before general ones.
TITLE_PATTERNS: list[tuple[str, re.Pattern]] = [
# VP patterns must come before PRESIDENT to avoid false matches
("PROVOST", re.compile(r"\bprovost\b", re.I)),
("VP_FINANCE", re.compile(r"(?:\bv\.?p\.?\b|\bvice\s+president\b).*\b(?:financ|budget|business|admin)|\b(?:financ|budget|business|admin).*(?:\bv\.?p\.?\b|\bvice\s+president\b)", re.I)),
("VP_RESEARCH", re.compile(r"(?:\bv\.?p\.?\b|\bvice\s+president\b).*\bresearch|\bresearch.*(?:\bv\.?p\.?\b|\bvice\s+president\b)", re.I)),
("VP_STUDENT_AFFAIRS", re.compile(r"(?:\bv\.?p\.?\b|\bvice\s+president\b).*\bstudent|\bstudent.*(?:\bv\.?p\.?\b|\bvice\s+president\b)", re.I)),
("VP_ADVANCEMENT", re.compile(r"(?:\bv\.?p\.?\b|\bvice\s+president\b).*\b(?:advancement|development|giving|fundrais)|\b(?:advancement|development|giving|fundrais).*(?:\bv\.?p\.?\b|\bvice\s+president\b)", re.I)),
("VP_OTHER", re.compile(r"\bv\.?p\.?\b|\bvice\s+president\b", re.I)),
("CFO", re.compile(r"\b(cfo|chief\s+financial)\b", re.I)),
("CIO", re.compile(r"\b(cio|chief\s+information)\b", re.I)),
("COO", re.compile(r"\b(coo|chief\s+operating)\b", re.I)),
("GENERAL_COUNSEL", re.compile(r"\b(general\s+counsel|chief\s+legal)\b", re.I)),
("PRESIDENT", re.compile(r"\bpresident\b", re.I)),
("DEAN", re.compile(r"\bdean\b", re.I)),
("TREASURER", re.compile(r"\btreasurer\b", re.I)),
("SECRETARY", re.compile(r"\bsecretary\b", re.I)),
("TRUSTEE", re.compile(r"\btrustee\b", re.I)),
("DIRECTOR", re.compile(r"\bdirector\b", re.I)),
]
def normalize_title(raw_title: str | None) -> str | None:
"""Map a raw 990 title string to a canonical role.
Returns the canonical role string, or "OTHER" for unmatched titles,
or None if the input is None/empty.
"""
if not raw_title or not raw_title.strip():
return None
title = raw_title.strip()
for role, pattern in TITLE_PATTERNS:
if pattern.search(title):
return role
return "OTHER"

View file