Phase 1 project prototype

This commit is contained in:
emfurst 2026-03-30 19:29:33 -04:00
commit 2c9ae1c312
29 changed files with 2967 additions and 22 deletions

View file

@ -0,0 +1,32 @@
"""BLS CPI-U flat-file download."""
from pathlib import Path
import httpx
from admin_analytics.config import BLS_CPI_URL, BLS_DATA_DIR
CPI_FILENAME = "cu.data.0.Current"
def download_cpi_file(force: bool = False) -> Path:
"""Download the BLS CPI-U flat file.
Returns the path to the downloaded file.
"""
BLS_DATA_DIR.mkdir(parents=True, exist_ok=True)
dest = BLS_DATA_DIR / CPI_FILENAME
if not force and dest.exists():
return dest
with httpx.Client(
follow_redirects=True,
timeout=120.0,
headers={"User-Agent": "admin-analytics/0.1 (research)"},
) as client:
resp = client.get(BLS_CPI_URL)
resp.raise_for_status()
dest.write_bytes(resp.content)
return dest

View file

@ -0,0 +1,56 @@
"""Parse and load BLS CPI-U data into DuckDB."""
from pathlib import Path
import duckdb
import polars as pl
def load_cpi(conn: duckdb.DuckDBPyConnection, file_path: Path) -> int:
"""Parse the BLS CPI-U flat file and load into raw_cpi_u.
Filters to series CUUR0000SA0 (All Urban Consumers, US City Average,
All Items, Not Seasonally Adjusted) and monthly periods M01-M12.
Returns number of rows loaded.
"""
df = pl.read_csv(
file_path,
separator="\t",
infer_schema_length=0, # read all as strings first
)
# BLS pads fields with whitespace — strip all string columns
df = df.with_columns(
pl.col(col).str.strip_chars() for col in df.columns
)
# Normalize column names (BLS has trailing spaces in headers too)
df = df.rename({col: col.strip() for col in df.columns})
# Filter to CPI-U series and monthly periods only
df = df.filter(
(pl.col("series_id") == "CUUR0000SA0")
& (pl.col("period").str.starts_with("M"))
& (pl.col("period") != "M13")
)
# Extract month from period (M01 -> 1, M12 -> 12)
df = df.with_columns(
pl.col("year").cast(pl.Int64).alias("year"),
pl.col("period").str.slice(1).cast(pl.Int64).alias("month"),
pl.col("value").cast(pl.Float64).alias("value"),
pl.lit("CUUR0000SA0").alias("series_id"),
).select("year", "month", "value", "series_id")
# Idempotent: delete all existing rows then insert
conn.execute("DELETE FROM raw_cpi_u")
rows = df.to_dicts()
for row in rows:
conn.execute(
"INSERT INTO raw_cpi_u (year, month, value, series_id) VALUES (?, ?, ?, ?)",
[row["year"], row["month"], row["value"], row["series_id"]],
)
return len(rows)

View file

@ -107,10 +107,83 @@ def irs990(
@ingest_app.command()
def cpi() -> None:
"""Ingest BLS CPI-U data. (Not yet implemented.)"""
typer.echo("CPI-U ingestion is not yet implemented (Sprint 3).")
raise typer.Exit(1)
def cpi(
force: Annotated[
bool, typer.Option("--force", help="Re-download even if file exists")
] = False,
) -> None:
"""Ingest BLS CPI-U data."""
conn = get_connection()
ensure_schema(conn)
from admin_analytics.bls.download import download_cpi_file
from admin_analytics.bls.loader import load_cpi
typer.echo("Downloading BLS CPI-U data...")
file_path = download_cpi_file(force=force)
typer.echo("Loading CPI-U data into database...")
count = load_cpi(conn, file_path)
typer.echo(f"CPI-U ingestion complete: {count} monthly observations loaded.")
conn.close()
@ingest_app.command()
def scrape() -> None:
"""Scrape UD staff directory pages for admin headcounts."""
conn = get_connection()
ensure_schema(conn)
from admin_analytics.scraper.directory import scrape_all
from admin_analytics.scraper.loader import load_scrape
from admin_analytics.scraper.classify import OVERHEAD_CATEGORIES, NON_OVERHEAD_CATEGORIES
typer.echo("Scraping UD staff directory pages...")
entries = scrape_all()
typer.echo("Loading scraped data into database...")
count = load_scrape(conn, entries)
# Summary by unit and category
typer.echo(f"\nLoaded {count} staff entries.\n")
unit_counts: dict[str, dict[str, int]] = {}
for e in entries:
unit_counts.setdefault(e.unit, {})
unit_counts[e.unit][e.category] = unit_counts[e.unit].get(e.category, 0) + 1
for unit, cats in sorted(unit_counts.items()):
total = sum(cats.values())
overhead = sum(v for k, v in cats.items() if k in OVERHEAD_CATEGORIES)
typer.echo(f" {unit}: {total} staff ({overhead} overhead)")
for cat, n in sorted(cats.items(), key=lambda x: -x[1]):
typer.echo(f" {cat}: {n}")
conn.close()
@app.command()
def dashboard(
port: Annotated[int, typer.Option(help="Port to serve on")] = 8050,
debug: Annotated[bool, typer.Option(help="Enable Dash debug mode")] = True,
) -> None:
"""Launch the analytics dashboard."""
from admin_analytics.dashboard.app import create_app
dash_app = create_app()
typer.echo(f"Starting dashboard at http://localhost:{port}/")
dash_app.run(debug=debug, port=port)
@app.command()
def validate() -> None:
"""Run data validation checks and print a report."""
conn = get_connection()
ensure_schema(conn)
from admin_analytics.validation import format_report
typer.echo(format_report(conn))
conn.close()
@ingest_app.command(name="all")
@ -124,3 +197,6 @@ def ingest_all(
) -> None:
"""Ingest all data sources."""
ipeds(year_range=year_range, component="all", force=force)
irs990(year_range=year_range, force=force)
cpi(force=force)
scrape()

View file

@ -14,6 +14,10 @@ IPEDS_DATA_DIR = DATA_DIR / "ipeds"
DEFAULT_YEAR_RANGE = range(2005, 2025) # 2005-2024 inclusive
# BLS CPI-U
BLS_CPI_URL = "https://download.bls.gov/pub/time.series/cu/cu.data.0.Current"
BLS_DATA_DIR = DATA_DIR / "bls"
# IRS 990
UD_EINS = [516000297, 516017306] # UD + UD Research Foundation
IRS990_BASE_URL = "https://apps.irs.gov/pub/epostcard/990/xml"

View file

@ -0,0 +1,53 @@
"""Dash application factory."""
import dash
from dash import dcc, html, Input, Output
from admin_analytics.db.connection import get_connection
from admin_analytics.db.schema import ensure_schema
from admin_analytics.dashboard.pages import overview, compensation, staffing, headcount
def create_app() -> dash.Dash:
"""Create and configure the Dash application."""
app = dash.Dash(__name__, suppress_callback_exceptions=True)
conn = get_connection()
ensure_schema(conn)
app.layout = html.Div(
[
html.H1(
"University of Delaware — Administrative Analytics",
style={"textAlign": "center", "padding": "20px", "color": "#00539F"},
),
dcc.Tabs(
id="tabs",
value="overview",
children=[
dcc.Tab(label="Admin Cost Overview", value="overview"),
dcc.Tab(label="Executive Compensation", value="compensation"),
dcc.Tab(label="Staffing & Enrollment", value="staffing"),
dcc.Tab(label="Current Headcount", value="headcount"),
],
style={"marginBottom": "20px"},
),
html.Div(id="tab-content", style={"padding": "0 20px 20px 20px"}),
],
style={"fontFamily": "system-ui, -apple-system, sans-serif", "maxWidth": "1400px", "margin": "0 auto"},
)
@app.callback(Output("tab-content", "children"), Input("tabs", "value"))
def render_tab(tab: str):
if tab == "overview":
return overview.layout(conn)
elif tab == "compensation":
return compensation.layout(conn)
elif tab == "staffing":
return staffing.layout(conn)
elif tab == "headcount":
return headcount.layout(conn)
return html.Div("Unknown tab")
compensation.register_callbacks(app, conn)
return app

View file

@ -0,0 +1,162 @@
"""Page 2: Executive Compensation."""
import dash
import duckdb
from dash import html, dcc, Input, Output, dash_table
import plotly.express as px
import plotly.graph_objects as go
from admin_analytics.dashboard.queries import (
query_top_earners,
query_comp_by_role,
query_comp_vs_cpi,
)
_NO_DATA = html.Div(
"No IRS 990 data loaded. Run: admin-analytics ingest irs990",
style={"textAlign": "center", "padding": "40px", "color": "#888"},
)
# Roles to highlight in trend chart
_KEY_ROLES = ["PRESIDENT", "PROVOST", "VP_FINANCE", "VP_RESEARCH", "VP_ADVANCEMENT", "CFO"]
def layout(conn: duckdb.DuckDBPyConnection):
all_earners = query_top_earners(conn)
if all_earners.height == 0:
return _NO_DATA
years = sorted(all_earners["tax_year"].unique().to_list())
year_options = [{"label": "All Years", "value": "all"}] + [
{"label": str(y), "value": y} for y in years
]
# Compensation by role trend
role_df = query_comp_by_role(conn)
role_fig = go.Figure()
if role_df.height > 0:
role_pd = role_df.to_pandas()
for role in _KEY_ROLES:
subset = role_pd[role_pd["canonical_role"] == role]
if len(subset) > 0:
role_fig.add_trace(go.Scatter(
x=subset["tax_year"],
y=subset["total_compensation"],
mode="lines+markers",
name=role.replace("_", " ").title(),
))
role_fig.update_layout(
title="Compensation Trends by Role",
xaxis_title="Tax Year", yaxis_title="Total Compensation ($)",
template="plotly_white", height=420,
)
# Comp vs CPI indexed
cpi_df = query_comp_vs_cpi(conn)
cpi_fig = go.Figure()
if cpi_df.height > 0:
cpi_pd = cpi_df.to_pandas()
cpi_fig.add_trace(go.Scatter(
x=cpi_pd["year"], y=cpi_pd["comp_index"],
mode="lines+markers", name="Top Compensation",
line={"color": "#00539F"},
))
cpi_fig.add_trace(go.Scatter(
x=cpi_pd["year"], y=cpi_pd["cpi_index"],
mode="lines+markers", name="CPI-U",
line={"color": "#FFD200", "dash": "dash"},
))
cpi_fig.update_layout(
title="Top Compensation vs CPI-U (Indexed, Base Year = 100)",
xaxis_title="Year", yaxis_title="Index",
template="plotly_white", height=380,
)
return html.Div([
html.Div(
[
html.Label("Filter by Tax Year: ", style={"fontWeight": "bold"}),
dcc.Dropdown(
id="comp-year-dropdown",
options=year_options,
value="all",
style={"width": "200px", "display": "inline-block"},
),
],
style={"marginBottom": "16px"},
),
dash_table.DataTable(
id="comp-table",
columns=[
{"name": "Year", "id": "tax_year"},
{"name": "Name", "id": "person_name"},
{"name": "Title", "id": "title"},
{"name": "Role", "id": "canonical_role"},
{"name": "Base", "id": "base_compensation", "type": "numeric",
"format": dash_table.Format.Format().group(True)},
{"name": "Bonus", "id": "bonus_compensation", "type": "numeric",
"format": dash_table.Format.Format().group(True)},
{"name": "Total", "id": "total_compensation", "type": "numeric",
"format": dash_table.Format.Format().group(True)},
],
data=all_earners.to_pandas().to_dict("records"),
page_size=15,
sort_action="native",
filter_action="native",
style_table={"overflowX": "auto"},
style_cell={"textAlign": "left", "padding": "8px", "fontSize": "13px"},
style_header={"fontWeight": "bold", "backgroundColor": "#f0f0f0"},
),
html.Div(
[
html.Div(dcc.Graph(id="comp-breakdown-chart"), style={"flex": "1"}),
html.Div(dcc.Graph(figure=cpi_fig), style={"flex": "1"}),
],
style={"display": "flex", "gap": "16px", "marginTop": "16px"},
),
dcc.Graph(figure=role_fig),
])
def register_callbacks(app: dash.Dash, conn: duckdb.DuckDBPyConnection) -> None:
"""Register interactive callbacks for the compensation page."""
@app.callback(
[Output("comp-table", "data"), Output("comp-breakdown-chart", "figure")],
Input("comp-year-dropdown", "value"),
)
def update_compensation(year_value):
year = None if year_value == "all" else int(year_value)
earners = query_top_earners(conn, year=year)
# Table data
table_data = earners.to_pandas().to_dict("records") if earners.height > 0 else []
# Breakdown chart — stacked bar of comp components
breakdown_fig = go.Figure()
if earners.height > 0:
ep = earners.to_pandas().head(10) # top 10 by total comp
short_names = [n.split(",")[0][:20] if "," in n else n.split()[-1][:20]
for n in ep["person_name"]]
for comp_type, label, color in [
("base_compensation", "Base", "#00539F"),
("bonus_compensation", "Bonus", "#FFD200"),
("deferred_compensation", "Deferred", "#7FB069"),
("nontaxable_benefits", "Benefits", "#E07A5F"),
("other_compensation", "Other", "#999"),
]:
if comp_type in ep.columns:
breakdown_fig.add_trace(go.Bar(
x=short_names, y=ep[comp_type],
name=label, marker_color=color,
))
breakdown_fig.update_layout(barmode="stack")
title_suffix = f" ({year})" if year else " (All Years)"
breakdown_fig.update_layout(
title=f"Compensation Breakdown — Top 10{title_suffix}",
xaxis_title="", yaxis_title="$",
template="plotly_white", height=380,
)
return table_data, breakdown_fig

View file

@ -0,0 +1,118 @@
"""Page 4: Current Admin Headcount (from scraper)."""
import duckdb
from dash import html, dcc, dash_table
import plotly.express as px
import plotly.graph_objects as go
from admin_analytics.dashboard.queries import (
query_admin_headcount,
query_headcount_summary,
)
_NO_DATA = html.Div(
"No headcount data loaded. Run: admin-analytics ingest scrape",
style={"textAlign": "center", "padding": "40px", "color": "#888"},
)
def _kpi_card(title: str, value: str) -> html.Div:
return html.Div(
[
html.H4(title, style={"margin": "0", "color": "#666", "fontSize": "14px"}),
html.H2(value, style={"margin": "5px 0", "color": "#00539F"}),
],
style={
"flex": "1",
"padding": "20px",
"backgroundColor": "#f8f9fa",
"borderRadius": "8px",
"textAlign": "center",
"margin": "0 8px",
},
)
def layout(conn: duckdb.DuckDBPyConnection):
detail_df = query_admin_headcount(conn)
if detail_df.height == 0:
return _NO_DATA
summary_df = query_headcount_summary(conn)
detail_pd = detail_df.to_pandas()
summary_pd = summary_df.to_pandas()
total = len(detail_pd)
overhead_count = int(detail_pd["is_overhead"].sum()) if "is_overhead" in detail_pd.columns else 0
overhead_pct = round(overhead_count * 100 / total, 1) if total > 0 else 0
# KPI cards
kpi_row = html.Div(
[
_kpi_card("Total Staff Scraped", str(total)),
_kpi_card("Overhead Staff", str(overhead_count)),
_kpi_card("Overhead %", f"{overhead_pct}%"),
],
style={"display": "flex", "marginBottom": "24px"},
)
# Staff by unit bar chart
unit_counts = summary_pd.groupby("unit")["count"].sum().reset_index().sort_values("count")
unit_fig = px.bar(
unit_counts, x="count", y="unit", orientation="h",
title="Staff Count by Unit",
labels={"count": "Staff", "unit": ""},
color_discrete_sequence=["#00539F"],
)
unit_fig.update_layout(template="plotly_white", height=max(300, len(unit_counts) * 30 + 100))
# Overhead pie
oh_data = detail_pd["is_overhead"].value_counts()
oh_labels = {True: "Overhead", False: "Non-Overhead"}
pie_fig = px.pie(
names=[oh_labels.get(k, "Debatable") for k in oh_data.index],
values=oh_data.values,
title="Overhead vs Non-Overhead",
color_discrete_sequence=["#E07A5F", "#7FB069", "#999"],
)
pie_fig.update_layout(template="plotly_white", height=350)
# Category distribution per unit
cat_fig = px.bar(
summary_pd, x="count", y="unit", color="category", orientation="h",
title="Category Distribution by Unit",
labels={"count": "Staff", "unit": "", "category": "Category"},
)
cat_fig.update_layout(template="plotly_white", height=max(300, len(unit_counts) * 30 + 100))
# Detail table
table = dash_table.DataTable(
columns=[
{"name": "Unit", "id": "unit"},
{"name": "Name", "id": "person_name"},
{"name": "Title", "id": "title"},
{"name": "Category", "id": "category"},
{"name": "Overhead", "id": "is_overhead"},
],
data=detail_pd.to_dict("records"),
page_size=20,
sort_action="native",
filter_action="native",
style_table={"overflowX": "auto"},
style_cell={"textAlign": "left", "padding": "8px", "fontSize": "13px"},
style_header={"fontWeight": "bold", "backgroundColor": "#f0f0f0"},
)
return html.Div([
kpi_row,
html.Div(
[
html.Div(dcc.Graph(figure=unit_fig), style={"flex": "1"}),
html.Div(dcc.Graph(figure=pie_fig), style={"flex": "1"}),
],
style={"display": "flex", "gap": "16px"},
),
dcc.Graph(figure=cat_fig),
html.H3("Staff Directory Detail", style={"marginTop": "24px"}),
table,
])

View file

@ -0,0 +1,168 @@
"""Page 1: Administrative Cost Overview."""
import duckdb
from dash import html, dcc
import plotly.express as px
import plotly.graph_objects as go
from admin_analytics.dashboard.queries import (
query_admin_cost_ratio,
query_expense_breakdown,
query_admin_per_student,
query_admin_faculty_ratio,
)
_NO_DATA = html.Div(
"No data loaded. Run: admin-analytics ingest all",
style={"textAlign": "center", "padding": "40px", "color": "#888"},
)
def _kpi_card(title: str, value: str, subtitle: str = "") -> html.Div:
return html.Div(
[
html.H4(title, style={"margin": "0", "color": "#666", "fontSize": "14px"}),
html.H2(value, style={"margin": "5px 0", "color": "#00539F"}),
html.P(subtitle, style={"margin": "0", "color": "#999", "fontSize": "12px"}),
],
style={
"flex": "1",
"padding": "20px",
"backgroundColor": "#f8f9fa",
"borderRadius": "8px",
"textAlign": "center",
"margin": "0 8px",
},
)
def layout(conn: duckdb.DuckDBPyConnection):
# Admin cost ratio
ratio_df = query_admin_cost_ratio(conn)
if ratio_df.height == 0:
return _NO_DATA
ratio_pd = ratio_df.to_pandas()
latest = ratio_pd.iloc[-1]
# Admin per student
aps_df = query_admin_per_student(conn)
aps_pd = aps_df.to_pandas() if aps_df.height > 0 else None
latest_aps = aps_pd.iloc[-1] if aps_pd is not None else None
# Admin-to-faculty ratio
afr_df = query_admin_faculty_ratio(conn)
afr_pd = afr_df.to_pandas() if afr_df.height > 0 else None
latest_afr = afr_pd.iloc[-1] if afr_pd is not None else None
# KPI cards
kpi_row = html.Div(
[
_kpi_card(
"Admin Cost Ratio",
f"{latest['admin_cost_pct']:.1f}%",
f"Institutional Support / Total Expenses ({int(latest['year'])})",
),
_kpi_card(
"Admin Cost per Student",
f"${int(latest_aps['admin_per_student']):,}" if latest_aps is not None else "N/A",
f"CPI-adjusted: ${int(latest_aps['admin_per_student_cpi']):,}" if latest_aps is not None else "",
),
_kpi_card(
"Admin-to-Faculty Ratio",
f"{latest_afr['admin_faculty_ratio']:.2f}" if latest_afr is not None else "N/A",
f"Management / Faculty ({int(latest_afr['year'])})" if latest_afr is not None else "",
),
],
style={"display": "flex", "marginBottom": "24px"},
)
# Admin cost ratio trend
ratio_fig = go.Figure()
ratio_fig.add_trace(go.Scatter(
x=ratio_pd["year"], y=ratio_pd["admin_cost_pct"],
mode="lines+markers", name="Admin Cost %",
line={"color": "#00539F"},
))
ratio_fig.update_layout(
title="Administrative Cost Ratio Over Time",
xaxis_title="Year", yaxis_title="Institutional Support / Total Expenses (%)",
template="plotly_white", height=400,
)
# Expense breakdown stacked area
breakdown_df = query_expense_breakdown(conn)
if breakdown_df.height > 0:
bk_pd = breakdown_df.to_pandas()
expense_cols = [c for c in bk_pd.columns if c != "year"]
labels = {
"instruction_expenses": "Instruction",
"research_expenses": "Research",
"public_service_expenses": "Public Service",
"academic_support_expenses": "Academic Support",
"student_services_expenses": "Student Services",
"institutional_support_expenses": "Institutional Support",
"auxiliary_expenses": "Auxiliary",
"hospital_expenses": "Hospital",
"other_expenses": "Other",
}
breakdown_fig = go.Figure()
for col in expense_cols:
breakdown_fig.add_trace(go.Scatter(
x=bk_pd["year"], y=bk_pd[col] / 1e6,
mode="lines", name=labels.get(col, col),
stackgroup="one",
))
breakdown_fig.update_layout(
title="Expenses by Function (Millions $)",
xaxis_title="Year", yaxis_title="Millions $",
template="plotly_white", height=450,
)
else:
breakdown_fig = go.Figure()
# Admin per student trend
aps_fig = go.Figure()
if aps_pd is not None:
aps_fig.add_trace(go.Scatter(
x=aps_pd["year"], y=aps_pd["admin_per_student"],
mode="lines+markers", name="Nominal",
line={"color": "#00539F"},
))
aps_fig.add_trace(go.Scatter(
x=aps_pd["year"], y=aps_pd["admin_per_student_cpi"],
mode="lines+markers", name="CPI-Adjusted",
line={"color": "#FFD200", "dash": "dash"},
))
aps_fig.update_layout(
title="Admin Cost per Student",
xaxis_title="Year", yaxis_title="$ per Student",
template="plotly_white", height=380,
)
# Admin-to-faculty ratio trend
afr_fig = go.Figure()
if afr_pd is not None:
afr_fig.add_trace(go.Scatter(
x=afr_pd["year"], y=afr_pd["admin_faculty_ratio"],
mode="lines+markers", name="Ratio",
line={"color": "#00539F"},
))
afr_fig.update_layout(
title="Admin-to-Faculty Ratio",
xaxis_title="Year", yaxis_title="Management / Faculty",
template="plotly_white", height=380,
)
return html.Div([
kpi_row,
dcc.Graph(figure=ratio_fig),
dcc.Graph(figure=breakdown_fig),
html.Div(
[
html.Div(dcc.Graph(figure=aps_fig), style={"flex": "1"}),
html.Div(dcc.Graph(figure=afr_fig), style={"flex": "1"}),
],
style={"display": "flex", "gap": "16px"},
),
])

View file

@ -0,0 +1,97 @@
"""Page 3: Staffing & Enrollment."""
import duckdb
from dash import html, dcc
import plotly.graph_objects as go
from admin_analytics.dashboard.queries import (
query_staff_composition,
query_student_staff_ratios,
query_growth_index,
)
_NO_DATA = html.Div(
"No IPEDS staff data loaded. Run: admin-analytics ingest ipeds",
style={"textAlign": "center", "padding": "40px", "color": "#888"},
)
def layout(conn: duckdb.DuckDBPyConnection):
staff_df = query_staff_composition(conn)
if staff_df.height == 0:
return _NO_DATA
staff_pd = staff_df.to_pandas()
# Staff composition stacked area
comp_fig = go.Figure()
for col, label, color in [
("faculty_total", "Faculty", "#00539F"),
("management_total", "Management", "#E07A5F"),
("other_staff", "Other Staff", "#7FB069"),
]:
comp_fig.add_trace(go.Scatter(
x=staff_pd["year"], y=staff_pd[col],
mode="lines", name=label,
stackgroup="one",
line={"color": color},
))
comp_fig.update_layout(
title="Staff Composition Over Time",
xaxis_title="Year", yaxis_title="Headcount",
template="plotly_white", height=420,
)
# Student-to-staff ratios
ratio_df = query_student_staff_ratios(conn)
ratio_fig = go.Figure()
if ratio_df.height > 0:
ratio_pd = ratio_df.to_pandas()
ratio_fig.add_trace(go.Scatter(
x=ratio_pd["year"], y=ratio_pd["students_per_staff"],
mode="lines+markers", name="Students per Staff",
line={"color": "#00539F"},
))
ratio_fig.add_trace(go.Scatter(
x=ratio_pd["year"], y=ratio_pd["students_per_faculty"],
mode="lines+markers", name="Students per Faculty",
line={"color": "#FFD200"},
))
ratio_fig.update_layout(
title="Student-to-Staff Ratios",
xaxis_title="Year", yaxis_title="Ratio",
template="plotly_white", height=380,
)
# Growth index
growth_df = query_growth_index(conn)
growth_fig = go.Figure()
if growth_df.height > 0:
growth_pd = growth_df.to_pandas()
growth_fig.add_trace(go.Scatter(
x=growth_pd["year"], y=growth_pd["mgmt_index"],
mode="lines+markers", name="Management Growth",
line={"color": "#E07A5F"},
))
growth_fig.add_trace(go.Scatter(
x=growth_pd["year"], y=growth_pd["enrollment_index"],
mode="lines+markers", name="Enrollment Growth",
line={"color": "#00539F"},
))
growth_fig.add_hline(y=100, line_dash="dot", line_color="#ccc")
growth_fig.update_layout(
title="Management vs Enrollment Growth (Indexed, Base Year = 100)",
xaxis_title="Year", yaxis_title="Index",
template="plotly_white", height=380,
)
return html.Div([
dcc.Graph(figure=comp_fig),
html.Div(
[
html.Div(dcc.Graph(figure=ratio_fig), style={"flex": "1"}),
html.Div(dcc.Graph(figure=growth_fig), style={"flex": "1"}),
],
style={"display": "flex", "gap": "16px"},
),
])

View file

@ -0,0 +1,263 @@
"""Dashboard query layer — all DuckDB queries returning polars DataFrames."""
from typing import Any
import duckdb
import polars as pl
from admin_analytics.config import UD_UNITID
from admin_analytics.irs990.titles import normalize_title
# Shared CTE for CPI adjustment
_CPI_CTE = """
WITH annual_cpi AS (
SELECT year, AVG(value) AS avg_cpi
FROM raw_cpi_u
GROUP BY year
),
latest_cpi AS (
SELECT avg_cpi FROM annual_cpi
WHERE year = (SELECT MAX(year) FROM annual_cpi)
)
"""
def query_admin_cost_ratio(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""Admin cost ratio trend with CPI-adjusted values."""
return conn.execute(f"""
{_CPI_CTE}
SELECT
f.year,
f.institutional_support_expenses,
f.total_expenses,
ROUND(f.institutional_support_expenses * 100.0
/ NULLIF(f.total_expenses, 0), 2) AS admin_cost_pct,
ROUND(f.institutional_support_expenses
* (SELECT avg_cpi FROM latest_cpi) / ac.avg_cpi, 0)
AS inst_support_cpi_adjusted,
ROUND(f.total_expenses
* (SELECT avg_cpi FROM latest_cpi) / ac.avg_cpi, 0)
AS total_expenses_cpi_adjusted
FROM raw_ipeds_finance f
LEFT JOIN annual_cpi ac ON ac.year = f.year
WHERE f.unitid = ?
ORDER BY f.year
""", [UD_UNITID]).pl()
def query_expense_breakdown(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""Expense breakdown by function over time."""
return conn.execute("""
SELECT year,
instruction_expenses, research_expenses, public_service_expenses,
academic_support_expenses, student_services_expenses,
institutional_support_expenses, auxiliary_expenses,
hospital_expenses, other_expenses
FROM raw_ipeds_finance
WHERE unitid = ?
ORDER BY year
""", [UD_UNITID]).pl()
def query_admin_per_student(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""Admin cost per student (nominal and CPI-adjusted)."""
return conn.execute(f"""
{_CPI_CTE}
SELECT
f.year,
f.institutional_support_expenses,
e.total_enrollment,
ROUND(f.institutional_support_expenses * 1.0
/ NULLIF(e.total_enrollment, 0), 0) AS admin_per_student,
ROUND(
(f.institutional_support_expenses
* (SELECT avg_cpi FROM latest_cpi) / ac.avg_cpi)
/ NULLIF(e.total_enrollment, 0), 0
) AS admin_per_student_cpi
FROM raw_ipeds_finance f
JOIN raw_ipeds_enrollment e ON e.unitid = f.unitid AND e.year = f.year
LEFT JOIN annual_cpi ac ON ac.year = f.year
WHERE f.unitid = ?
ORDER BY f.year
""", [UD_UNITID]).pl()
def query_admin_faculty_ratio(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""Admin-to-faculty ratio over time."""
return conn.execute("""
SELECT year,
management_total,
faculty_total,
ROUND(management_total * 1.0 / NULLIF(faculty_total, 0), 3)
AS admin_faculty_ratio
FROM raw_ipeds_staff
WHERE unitid = ?
ORDER BY year
""", [UD_UNITID]).pl()
def query_top_earners(
conn: duckdb.DuckDBPyConnection, year: int | None = None
) -> pl.DataFrame:
"""Top earners from Schedule J, optionally filtered by year."""
where = "WHERE j.total_compensation > 0"
params: list[Any] = []
if year is not None:
where += " AND j.tax_year = ?"
params.append(year)
df = conn.execute(f"""
SELECT
j.tax_year,
j.person_name,
j.title,
j.base_compensation,
j.bonus_compensation,
j.other_compensation,
j.deferred_compensation,
j.nontaxable_benefits,
j.total_compensation,
f.organization_name
FROM raw_990_schedule_j j
JOIN raw_990_filing f ON f.object_id = j.object_id
{where}
ORDER BY j.tax_year DESC, j.total_compensation DESC
""", params).pl()
if df.height > 0:
df = df.with_columns(
pl.col("title").map_elements(
normalize_title, return_dtype=pl.Utf8
).alias("canonical_role")
)
return df
def query_comp_by_role(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""Compensation trends by canonical role across years."""
df = conn.execute("""
SELECT j.tax_year, j.person_name, j.title, j.total_compensation
FROM raw_990_schedule_j j
JOIN raw_990_filing f ON f.object_id = j.object_id
WHERE j.total_compensation > 0
ORDER BY j.tax_year, j.total_compensation DESC
""").pl()
if df.height == 0:
return df
df = df.with_columns(
pl.col("title").map_elements(
normalize_title, return_dtype=pl.Utf8
).alias("canonical_role")
)
# Keep highest-paid person per role per year
return (
df.sort("total_compensation", descending=True)
.group_by(["tax_year", "canonical_role"])
.first()
.sort(["tax_year", "canonical_role"])
)
def query_comp_vs_cpi(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""Compensation growth vs CPI growth, indexed to first available year = 100."""
return conn.execute("""
WITH yearly_max_comp AS (
SELECT tax_year, MAX(total_compensation) AS top_comp
FROM raw_990_schedule_j
GROUP BY tax_year
),
annual_cpi AS (
SELECT year, AVG(value) AS avg_cpi
FROM raw_cpi_u GROUP BY year
),
base AS (
SELECT c.top_comp AS base_comp, ac.avg_cpi AS base_cpi
FROM yearly_max_comp c
JOIN annual_cpi ac ON ac.year = c.tax_year
ORDER BY c.tax_year LIMIT 1
)
SELECT
c.tax_year AS year,
c.top_comp,
ac.avg_cpi,
ROUND(c.top_comp * 100.0 / NULLIF((SELECT base_comp FROM base), 0), 1)
AS comp_index,
ROUND(ac.avg_cpi * 100.0 / NULLIF((SELECT base_cpi FROM base), 0), 1)
AS cpi_index
FROM yearly_max_comp c
JOIN annual_cpi ac ON ac.year = c.tax_year
ORDER BY year
""").pl()
def query_staff_composition(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""Staff composition over time."""
return conn.execute("""
SELECT year, total_staff, faculty_total, management_total,
total_staff - COALESCE(faculty_total, 0) - COALESCE(management_total, 0)
AS other_staff
FROM raw_ipeds_staff
WHERE unitid = ?
ORDER BY year
""", [UD_UNITID]).pl()
def query_student_staff_ratios(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""Student-to-staff and student-to-faculty ratios."""
return conn.execute("""
SELECT s.year, e.total_enrollment, s.total_staff, s.faculty_total,
ROUND(e.total_enrollment * 1.0 / NULLIF(s.total_staff, 0), 1)
AS students_per_staff,
ROUND(e.total_enrollment * 1.0 / NULLIF(s.faculty_total, 0), 1)
AS students_per_faculty
FROM raw_ipeds_staff s
JOIN raw_ipeds_enrollment e ON e.unitid = s.unitid AND e.year = s.year
WHERE s.unitid = ?
ORDER BY s.year
""", [UD_UNITID]).pl()
def query_growth_index(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""Management vs enrollment growth, indexed to first year = 100."""
return conn.execute("""
WITH base AS (
SELECT s.management_total AS base_mgmt, e.total_enrollment AS base_enrl
FROM raw_ipeds_staff s
JOIN raw_ipeds_enrollment e ON e.unitid = s.unitid AND e.year = s.year
WHERE s.unitid = ?
ORDER BY s.year LIMIT 1
)
SELECT s.year,
s.management_total,
e.total_enrollment,
ROUND(s.management_total * 100.0
/ NULLIF((SELECT base_mgmt FROM base), 0), 1) AS mgmt_index,
ROUND(e.total_enrollment * 100.0
/ NULLIF((SELECT base_enrl FROM base), 0), 1) AS enrollment_index
FROM raw_ipeds_staff s
JOIN raw_ipeds_enrollment e ON e.unitid = s.unitid AND e.year = s.year
WHERE s.unitid = ?
ORDER BY s.year
""", [UD_UNITID, UD_UNITID]).pl()
def query_admin_headcount(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""All scraped admin headcount entries."""
return conn.execute("""
SELECT unit, person_name, title, category, is_overhead, scrape_date
FROM raw_admin_headcount
ORDER BY unit, category, person_name
""").pl()
def query_headcount_summary(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""Headcount summary by unit and category."""
return conn.execute("""
SELECT unit, category, is_overhead, COUNT(*) AS count
FROM raw_admin_headcount
GROUP BY unit, category, is_overhead
ORDER BY unit, count DESC
""").pl()

View file

@ -109,6 +109,19 @@ TABLES = {
PRIMARY KEY (year, month)
)
""",
"raw_admin_headcount": """
CREATE SEQUENCE IF NOT EXISTS seq_admin_headcount START 1;
CREATE TABLE IF NOT EXISTS raw_admin_headcount (
id INTEGER PRIMARY KEY DEFAULT nextval('seq_admin_headcount'),
scrape_date DATE NOT NULL,
unit VARCHAR NOT NULL,
person_name VARCHAR,
title VARCHAR,
email VARCHAR,
category VARCHAR,
is_overhead BOOLEAN
)
""",
}

View file

@ -2,6 +2,7 @@
import io
import re
import subprocess
import tempfile
import zipfile
from pathlib import Path
@ -57,11 +58,14 @@ def filter_index(
def get_batch_list(year: int) -> list[str]:
"""Get the list of available TEOS XML batch ZIP IDs for a year.
Tries common naming patterns: {YEAR}_TEOS_XML_{MM}{L} where
MM is 01-12 and L is A-D.
Two naming conventions exist:
- 2021+: {YEAR}_TEOS_XML_{MM}{L}.zip (e.g. 2023_TEOS_XML_05A)
- 2017-2020: download990xml_{YEAR}_{N}.zip (e.g. download990xml_2020_1)
Also checks for _CT1 (correction/catchup) batches.
"""
batches = []
with httpx.Client(follow_redirects=True, timeout=30.0) as client:
# Modern pattern (2021+)
for month in range(1, 13):
for letter in "ABCD":
batch_id = f"{year}_TEOS_XML_{month:02d}{letter}"
@ -72,9 +76,61 @@ def get_batch_list(year: int) -> list[str]:
batches.append(batch_id)
except httpx.HTTPError:
continue
# Legacy pattern (2017-2020)
for n in range(1, 20):
batch_id = f"download990xml_{year}_{n}"
url = f"{config.IRS990_BASE_URL}/{year}/{batch_id}.zip"
try:
resp = client.head(url)
if resp.status_code == 200:
batches.append(batch_id)
else:
break # sequential numbering, stop at first miss
except httpx.HTTPError:
break
# Correction/catchup batch
ct_id = f"{year}_TEOS_XML_CT1"
url = f"{config.IRS990_BASE_URL}/{year}/{ct_id}.zip"
try:
resp = client.head(url)
if resp.status_code == 200:
batches.append(ct_id)
except httpx.HTTPError:
pass
return batches
def _extract_with_system_unzip(
zip_path: Path, target_name: str, dest: Path
) -> bool:
"""Extract a file from a ZIP using the system 7z or unzip command."""
dest.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory() as tmp_dir:
# Try 7z first (handles more compression methods), then unzip
for cmd in [
["7z", "x", str(zip_path), f"-o{tmp_dir}", f"*{target_name}", "-y"],
["unzip", "-o", "-j", str(zip_path), f"*{target_name}", "-d", tmp_dir],
]:
try:
result = subprocess.run(
cmd, capture_output=True, timeout=300
)
if result.returncode == 0:
extracted = list(Path(tmp_dir).rglob(target_name))
if extracted:
import shutil
shutil.move(str(extracted[0]), str(dest))
return True
except FileNotFoundError:
continue
return False
def _extract_xml_from_zip(
zip_url: str, object_id: str, dest: Path
) -> bool:
@ -96,20 +152,27 @@ def _extract_xml_from_zip(
for chunk in resp.iter_bytes(chunk_size=1024 * 1024):
f.write(chunk)
with zipfile.ZipFile(tmp_path) as zf:
names = zf.namelist()
# Look for our target file (case-insensitive)
match = None
for name in names:
if name.lower() == target_name.lower():
match = name
break
if match is None:
return False
dest.parent.mkdir(parents=True, exist_ok=True)
with zf.open(match) as src, open(dest, "wb") as dst:
dst.write(src.read())
return True
try:
with zipfile.ZipFile(tmp_path) as zf:
names = zf.namelist()
# Look for our target file — may be in a subdirectory within the ZIP
# e.g. "2024_TEOS_XML_04A/202421029349301032_public.xml"
match = None
for name in names:
basename = name.rsplit("/", 1)[-1] if "/" in name else name
if basename.lower() == target_name.lower():
match = name
break
if match is None:
return False
dest.parent.mkdir(parents=True, exist_ok=True)
with zf.open(match) as src, open(dest, "wb") as dst:
dst.write(src.read())
return True
except NotImplementedError:
# Unsupported compression (e.g. Zstandard in newer ZIPs).
# Fall back to system unzip.
return _extract_with_system_unzip(tmp_path, target_name, dest)
finally:
tmp_path.unlink(missing_ok=True)
@ -136,8 +199,9 @@ def download_filing_xml(
return dest
if batch_id:
# We know which ZIP to look in
url = f"{config.IRS990_BASE_URL}/{year}/{batch_id}.zip"
# We know which ZIP to look in. Index may have lowercase but URLs need uppercase.
batch_id_upper = batch_id.strip().upper()
url = f"{config.IRS990_BASE_URL}/{year}/{batch_id_upper}.zip"
if _extract_xml_from_zip(url, object_id, dest):
return dest
return None

View file

@ -0,0 +1,121 @@
"""Classify staff by title into functional categories.
Categories distinguish admin overhead from grant-funded, student-facing,
and technical roles. This is critical because IPEDS/IRE "staff" counts
lump everyone who isn't tenure-track faculty — including postdocs,
research scientists, and lab technicians who are soft-funded through
extramural research and are NOT administrative overhead.
"""
import re
# Order matters — first match wins.
CATEGORY_PATTERNS: list[tuple[str, re.Pattern]] = [
# Leadership / structural overhead
("LEADERSHIP", re.compile(
r"\b(dean|chief of staff|associate dean|assistant dean)\b", re.I)),
# Faculty listed on staff page — not admin (must come before RESEARCH
# to catch "Adjunct Professor NIST" as faculty, not research)
("FACULTY", re.compile(
r"\b(professor|lecturer|instructor|faculty|adjunct|affiliated)", re.I)),
# Grants administration — debatable; supports extramural funding
("GRANTS_ADMIN", re.compile(
r"\b(grants?\s+anal|pre-?award|post-?award|closeout\s+coord"
r"|sponsored\s+program|grants?\s+admin|grants?\s+manag|grants?\s+coord)", re.I)),
# Research staff — soft-funded, NOT admin bloat
("RESEARCH", re.compile(
r"\b(research\s+(?:associate|assistant|scientist|scholar|fellow)"
r"|postdoc|post-?doctoral|(?:associate\s+)?scientist\b)", re.I)),
# Academic / student-facing support
("ACADEMIC_SUPPORT", re.compile(
r"\b(academic\s+(?:advisor|analyst|program)|undergrad\w*\s+(?:recruit|advisor|affairs)"
r"|graduate\s+(?:services|advisor)|student\s+(?:develop|support|services)"
r"|program\s+(?:coordinator|manager))", re.I)),
# Advancement / development — revenue-generating (fundraising)
("ADVANCEMENT", re.compile(
r"\b(development|fundrais|advancement|alumni\s+relation|donor|giving)", re.I)),
# Finance / procurement
("FINANCE", re.compile(
r"\b(financial|fiscal|budget|procurement|business\s+(?:officer|admin)"
r"|sr\.?\s+business)", re.I)),
# IT / computing
("IT", re.compile(
r"\b(computing|systems?\s+(?:prog|admin)|it\s+|information\s+tech"
r"|support\s+specialist|service\s+desk|digital\s+tech)", re.I)),
# Communications / marketing (must come before DIRECTOR)
("COMMUNICATIONS", re.compile(
r"\b(communicat\w+|marketing|media\s+(?:specialist|coord|director)"
r"|web\s+(?:develop|design|content)|event\s+(?:coord|plan|manag))", re.I)),
# Human resources
("HR", re.compile(
r"\b(human\s+resource|hr\s+analyst|talent|workforce)", re.I)),
# Facilities / space management
("FACILITIES", re.compile(
r"\b(facilit|building|space\s+(?:plan|manag)|safety|engineer\w+\s+facilit)", re.I)),
# Technical / lab operations — not admin bloat
("TECHNICAL", re.compile(
r"\b(machinist|lab\s+(?:manager|coord|tech)|equipment|technician"
r"|instrument)", re.I)),
# Administrative support
("ADMIN_SUPPORT", re.compile(
r"\b(admin\w*\s+(?:assistant|specialist|support|secretary|coord)"
r"|secretary|receptionist|office\s+(?:manager|coord))", re.I)),
# Director-level (catch remaining directors)
("DIRECTOR", re.compile(
r"\b(director|associate\s+director|sr\.?\s+director)\b", re.I)),
]
# Which categories count as administrative overhead
OVERHEAD_CATEGORIES = {
"LEADERSHIP", "FINANCE", "IT", "COMMUNICATIONS", "HR",
"FACILITIES", "ADMIN_SUPPORT", "DIRECTOR",
}
# Debatable — could go either way depending on analysis
DEBATABLE_CATEGORIES = {"GRANTS_ADMIN"}
# NOT overhead — these are mission-aligned or revenue-generating
NON_OVERHEAD_CATEGORIES = {
"RESEARCH", "ACADEMIC_SUPPORT", "ADVANCEMENT", "TECHNICAL", "FACULTY",
}
def classify_title(title: str | None) -> str:
"""Classify a staff title into a functional category.
Returns the category string, or "UNKNOWN" if no pattern matches.
"""
if not title or not title.strip():
return "UNKNOWN"
for category, pattern in CATEGORY_PATTERNS:
if pattern.search(title):
return category
return "UNKNOWN"
def is_overhead(category: str) -> bool | None:
"""Return True if the category is administrative overhead,
False if not, None if debatable.
"""
if category in OVERHEAD_CATEGORIES:
return True
if category in NON_OVERHEAD_CATEGORIES:
return False
if category in DEBATABLE_CATEGORIES:
return None
return None

View file

@ -0,0 +1,292 @@
"""Scrape UD staff directory pages for headcounts and title classification.
Two rendering modes:
- Static (httpx): for pages that embed staff data in HTML source
- Dynamic (Playwright): for JS-rendered pages (Divi/WordPress AJAX)
The scraper targets the Engineering management line:
Department staff → COE Central → Provost → President
"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from datetime import date
from pathlib import Path
import httpx
from admin_analytics.scraper.classify import classify_title
@dataclass
class StaffEntry:
name: str
title: str | None
email: str | None
category: str = ""
unit: str = ""
def __post_init__(self):
if not self.category and self.title:
self.category = classify_title(self.title)
@dataclass
class DirectoryPage:
"""Configuration for a single directory page to scrape."""
unit: str # e.g. "COE Central", "ME", "Provost Office"
url: str
requires_js: bool = False # True = needs Playwright
# UD College of Engineering management line + departments
DIRECTORY_PAGES = [
# COE Central admin (JS-rendered)
DirectoryPage("COE Central", "https://engr.udel.edu/our-people/directory/", requires_js=True),
# Engineering departments
DirectoryPage("ME", "https://me.udel.edu/people/staff/", requires_js=True),
DirectoryPage("CBE", "https://cbe.udel.edu/people/staff/", requires_js=False),
DirectoryPage("ECE", "https://www.ece.udel.edu/people/staff/", requires_js=False),
DirectoryPage("CIS", "https://www.cis.udel.edu/people/staff/", requires_js=False),
DirectoryPage("BME", "https://bme.udel.edu/people/staff/", requires_js=True),
DirectoryPage("CCEE", "https://www.ccee.udel.edu/people/staff/", requires_js=False),
DirectoryPage("MSE", "https://mseg.udel.edu/people/staff/", requires_js=True),
# Management line above Engineering
DirectoryPage("Provost Office", "https://provost.udel.edu/about/our-staff/", requires_js=False),
DirectoryPage("President Office", "https://www.udel.edu/about/leadership/", requires_js=False),
]
def _extract_from_rendered_text(text: str, unit: str) -> list[StaffEntry]:
"""Extract staff entries from rendered page text.
The JS-rendered Divi pages display staff as:
Name
Title
[Phone]
email@udel.edu
We anchor on email addresses and look backwards for name/title.
"""
lines = [line.strip() for line in text.split("\n") if line.strip()]
entries: list[StaffEntry] = []
seen_emails: set[str] = set()
for i, line in enumerate(lines):
emails = re.findall(r"[\w.-]+@udel\.edu", line)
for email in emails:
if email in seen_emails:
continue
# Skip generic/departmental emails
if any(k in email for k in ["-info", "dept", "help", "college", "engr@", "coe-"]):
continue
seen_emails.add(email)
# Look backwards for name and title
name, title = _find_name_title(lines, i)
if name:
entries.append(StaffEntry(
name=name, title=title, email=email, unit=unit,
))
return entries
def _find_name_title(lines: list[str], email_idx: int) -> tuple[str | None, str | None]:
"""Look backwards from an email line to find the person's name and title."""
name = None
title = None
# Title keywords that distinguish a title line from a name line
title_keywords = re.compile(
r"(director|manager|specialist|advisor|analyst|coordinator|assistant|associate"
r"|officer|chief|dean|admin|support|computing|systems|financial|grants?"
r"|machinist|lab |academic|program|human|facilit|communicat|development"
r"|procurement|sr\.|senior|recruit|budget|research|scientist|engineer"
r"|technician|professor|lecturer)", re.I,
)
for back in range(1, 6):
idx = email_idx - back
if idx < 0:
break
prev = lines[idx].strip()
# Stop if we hit another email (previous person's entry)
if "@udel.edu" in prev:
break
# Skip phone numbers
if re.match(r"^[\d\-\(\)\s]{7,}$", prev):
continue
# Is this a title or a name?
if title is None and title_keywords.search(prev):
title = prev
elif name is None and re.match(r"^[A-Z][a-z]", prev) and len(prev) < 50:
# Looks like a name (starts with capital, not too long)
if not title_keywords.search(prev):
name = prev
elif title is None:
# It has title keywords — it's actually a title
title = prev
else:
name = prev
if name and title:
break
return name, title
def _extract_from_html(html_text: str, unit: str) -> list[StaffEntry]:
"""Extract staff from static HTML source using email anchoring."""
# Strip HTML tags to get plain text
text = re.sub(r"<[^>]+>", " ", html_text)
text = re.sub(r"&\w+;", " ", text)
text = re.sub(r"\s+", " ", text).strip()
entries: list[StaffEntry] = []
seen_emails: set[str] = set()
for m in re.finditer(r"[\w.-]+@udel\.edu", text):
email = m.group()
if email in seen_emails:
continue
if any(k in email for k in ["-info", "dept", "help", "college", "engr@", "coe-"]):
continue
seen_emails.add(email)
# Get context before the email
start = max(0, m.start() - 300)
ctx = text[start:m.start()]
# Try to extract name + title from context
name, title = _parse_context_for_name_title(ctx)
if name:
entries.append(StaffEntry(
name=name, title=title, email=email, unit=unit,
))
return entries
def _parse_context_for_name_title(ctx: str) -> tuple[str | None, str | None]:
"""Parse a text context block (before an email) for name and title."""
# Pattern: Name Title [Location] [Phone]
m = re.search(
r"([A-Z][a-z]+(?:\s[A-Z]\.?)?\s(?:Mc|De|Van|La)?[A-Z][a-z]+(?:-[A-Z][a-z]+)?)\s+"
r"([A-Za-z/,.& ]+?(?:I{1,3}|IV|V)?)\s+"
r"(?:\d+\s+\w|\d{3}-)",
ctx,
)
if m:
return m.group(1).strip(), m.group(2).strip()
# Fallback: just find the last proper name in the context
names = re.findall(
r"([A-Z][a-z]+(?:\s[A-Z]\.?)?\s(?:Mc|De|Van|La)?[A-Z][a-z]+(?:-[A-Z][a-z]+)?)",
ctx,
)
if names:
return names[-1], None
return None, None
def _extract_provost_staff(html_text: str) -> list[StaffEntry]:
"""Custom parser for the Provost office page which has a distinct structure."""
from lxml import html as lxml_html
tree = lxml_html.fromstring(html_text)
entries: list[StaffEntry] = []
# Leadership: h2 tags contain titles, strong tags contain names
h2s = tree.xpath("//h2")
strongs = tree.xpath("//strong")
strong_texts = [s.text_content().strip() for s in strongs if s.text_content().strip()]
for h2 in h2s:
title = h2.text_content().strip()
if not title or title in ("Administrative Staff", "Staff", "Quick Links", "Recent News"):
continue
# The name follows the h2 in a nearby strong tag or paragraph
# Match by position — h2 title, then the next unmatched strong is the name
if strong_texts:
name = strong_texts.pop(0)
entries.append(StaffEntry(
name=name, title=title, email=None, unit="Provost Office",
))
# Administrative staff section: "Title, Name" format in the remaining strongs
for st in strong_texts:
if "," in st:
parts = st.split(",", 1)
title = parts[0].strip()
name = parts[1].strip()
entries.append(StaffEntry(
name=name, title=title, email=None, unit="Provost Office",
))
return entries
def scrape_static(page: DirectoryPage) -> list[StaffEntry]:
"""Scrape a static HTML page using httpx."""
with httpx.Client(
follow_redirects=True,
timeout=30.0,
headers={"User-Agent": "Mozilla/5.0 (compatible; UDAdminAnalytics/0.1)"},
) as client:
resp = client.get(page.url)
resp.raise_for_status()
if page.unit == "Provost Office":
return _extract_provost_staff(resp.text)
return _extract_from_html(resp.text, page.unit)
def scrape_dynamic(page: DirectoryPage) -> list[StaffEntry]:
"""Scrape a JS-rendered page using Playwright."""
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
pw_page = browser.new_page()
pw_page.goto(page.url, wait_until="networkidle", timeout=45000)
text = pw_page.inner_text("body")
browser.close()
return _extract_from_rendered_text(text, page.unit)
def scrape_page(page: DirectoryPage) -> list[StaffEntry]:
"""Scrape a single directory page, choosing the right method."""
if page.requires_js:
return scrape_dynamic(page)
return scrape_static(page)
def scrape_all(pages: list[DirectoryPage] | None = None) -> list[StaffEntry]:
"""Scrape all configured directory pages.
Returns a flat list of all staff entries across all units.
"""
pages = pages or DIRECTORY_PAGES
all_entries: list[StaffEntry] = []
for page in pages:
method = "Playwright" if page.requires_js else "httpx"
print(f" Scraping {page.unit} ({method})...")
try:
entries = scrape_page(page)
print(f" Found {len(entries)} staff")
all_entries.extend(entries)
except Exception as e:
print(f" Error: {e}")
return all_entries

View file

@ -0,0 +1,49 @@
"""Load scraped staff directory data into DuckDB."""
from datetime import date
import duckdb
from admin_analytics.scraper.classify import is_overhead
from admin_analytics.scraper.directory import StaffEntry
def load_scrape(
conn: duckdb.DuckDBPyConnection,
entries: list[StaffEntry],
scrape_date: date | None = None,
) -> int:
"""Load scraped staff entries into raw_admin_headcount.
Clears previous data for the same scrape_date before inserting,
making re-runs idempotent.
Returns number of rows inserted.
"""
scrape_date = scrape_date or date.today()
conn.execute(
"DELETE FROM raw_admin_headcount WHERE scrape_date = ?",
[scrape_date],
)
count = 0
for entry in entries:
overhead = is_overhead(entry.category)
conn.execute(
"""INSERT INTO raw_admin_headcount
(scrape_date, unit, person_name, title, email, category, is_overhead)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
[
scrape_date,
entry.unit,
entry.name,
entry.title,
entry.email,
entry.category,
overhead,
],
)
count += 1
return count

View file

@ -0,0 +1,168 @@
"""Data validation queries for the raw data layer."""
from typing import Any
import duckdb
from admin_analytics.config import DEFAULT_YEAR_RANGE
from admin_analytics.db.schema import TABLES
# Key columns to check for NULLs per table
KEY_COLUMNS: dict[str, list[str]] = {
"raw_institution": ["unitid", "year", "institution_name"],
"raw_ipeds_finance": ["unitid", "year", "total_expenses"],
"raw_ipeds_staff": ["unitid", "year", "total_staff"],
"raw_ipeds_enrollment": ["unitid", "year", "total_enrollment"],
"raw_990_filing": ["ein", "tax_year", "total_revenue", "total_expenses"],
"raw_990_part_vii": ["ein", "tax_year", "person_name", "reportable_comp_from_org"],
"raw_990_schedule_j": ["ein", "tax_year", "person_name", "total_compensation"],
"raw_cpi_u": ["year", "month", "value"],
"raw_admin_headcount": ["unit", "person_name", "category"],
}
# Year column name per table (some use tax_year, some have no year)
YEAR_COLUMN: dict[str, str] = {
"raw_institution": "year",
"raw_ipeds_finance": "year",
"raw_ipeds_staff": "year",
"raw_ipeds_enrollment": "year",
"raw_990_filing": "tax_year",
"raw_990_part_vii": "tax_year",
"raw_990_schedule_j": "tax_year",
"raw_cpi_u": "year",
}
def validate_row_counts(conn: duckdb.DuckDBPyConnection) -> dict[str, int]:
"""Return row count per raw table."""
counts = {}
for table in TABLES:
row = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()
counts[table] = row[0]
return counts
def validate_null_rates(conn: duckdb.DuckDBPyConnection) -> dict[str, dict[str, float]]:
"""Return NULL percentage for key columns per table.
Only tables with rows are included. Returns {table: {column: pct}}.
"""
results: dict[str, dict[str, float]] = {}
for table, columns in KEY_COLUMNS.items():
total = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
if total == 0:
continue
rates: dict[str, float] = {}
for col in columns:
null_count = conn.execute(
f"SELECT COUNT(*) FROM {table} WHERE {col} IS NULL"
).fetchone()[0]
rates[col] = round(null_count * 100.0 / total, 1)
results[table] = rates
return results
def validate_year_coverage(
conn: duckdb.DuckDBPyConnection,
) -> dict[str, dict[str, Any]]:
"""Return year coverage info per table.
Returns {table: {"years": [...], "gaps": [...]}}.
"""
results: dict[str, dict[str, Any]] = {}
expected = set(DEFAULT_YEAR_RANGE)
for table, col in YEAR_COLUMN.items():
rows = conn.execute(
f"SELECT DISTINCT {col} FROM {table} ORDER BY {col}"
).fetchall()
years = [r[0] for r in rows]
if not years:
results[table] = {"years": [], "gaps": sorted(expected)}
continue
present = set(years)
gaps = sorted(expected - present)
results[table] = {"years": years, "gaps": gaps}
return results
def validate_cross_source_consistency(
conn: duckdb.DuckDBPyConnection,
) -> dict[str, Any]:
"""Check year overlap across IPEDS, IRS 990, and CPI sources."""
sources = {
"ipeds_finance": ("raw_ipeds_finance", "year"),
"irs990": ("raw_990_filing", "tax_year"),
"cpi": ("raw_cpi_u", "year"),
}
year_sets: dict[str, set[int]] = {}
for key, (table, col) in sources.items():
rows = conn.execute(
f"SELECT DISTINCT {col} FROM {table}"
).fetchall()
year_sets[key] = {r[0] for r in rows}
non_empty = {k: v for k, v in year_sets.items() if v}
if len(non_empty) < 2:
all_years: set[int] = set()
else:
all_years = set.intersection(*non_empty.values())
return {
"source_years": {k: sorted(v) for k, v in year_sets.items()},
"years_in_all_sources": sorted(all_years),
}
def format_report(conn: duckdb.DuckDBPyConnection) -> str:
"""Generate a full text validation report."""
lines: list[str] = []
# Row counts
lines.append("=== Row Counts ===")
counts = validate_row_counts(conn)
for table, count in counts.items():
lines.append(f" {table:<30s} {count:>8d}")
# NULL rates
lines.append("\n=== NULL Rates (%) ===")
nulls = validate_null_rates(conn)
if not nulls:
lines.append(" (no data loaded)")
for table, rates in nulls.items():
lines.append(f" {table}:")
for col, pct in rates.items():
flag = " !" if pct > 0 else ""
lines.append(f" {col:<35s} {pct:>5.1f}%{flag}")
# Year coverage
lines.append("\n=== Year Coverage ===")
coverage = validate_year_coverage(conn)
for table, info in coverage.items():
years = info["years"]
gaps = info["gaps"]
if not years:
lines.append(f" {table:<30s} no data")
continue
yr_range = f"{min(years)}-{max(years)} ({len(years)} years)"
lines.append(f" {table:<30s} {yr_range}")
if gaps:
lines.append(f" gaps: {gaps}")
# Cross-source
lines.append("\n=== Cross-Source Consistency ===")
cs = validate_cross_source_consistency(conn)
for source, years in cs["source_years"].items():
if years:
lines.append(f" {source:<20s} {min(years)}-{max(years)} ({len(years)} years)")
else:
lines.append(f" {source:<20s} no data")
overlap = cs["years_in_all_sources"]
if overlap:
lines.append(f" Years in all sources: {min(overlap)}-{max(overlap)} ({len(overlap)} years)")
else:
lines.append(" Years in all sources: none")
return "\n".join(lines)