Phase 1 project prototype
This commit is contained in:
parent
29215e2bd2
commit
2c9ae1c312
29 changed files with 2967 additions and 22 deletions
32
src/admin_analytics/bls/download.py
Normal file
32
src/admin_analytics/bls/download.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
"""BLS CPI-U flat-file download."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from admin_analytics.config import BLS_CPI_URL, BLS_DATA_DIR
|
||||
|
||||
CPI_FILENAME = "cu.data.0.Current"
|
||||
|
||||
|
||||
def download_cpi_file(force: bool = False) -> Path:
|
||||
"""Download the BLS CPI-U flat file.
|
||||
|
||||
Returns the path to the downloaded file.
|
||||
"""
|
||||
BLS_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
dest = BLS_DATA_DIR / CPI_FILENAME
|
||||
|
||||
if not force and dest.exists():
|
||||
return dest
|
||||
|
||||
with httpx.Client(
|
||||
follow_redirects=True,
|
||||
timeout=120.0,
|
||||
headers={"User-Agent": "admin-analytics/0.1 (research)"},
|
||||
) as client:
|
||||
resp = client.get(BLS_CPI_URL)
|
||||
resp.raise_for_status()
|
||||
|
||||
dest.write_bytes(resp.content)
|
||||
return dest
|
||||
56
src/admin_analytics/bls/loader.py
Normal file
56
src/admin_analytics/bls/loader.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
"""Parse and load BLS CPI-U data into DuckDB."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import polars as pl
|
||||
|
||||
|
||||
def load_cpi(conn: duckdb.DuckDBPyConnection, file_path: Path) -> int:
|
||||
"""Parse the BLS CPI-U flat file and load into raw_cpi_u.
|
||||
|
||||
Filters to series CUUR0000SA0 (All Urban Consumers, US City Average,
|
||||
All Items, Not Seasonally Adjusted) and monthly periods M01-M12.
|
||||
|
||||
Returns number of rows loaded.
|
||||
"""
|
||||
df = pl.read_csv(
|
||||
file_path,
|
||||
separator="\t",
|
||||
infer_schema_length=0, # read all as strings first
|
||||
)
|
||||
|
||||
# BLS pads fields with whitespace — strip all string columns
|
||||
df = df.with_columns(
|
||||
pl.col(col).str.strip_chars() for col in df.columns
|
||||
)
|
||||
|
||||
# Normalize column names (BLS has trailing spaces in headers too)
|
||||
df = df.rename({col: col.strip() for col in df.columns})
|
||||
|
||||
# Filter to CPI-U series and monthly periods only
|
||||
df = df.filter(
|
||||
(pl.col("series_id") == "CUUR0000SA0")
|
||||
& (pl.col("period").str.starts_with("M"))
|
||||
& (pl.col("period") != "M13")
|
||||
)
|
||||
|
||||
# Extract month from period (M01 -> 1, M12 -> 12)
|
||||
df = df.with_columns(
|
||||
pl.col("year").cast(pl.Int64).alias("year"),
|
||||
pl.col("period").str.slice(1).cast(pl.Int64).alias("month"),
|
||||
pl.col("value").cast(pl.Float64).alias("value"),
|
||||
pl.lit("CUUR0000SA0").alias("series_id"),
|
||||
).select("year", "month", "value", "series_id")
|
||||
|
||||
# Idempotent: delete all existing rows then insert
|
||||
conn.execute("DELETE FROM raw_cpi_u")
|
||||
|
||||
rows = df.to_dicts()
|
||||
for row in rows:
|
||||
conn.execute(
|
||||
"INSERT INTO raw_cpi_u (year, month, value, series_id) VALUES (?, ?, ?, ?)",
|
||||
[row["year"], row["month"], row["value"], row["series_id"]],
|
||||
)
|
||||
|
||||
return len(rows)
|
||||
|
|
@ -107,10 +107,83 @@ def irs990(
|
|||
|
||||
|
||||
@ingest_app.command()
|
||||
def cpi() -> None:
|
||||
"""Ingest BLS CPI-U data. (Not yet implemented.)"""
|
||||
typer.echo("CPI-U ingestion is not yet implemented (Sprint 3).")
|
||||
raise typer.Exit(1)
|
||||
def cpi(
|
||||
force: Annotated[
|
||||
bool, typer.Option("--force", help="Re-download even if file exists")
|
||||
] = False,
|
||||
) -> None:
|
||||
"""Ingest BLS CPI-U data."""
|
||||
conn = get_connection()
|
||||
ensure_schema(conn)
|
||||
|
||||
from admin_analytics.bls.download import download_cpi_file
|
||||
from admin_analytics.bls.loader import load_cpi
|
||||
|
||||
typer.echo("Downloading BLS CPI-U data...")
|
||||
file_path = download_cpi_file(force=force)
|
||||
|
||||
typer.echo("Loading CPI-U data into database...")
|
||||
count = load_cpi(conn, file_path)
|
||||
typer.echo(f"CPI-U ingestion complete: {count} monthly observations loaded.")
|
||||
conn.close()
|
||||
|
||||
|
||||
@ingest_app.command()
|
||||
def scrape() -> None:
|
||||
"""Scrape UD staff directory pages for admin headcounts."""
|
||||
conn = get_connection()
|
||||
ensure_schema(conn)
|
||||
|
||||
from admin_analytics.scraper.directory import scrape_all
|
||||
from admin_analytics.scraper.loader import load_scrape
|
||||
from admin_analytics.scraper.classify import OVERHEAD_CATEGORIES, NON_OVERHEAD_CATEGORIES
|
||||
|
||||
typer.echo("Scraping UD staff directory pages...")
|
||||
entries = scrape_all()
|
||||
|
||||
typer.echo("Loading scraped data into database...")
|
||||
count = load_scrape(conn, entries)
|
||||
|
||||
# Summary by unit and category
|
||||
typer.echo(f"\nLoaded {count} staff entries.\n")
|
||||
unit_counts: dict[str, dict[str, int]] = {}
|
||||
for e in entries:
|
||||
unit_counts.setdefault(e.unit, {})
|
||||
unit_counts[e.unit][e.category] = unit_counts[e.unit].get(e.category, 0) + 1
|
||||
|
||||
for unit, cats in sorted(unit_counts.items()):
|
||||
total = sum(cats.values())
|
||||
overhead = sum(v for k, v in cats.items() if k in OVERHEAD_CATEGORIES)
|
||||
typer.echo(f" {unit}: {total} staff ({overhead} overhead)")
|
||||
for cat, n in sorted(cats.items(), key=lambda x: -x[1]):
|
||||
typer.echo(f" {cat}: {n}")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
@app.command()
|
||||
def dashboard(
|
||||
port: Annotated[int, typer.Option(help="Port to serve on")] = 8050,
|
||||
debug: Annotated[bool, typer.Option(help="Enable Dash debug mode")] = True,
|
||||
) -> None:
|
||||
"""Launch the analytics dashboard."""
|
||||
from admin_analytics.dashboard.app import create_app
|
||||
|
||||
dash_app = create_app()
|
||||
typer.echo(f"Starting dashboard at http://localhost:{port}/")
|
||||
dash_app.run(debug=debug, port=port)
|
||||
|
||||
|
||||
@app.command()
|
||||
def validate() -> None:
|
||||
"""Run data validation checks and print a report."""
|
||||
conn = get_connection()
|
||||
ensure_schema(conn)
|
||||
|
||||
from admin_analytics.validation import format_report
|
||||
|
||||
typer.echo(format_report(conn))
|
||||
conn.close()
|
||||
|
||||
|
||||
@ingest_app.command(name="all")
|
||||
|
|
@ -124,3 +197,6 @@ def ingest_all(
|
|||
) -> None:
|
||||
"""Ingest all data sources."""
|
||||
ipeds(year_range=year_range, component="all", force=force)
|
||||
irs990(year_range=year_range, force=force)
|
||||
cpi(force=force)
|
||||
scrape()
|
||||
|
|
|
|||
|
|
@ -14,6 +14,10 @@ IPEDS_DATA_DIR = DATA_DIR / "ipeds"
|
|||
|
||||
DEFAULT_YEAR_RANGE = range(2005, 2025) # 2005-2024 inclusive
|
||||
|
||||
# BLS CPI-U
|
||||
BLS_CPI_URL = "https://download.bls.gov/pub/time.series/cu/cu.data.0.Current"
|
||||
BLS_DATA_DIR = DATA_DIR / "bls"
|
||||
|
||||
# IRS 990
|
||||
UD_EINS = [516000297, 516017306] # UD + UD Research Foundation
|
||||
IRS990_BASE_URL = "https://apps.irs.gov/pub/epostcard/990/xml"
|
||||
|
|
|
|||
0
src/admin_analytics/dashboard/__init__.py
Normal file
0
src/admin_analytics/dashboard/__init__.py
Normal file
53
src/admin_analytics/dashboard/app.py
Normal file
53
src/admin_analytics/dashboard/app.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
"""Dash application factory."""
|
||||
|
||||
import dash
|
||||
from dash import dcc, html, Input, Output
|
||||
|
||||
from admin_analytics.db.connection import get_connection
|
||||
from admin_analytics.db.schema import ensure_schema
|
||||
from admin_analytics.dashboard.pages import overview, compensation, staffing, headcount
|
||||
|
||||
|
||||
def create_app() -> dash.Dash:
|
||||
"""Create and configure the Dash application."""
|
||||
app = dash.Dash(__name__, suppress_callback_exceptions=True)
|
||||
conn = get_connection()
|
||||
ensure_schema(conn)
|
||||
|
||||
app.layout = html.Div(
|
||||
[
|
||||
html.H1(
|
||||
"University of Delaware — Administrative Analytics",
|
||||
style={"textAlign": "center", "padding": "20px", "color": "#00539F"},
|
||||
),
|
||||
dcc.Tabs(
|
||||
id="tabs",
|
||||
value="overview",
|
||||
children=[
|
||||
dcc.Tab(label="Admin Cost Overview", value="overview"),
|
||||
dcc.Tab(label="Executive Compensation", value="compensation"),
|
||||
dcc.Tab(label="Staffing & Enrollment", value="staffing"),
|
||||
dcc.Tab(label="Current Headcount", value="headcount"),
|
||||
],
|
||||
style={"marginBottom": "20px"},
|
||||
),
|
||||
html.Div(id="tab-content", style={"padding": "0 20px 20px 20px"}),
|
||||
],
|
||||
style={"fontFamily": "system-ui, -apple-system, sans-serif", "maxWidth": "1400px", "margin": "0 auto"},
|
||||
)
|
||||
|
||||
@app.callback(Output("tab-content", "children"), Input("tabs", "value"))
|
||||
def render_tab(tab: str):
|
||||
if tab == "overview":
|
||||
return overview.layout(conn)
|
||||
elif tab == "compensation":
|
||||
return compensation.layout(conn)
|
||||
elif tab == "staffing":
|
||||
return staffing.layout(conn)
|
||||
elif tab == "headcount":
|
||||
return headcount.layout(conn)
|
||||
return html.Div("Unknown tab")
|
||||
|
||||
compensation.register_callbacks(app, conn)
|
||||
|
||||
return app
|
||||
0
src/admin_analytics/dashboard/pages/__init__.py
Normal file
0
src/admin_analytics/dashboard/pages/__init__.py
Normal file
162
src/admin_analytics/dashboard/pages/compensation.py
Normal file
162
src/admin_analytics/dashboard/pages/compensation.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
"""Page 2: Executive Compensation."""
|
||||
|
||||
import dash
|
||||
import duckdb
|
||||
from dash import html, dcc, Input, Output, dash_table
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
|
||||
from admin_analytics.dashboard.queries import (
|
||||
query_top_earners,
|
||||
query_comp_by_role,
|
||||
query_comp_vs_cpi,
|
||||
)
|
||||
|
||||
_NO_DATA = html.Div(
|
||||
"No IRS 990 data loaded. Run: admin-analytics ingest irs990",
|
||||
style={"textAlign": "center", "padding": "40px", "color": "#888"},
|
||||
)
|
||||
|
||||
# Roles to highlight in trend chart
|
||||
_KEY_ROLES = ["PRESIDENT", "PROVOST", "VP_FINANCE", "VP_RESEARCH", "VP_ADVANCEMENT", "CFO"]
|
||||
|
||||
|
||||
def layout(conn: duckdb.DuckDBPyConnection):
|
||||
all_earners = query_top_earners(conn)
|
||||
if all_earners.height == 0:
|
||||
return _NO_DATA
|
||||
|
||||
years = sorted(all_earners["tax_year"].unique().to_list())
|
||||
year_options = [{"label": "All Years", "value": "all"}] + [
|
||||
{"label": str(y), "value": y} for y in years
|
||||
]
|
||||
|
||||
# Compensation by role trend
|
||||
role_df = query_comp_by_role(conn)
|
||||
role_fig = go.Figure()
|
||||
if role_df.height > 0:
|
||||
role_pd = role_df.to_pandas()
|
||||
for role in _KEY_ROLES:
|
||||
subset = role_pd[role_pd["canonical_role"] == role]
|
||||
if len(subset) > 0:
|
||||
role_fig.add_trace(go.Scatter(
|
||||
x=subset["tax_year"],
|
||||
y=subset["total_compensation"],
|
||||
mode="lines+markers",
|
||||
name=role.replace("_", " ").title(),
|
||||
))
|
||||
role_fig.update_layout(
|
||||
title="Compensation Trends by Role",
|
||||
xaxis_title="Tax Year", yaxis_title="Total Compensation ($)",
|
||||
template="plotly_white", height=420,
|
||||
)
|
||||
|
||||
# Comp vs CPI indexed
|
||||
cpi_df = query_comp_vs_cpi(conn)
|
||||
cpi_fig = go.Figure()
|
||||
if cpi_df.height > 0:
|
||||
cpi_pd = cpi_df.to_pandas()
|
||||
cpi_fig.add_trace(go.Scatter(
|
||||
x=cpi_pd["year"], y=cpi_pd["comp_index"],
|
||||
mode="lines+markers", name="Top Compensation",
|
||||
line={"color": "#00539F"},
|
||||
))
|
||||
cpi_fig.add_trace(go.Scatter(
|
||||
x=cpi_pd["year"], y=cpi_pd["cpi_index"],
|
||||
mode="lines+markers", name="CPI-U",
|
||||
line={"color": "#FFD200", "dash": "dash"},
|
||||
))
|
||||
cpi_fig.update_layout(
|
||||
title="Top Compensation vs CPI-U (Indexed, Base Year = 100)",
|
||||
xaxis_title="Year", yaxis_title="Index",
|
||||
template="plotly_white", height=380,
|
||||
)
|
||||
|
||||
return html.Div([
|
||||
html.Div(
|
||||
[
|
||||
html.Label("Filter by Tax Year: ", style={"fontWeight": "bold"}),
|
||||
dcc.Dropdown(
|
||||
id="comp-year-dropdown",
|
||||
options=year_options,
|
||||
value="all",
|
||||
style={"width": "200px", "display": "inline-block"},
|
||||
),
|
||||
],
|
||||
style={"marginBottom": "16px"},
|
||||
),
|
||||
dash_table.DataTable(
|
||||
id="comp-table",
|
||||
columns=[
|
||||
{"name": "Year", "id": "tax_year"},
|
||||
{"name": "Name", "id": "person_name"},
|
||||
{"name": "Title", "id": "title"},
|
||||
{"name": "Role", "id": "canonical_role"},
|
||||
{"name": "Base", "id": "base_compensation", "type": "numeric",
|
||||
"format": dash_table.Format.Format().group(True)},
|
||||
{"name": "Bonus", "id": "bonus_compensation", "type": "numeric",
|
||||
"format": dash_table.Format.Format().group(True)},
|
||||
{"name": "Total", "id": "total_compensation", "type": "numeric",
|
||||
"format": dash_table.Format.Format().group(True)},
|
||||
],
|
||||
data=all_earners.to_pandas().to_dict("records"),
|
||||
page_size=15,
|
||||
sort_action="native",
|
||||
filter_action="native",
|
||||
style_table={"overflowX": "auto"},
|
||||
style_cell={"textAlign": "left", "padding": "8px", "fontSize": "13px"},
|
||||
style_header={"fontWeight": "bold", "backgroundColor": "#f0f0f0"},
|
||||
),
|
||||
html.Div(
|
||||
[
|
||||
html.Div(dcc.Graph(id="comp-breakdown-chart"), style={"flex": "1"}),
|
||||
html.Div(dcc.Graph(figure=cpi_fig), style={"flex": "1"}),
|
||||
],
|
||||
style={"display": "flex", "gap": "16px", "marginTop": "16px"},
|
||||
),
|
||||
dcc.Graph(figure=role_fig),
|
||||
])
|
||||
|
||||
|
||||
def register_callbacks(app: dash.Dash, conn: duckdb.DuckDBPyConnection) -> None:
|
||||
"""Register interactive callbacks for the compensation page."""
|
||||
|
||||
@app.callback(
|
||||
[Output("comp-table", "data"), Output("comp-breakdown-chart", "figure")],
|
||||
Input("comp-year-dropdown", "value"),
|
||||
)
|
||||
def update_compensation(year_value):
|
||||
year = None if year_value == "all" else int(year_value)
|
||||
earners = query_top_earners(conn, year=year)
|
||||
|
||||
# Table data
|
||||
table_data = earners.to_pandas().to_dict("records") if earners.height > 0 else []
|
||||
|
||||
# Breakdown chart — stacked bar of comp components
|
||||
breakdown_fig = go.Figure()
|
||||
if earners.height > 0:
|
||||
ep = earners.to_pandas().head(10) # top 10 by total comp
|
||||
short_names = [n.split(",")[0][:20] if "," in n else n.split()[-1][:20]
|
||||
for n in ep["person_name"]]
|
||||
for comp_type, label, color in [
|
||||
("base_compensation", "Base", "#00539F"),
|
||||
("bonus_compensation", "Bonus", "#FFD200"),
|
||||
("deferred_compensation", "Deferred", "#7FB069"),
|
||||
("nontaxable_benefits", "Benefits", "#E07A5F"),
|
||||
("other_compensation", "Other", "#999"),
|
||||
]:
|
||||
if comp_type in ep.columns:
|
||||
breakdown_fig.add_trace(go.Bar(
|
||||
x=short_names, y=ep[comp_type],
|
||||
name=label, marker_color=color,
|
||||
))
|
||||
breakdown_fig.update_layout(barmode="stack")
|
||||
|
||||
title_suffix = f" ({year})" if year else " (All Years)"
|
||||
breakdown_fig.update_layout(
|
||||
title=f"Compensation Breakdown — Top 10{title_suffix}",
|
||||
xaxis_title="", yaxis_title="$",
|
||||
template="plotly_white", height=380,
|
||||
)
|
||||
|
||||
return table_data, breakdown_fig
|
||||
118
src/admin_analytics/dashboard/pages/headcount.py
Normal file
118
src/admin_analytics/dashboard/pages/headcount.py
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
"""Page 4: Current Admin Headcount (from scraper)."""
|
||||
|
||||
import duckdb
|
||||
from dash import html, dcc, dash_table
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
|
||||
from admin_analytics.dashboard.queries import (
|
||||
query_admin_headcount,
|
||||
query_headcount_summary,
|
||||
)
|
||||
|
||||
_NO_DATA = html.Div(
|
||||
"No headcount data loaded. Run: admin-analytics ingest scrape",
|
||||
style={"textAlign": "center", "padding": "40px", "color": "#888"},
|
||||
)
|
||||
|
||||
|
||||
def _kpi_card(title: str, value: str) -> html.Div:
|
||||
return html.Div(
|
||||
[
|
||||
html.H4(title, style={"margin": "0", "color": "#666", "fontSize": "14px"}),
|
||||
html.H2(value, style={"margin": "5px 0", "color": "#00539F"}),
|
||||
],
|
||||
style={
|
||||
"flex": "1",
|
||||
"padding": "20px",
|
||||
"backgroundColor": "#f8f9fa",
|
||||
"borderRadius": "8px",
|
||||
"textAlign": "center",
|
||||
"margin": "0 8px",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def layout(conn: duckdb.DuckDBPyConnection):
|
||||
detail_df = query_admin_headcount(conn)
|
||||
if detail_df.height == 0:
|
||||
return _NO_DATA
|
||||
|
||||
summary_df = query_headcount_summary(conn)
|
||||
detail_pd = detail_df.to_pandas()
|
||||
summary_pd = summary_df.to_pandas()
|
||||
|
||||
total = len(detail_pd)
|
||||
overhead_count = int(detail_pd["is_overhead"].sum()) if "is_overhead" in detail_pd.columns else 0
|
||||
overhead_pct = round(overhead_count * 100 / total, 1) if total > 0 else 0
|
||||
|
||||
# KPI cards
|
||||
kpi_row = html.Div(
|
||||
[
|
||||
_kpi_card("Total Staff Scraped", str(total)),
|
||||
_kpi_card("Overhead Staff", str(overhead_count)),
|
||||
_kpi_card("Overhead %", f"{overhead_pct}%"),
|
||||
],
|
||||
style={"display": "flex", "marginBottom": "24px"},
|
||||
)
|
||||
|
||||
# Staff by unit bar chart
|
||||
unit_counts = summary_pd.groupby("unit")["count"].sum().reset_index().sort_values("count")
|
||||
unit_fig = px.bar(
|
||||
unit_counts, x="count", y="unit", orientation="h",
|
||||
title="Staff Count by Unit",
|
||||
labels={"count": "Staff", "unit": ""},
|
||||
color_discrete_sequence=["#00539F"],
|
||||
)
|
||||
unit_fig.update_layout(template="plotly_white", height=max(300, len(unit_counts) * 30 + 100))
|
||||
|
||||
# Overhead pie
|
||||
oh_data = detail_pd["is_overhead"].value_counts()
|
||||
oh_labels = {True: "Overhead", False: "Non-Overhead"}
|
||||
pie_fig = px.pie(
|
||||
names=[oh_labels.get(k, "Debatable") for k in oh_data.index],
|
||||
values=oh_data.values,
|
||||
title="Overhead vs Non-Overhead",
|
||||
color_discrete_sequence=["#E07A5F", "#7FB069", "#999"],
|
||||
)
|
||||
pie_fig.update_layout(template="plotly_white", height=350)
|
||||
|
||||
# Category distribution per unit
|
||||
cat_fig = px.bar(
|
||||
summary_pd, x="count", y="unit", color="category", orientation="h",
|
||||
title="Category Distribution by Unit",
|
||||
labels={"count": "Staff", "unit": "", "category": "Category"},
|
||||
)
|
||||
cat_fig.update_layout(template="plotly_white", height=max(300, len(unit_counts) * 30 + 100))
|
||||
|
||||
# Detail table
|
||||
table = dash_table.DataTable(
|
||||
columns=[
|
||||
{"name": "Unit", "id": "unit"},
|
||||
{"name": "Name", "id": "person_name"},
|
||||
{"name": "Title", "id": "title"},
|
||||
{"name": "Category", "id": "category"},
|
||||
{"name": "Overhead", "id": "is_overhead"},
|
||||
],
|
||||
data=detail_pd.to_dict("records"),
|
||||
page_size=20,
|
||||
sort_action="native",
|
||||
filter_action="native",
|
||||
style_table={"overflowX": "auto"},
|
||||
style_cell={"textAlign": "left", "padding": "8px", "fontSize": "13px"},
|
||||
style_header={"fontWeight": "bold", "backgroundColor": "#f0f0f0"},
|
||||
)
|
||||
|
||||
return html.Div([
|
||||
kpi_row,
|
||||
html.Div(
|
||||
[
|
||||
html.Div(dcc.Graph(figure=unit_fig), style={"flex": "1"}),
|
||||
html.Div(dcc.Graph(figure=pie_fig), style={"flex": "1"}),
|
||||
],
|
||||
style={"display": "flex", "gap": "16px"},
|
||||
),
|
||||
dcc.Graph(figure=cat_fig),
|
||||
html.H3("Staff Directory Detail", style={"marginTop": "24px"}),
|
||||
table,
|
||||
])
|
||||
168
src/admin_analytics/dashboard/pages/overview.py
Normal file
168
src/admin_analytics/dashboard/pages/overview.py
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
"""Page 1: Administrative Cost Overview."""
|
||||
|
||||
import duckdb
|
||||
from dash import html, dcc
|
||||
import plotly.express as px
|
||||
import plotly.graph_objects as go
|
||||
|
||||
from admin_analytics.dashboard.queries import (
|
||||
query_admin_cost_ratio,
|
||||
query_expense_breakdown,
|
||||
query_admin_per_student,
|
||||
query_admin_faculty_ratio,
|
||||
)
|
||||
|
||||
_NO_DATA = html.Div(
|
||||
"No data loaded. Run: admin-analytics ingest all",
|
||||
style={"textAlign": "center", "padding": "40px", "color": "#888"},
|
||||
)
|
||||
|
||||
|
||||
def _kpi_card(title: str, value: str, subtitle: str = "") -> html.Div:
|
||||
return html.Div(
|
||||
[
|
||||
html.H4(title, style={"margin": "0", "color": "#666", "fontSize": "14px"}),
|
||||
html.H2(value, style={"margin": "5px 0", "color": "#00539F"}),
|
||||
html.P(subtitle, style={"margin": "0", "color": "#999", "fontSize": "12px"}),
|
||||
],
|
||||
style={
|
||||
"flex": "1",
|
||||
"padding": "20px",
|
||||
"backgroundColor": "#f8f9fa",
|
||||
"borderRadius": "8px",
|
||||
"textAlign": "center",
|
||||
"margin": "0 8px",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def layout(conn: duckdb.DuckDBPyConnection):
|
||||
# Admin cost ratio
|
||||
ratio_df = query_admin_cost_ratio(conn)
|
||||
if ratio_df.height == 0:
|
||||
return _NO_DATA
|
||||
|
||||
ratio_pd = ratio_df.to_pandas()
|
||||
latest = ratio_pd.iloc[-1]
|
||||
|
||||
# Admin per student
|
||||
aps_df = query_admin_per_student(conn)
|
||||
aps_pd = aps_df.to_pandas() if aps_df.height > 0 else None
|
||||
latest_aps = aps_pd.iloc[-1] if aps_pd is not None else None
|
||||
|
||||
# Admin-to-faculty ratio
|
||||
afr_df = query_admin_faculty_ratio(conn)
|
||||
afr_pd = afr_df.to_pandas() if afr_df.height > 0 else None
|
||||
latest_afr = afr_pd.iloc[-1] if afr_pd is not None else None
|
||||
|
||||
# KPI cards
|
||||
kpi_row = html.Div(
|
||||
[
|
||||
_kpi_card(
|
||||
"Admin Cost Ratio",
|
||||
f"{latest['admin_cost_pct']:.1f}%",
|
||||
f"Institutional Support / Total Expenses ({int(latest['year'])})",
|
||||
),
|
||||
_kpi_card(
|
||||
"Admin Cost per Student",
|
||||
f"${int(latest_aps['admin_per_student']):,}" if latest_aps is not None else "N/A",
|
||||
f"CPI-adjusted: ${int(latest_aps['admin_per_student_cpi']):,}" if latest_aps is not None else "",
|
||||
),
|
||||
_kpi_card(
|
||||
"Admin-to-Faculty Ratio",
|
||||
f"{latest_afr['admin_faculty_ratio']:.2f}" if latest_afr is not None else "N/A",
|
||||
f"Management / Faculty ({int(latest_afr['year'])})" if latest_afr is not None else "",
|
||||
),
|
||||
],
|
||||
style={"display": "flex", "marginBottom": "24px"},
|
||||
)
|
||||
|
||||
# Admin cost ratio trend
|
||||
ratio_fig = go.Figure()
|
||||
ratio_fig.add_trace(go.Scatter(
|
||||
x=ratio_pd["year"], y=ratio_pd["admin_cost_pct"],
|
||||
mode="lines+markers", name="Admin Cost %",
|
||||
line={"color": "#00539F"},
|
||||
))
|
||||
ratio_fig.update_layout(
|
||||
title="Administrative Cost Ratio Over Time",
|
||||
xaxis_title="Year", yaxis_title="Institutional Support / Total Expenses (%)",
|
||||
template="plotly_white", height=400,
|
||||
)
|
||||
|
||||
# Expense breakdown stacked area
|
||||
breakdown_df = query_expense_breakdown(conn)
|
||||
if breakdown_df.height > 0:
|
||||
bk_pd = breakdown_df.to_pandas()
|
||||
expense_cols = [c for c in bk_pd.columns if c != "year"]
|
||||
labels = {
|
||||
"instruction_expenses": "Instruction",
|
||||
"research_expenses": "Research",
|
||||
"public_service_expenses": "Public Service",
|
||||
"academic_support_expenses": "Academic Support",
|
||||
"student_services_expenses": "Student Services",
|
||||
"institutional_support_expenses": "Institutional Support",
|
||||
"auxiliary_expenses": "Auxiliary",
|
||||
"hospital_expenses": "Hospital",
|
||||
"other_expenses": "Other",
|
||||
}
|
||||
breakdown_fig = go.Figure()
|
||||
for col in expense_cols:
|
||||
breakdown_fig.add_trace(go.Scatter(
|
||||
x=bk_pd["year"], y=bk_pd[col] / 1e6,
|
||||
mode="lines", name=labels.get(col, col),
|
||||
stackgroup="one",
|
||||
))
|
||||
breakdown_fig.update_layout(
|
||||
title="Expenses by Function (Millions $)",
|
||||
xaxis_title="Year", yaxis_title="Millions $",
|
||||
template="plotly_white", height=450,
|
||||
)
|
||||
else:
|
||||
breakdown_fig = go.Figure()
|
||||
|
||||
# Admin per student trend
|
||||
aps_fig = go.Figure()
|
||||
if aps_pd is not None:
|
||||
aps_fig.add_trace(go.Scatter(
|
||||
x=aps_pd["year"], y=aps_pd["admin_per_student"],
|
||||
mode="lines+markers", name="Nominal",
|
||||
line={"color": "#00539F"},
|
||||
))
|
||||
aps_fig.add_trace(go.Scatter(
|
||||
x=aps_pd["year"], y=aps_pd["admin_per_student_cpi"],
|
||||
mode="lines+markers", name="CPI-Adjusted",
|
||||
line={"color": "#FFD200", "dash": "dash"},
|
||||
))
|
||||
aps_fig.update_layout(
|
||||
title="Admin Cost per Student",
|
||||
xaxis_title="Year", yaxis_title="$ per Student",
|
||||
template="plotly_white", height=380,
|
||||
)
|
||||
|
||||
# Admin-to-faculty ratio trend
|
||||
afr_fig = go.Figure()
|
||||
if afr_pd is not None:
|
||||
afr_fig.add_trace(go.Scatter(
|
||||
x=afr_pd["year"], y=afr_pd["admin_faculty_ratio"],
|
||||
mode="lines+markers", name="Ratio",
|
||||
line={"color": "#00539F"},
|
||||
))
|
||||
afr_fig.update_layout(
|
||||
title="Admin-to-Faculty Ratio",
|
||||
xaxis_title="Year", yaxis_title="Management / Faculty",
|
||||
template="plotly_white", height=380,
|
||||
)
|
||||
|
||||
return html.Div([
|
||||
kpi_row,
|
||||
dcc.Graph(figure=ratio_fig),
|
||||
dcc.Graph(figure=breakdown_fig),
|
||||
html.Div(
|
||||
[
|
||||
html.Div(dcc.Graph(figure=aps_fig), style={"flex": "1"}),
|
||||
html.Div(dcc.Graph(figure=afr_fig), style={"flex": "1"}),
|
||||
],
|
||||
style={"display": "flex", "gap": "16px"},
|
||||
),
|
||||
])
|
||||
97
src/admin_analytics/dashboard/pages/staffing.py
Normal file
97
src/admin_analytics/dashboard/pages/staffing.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
"""Page 3: Staffing & Enrollment."""
|
||||
|
||||
import duckdb
|
||||
from dash import html, dcc
|
||||
import plotly.graph_objects as go
|
||||
|
||||
from admin_analytics.dashboard.queries import (
|
||||
query_staff_composition,
|
||||
query_student_staff_ratios,
|
||||
query_growth_index,
|
||||
)
|
||||
|
||||
_NO_DATA = html.Div(
|
||||
"No IPEDS staff data loaded. Run: admin-analytics ingest ipeds",
|
||||
style={"textAlign": "center", "padding": "40px", "color": "#888"},
|
||||
)
|
||||
|
||||
|
||||
def layout(conn: duckdb.DuckDBPyConnection):
|
||||
staff_df = query_staff_composition(conn)
|
||||
if staff_df.height == 0:
|
||||
return _NO_DATA
|
||||
|
||||
staff_pd = staff_df.to_pandas()
|
||||
|
||||
# Staff composition stacked area
|
||||
comp_fig = go.Figure()
|
||||
for col, label, color in [
|
||||
("faculty_total", "Faculty", "#00539F"),
|
||||
("management_total", "Management", "#E07A5F"),
|
||||
("other_staff", "Other Staff", "#7FB069"),
|
||||
]:
|
||||
comp_fig.add_trace(go.Scatter(
|
||||
x=staff_pd["year"], y=staff_pd[col],
|
||||
mode="lines", name=label,
|
||||
stackgroup="one",
|
||||
line={"color": color},
|
||||
))
|
||||
comp_fig.update_layout(
|
||||
title="Staff Composition Over Time",
|
||||
xaxis_title="Year", yaxis_title="Headcount",
|
||||
template="plotly_white", height=420,
|
||||
)
|
||||
|
||||
# Student-to-staff ratios
|
||||
ratio_df = query_student_staff_ratios(conn)
|
||||
ratio_fig = go.Figure()
|
||||
if ratio_df.height > 0:
|
||||
ratio_pd = ratio_df.to_pandas()
|
||||
ratio_fig.add_trace(go.Scatter(
|
||||
x=ratio_pd["year"], y=ratio_pd["students_per_staff"],
|
||||
mode="lines+markers", name="Students per Staff",
|
||||
line={"color": "#00539F"},
|
||||
))
|
||||
ratio_fig.add_trace(go.Scatter(
|
||||
x=ratio_pd["year"], y=ratio_pd["students_per_faculty"],
|
||||
mode="lines+markers", name="Students per Faculty",
|
||||
line={"color": "#FFD200"},
|
||||
))
|
||||
ratio_fig.update_layout(
|
||||
title="Student-to-Staff Ratios",
|
||||
xaxis_title="Year", yaxis_title="Ratio",
|
||||
template="plotly_white", height=380,
|
||||
)
|
||||
|
||||
# Growth index
|
||||
growth_df = query_growth_index(conn)
|
||||
growth_fig = go.Figure()
|
||||
if growth_df.height > 0:
|
||||
growth_pd = growth_df.to_pandas()
|
||||
growth_fig.add_trace(go.Scatter(
|
||||
x=growth_pd["year"], y=growth_pd["mgmt_index"],
|
||||
mode="lines+markers", name="Management Growth",
|
||||
line={"color": "#E07A5F"},
|
||||
))
|
||||
growth_fig.add_trace(go.Scatter(
|
||||
x=growth_pd["year"], y=growth_pd["enrollment_index"],
|
||||
mode="lines+markers", name="Enrollment Growth",
|
||||
line={"color": "#00539F"},
|
||||
))
|
||||
growth_fig.add_hline(y=100, line_dash="dot", line_color="#ccc")
|
||||
growth_fig.update_layout(
|
||||
title="Management vs Enrollment Growth (Indexed, Base Year = 100)",
|
||||
xaxis_title="Year", yaxis_title="Index",
|
||||
template="plotly_white", height=380,
|
||||
)
|
||||
|
||||
return html.Div([
|
||||
dcc.Graph(figure=comp_fig),
|
||||
html.Div(
|
||||
[
|
||||
html.Div(dcc.Graph(figure=ratio_fig), style={"flex": "1"}),
|
||||
html.Div(dcc.Graph(figure=growth_fig), style={"flex": "1"}),
|
||||
],
|
||||
style={"display": "flex", "gap": "16px"},
|
||||
),
|
||||
])
|
||||
263
src/admin_analytics/dashboard/queries.py
Normal file
263
src/admin_analytics/dashboard/queries.py
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
"""Dashboard query layer — all DuckDB queries returning polars DataFrames."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import duckdb
|
||||
import polars as pl
|
||||
|
||||
from admin_analytics.config import UD_UNITID
|
||||
from admin_analytics.irs990.titles import normalize_title
|
||||
|
||||
# Shared CTE for CPI adjustment
|
||||
_CPI_CTE = """
|
||||
WITH annual_cpi AS (
|
||||
SELECT year, AVG(value) AS avg_cpi
|
||||
FROM raw_cpi_u
|
||||
GROUP BY year
|
||||
),
|
||||
latest_cpi AS (
|
||||
SELECT avg_cpi FROM annual_cpi
|
||||
WHERE year = (SELECT MAX(year) FROM annual_cpi)
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
def query_admin_cost_ratio(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
|
||||
"""Admin cost ratio trend with CPI-adjusted values."""
|
||||
return conn.execute(f"""
|
||||
{_CPI_CTE}
|
||||
SELECT
|
||||
f.year,
|
||||
f.institutional_support_expenses,
|
||||
f.total_expenses,
|
||||
ROUND(f.institutional_support_expenses * 100.0
|
||||
/ NULLIF(f.total_expenses, 0), 2) AS admin_cost_pct,
|
||||
ROUND(f.institutional_support_expenses
|
||||
* (SELECT avg_cpi FROM latest_cpi) / ac.avg_cpi, 0)
|
||||
AS inst_support_cpi_adjusted,
|
||||
ROUND(f.total_expenses
|
||||
* (SELECT avg_cpi FROM latest_cpi) / ac.avg_cpi, 0)
|
||||
AS total_expenses_cpi_adjusted
|
||||
FROM raw_ipeds_finance f
|
||||
LEFT JOIN annual_cpi ac ON ac.year = f.year
|
||||
WHERE f.unitid = ?
|
||||
ORDER BY f.year
|
||||
""", [UD_UNITID]).pl()
|
||||
|
||||
|
||||
def query_expense_breakdown(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
|
||||
"""Expense breakdown by function over time."""
|
||||
return conn.execute("""
|
||||
SELECT year,
|
||||
instruction_expenses, research_expenses, public_service_expenses,
|
||||
academic_support_expenses, student_services_expenses,
|
||||
institutional_support_expenses, auxiliary_expenses,
|
||||
hospital_expenses, other_expenses
|
||||
FROM raw_ipeds_finance
|
||||
WHERE unitid = ?
|
||||
ORDER BY year
|
||||
""", [UD_UNITID]).pl()
|
||||
|
||||
|
||||
def query_admin_per_student(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
|
||||
"""Admin cost per student (nominal and CPI-adjusted)."""
|
||||
return conn.execute(f"""
|
||||
{_CPI_CTE}
|
||||
SELECT
|
||||
f.year,
|
||||
f.institutional_support_expenses,
|
||||
e.total_enrollment,
|
||||
ROUND(f.institutional_support_expenses * 1.0
|
||||
/ NULLIF(e.total_enrollment, 0), 0) AS admin_per_student,
|
||||
ROUND(
|
||||
(f.institutional_support_expenses
|
||||
* (SELECT avg_cpi FROM latest_cpi) / ac.avg_cpi)
|
||||
/ NULLIF(e.total_enrollment, 0), 0
|
||||
) AS admin_per_student_cpi
|
||||
FROM raw_ipeds_finance f
|
||||
JOIN raw_ipeds_enrollment e ON e.unitid = f.unitid AND e.year = f.year
|
||||
LEFT JOIN annual_cpi ac ON ac.year = f.year
|
||||
WHERE f.unitid = ?
|
||||
ORDER BY f.year
|
||||
""", [UD_UNITID]).pl()
|
||||
|
||||
|
||||
def query_admin_faculty_ratio(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
|
||||
"""Admin-to-faculty ratio over time."""
|
||||
return conn.execute("""
|
||||
SELECT year,
|
||||
management_total,
|
||||
faculty_total,
|
||||
ROUND(management_total * 1.0 / NULLIF(faculty_total, 0), 3)
|
||||
AS admin_faculty_ratio
|
||||
FROM raw_ipeds_staff
|
||||
WHERE unitid = ?
|
||||
ORDER BY year
|
||||
""", [UD_UNITID]).pl()
|
||||
|
||||
|
||||
def query_top_earners(
|
||||
conn: duckdb.DuckDBPyConnection, year: int | None = None
|
||||
) -> pl.DataFrame:
|
||||
"""Top earners from Schedule J, optionally filtered by year."""
|
||||
where = "WHERE j.total_compensation > 0"
|
||||
params: list[Any] = []
|
||||
if year is not None:
|
||||
where += " AND j.tax_year = ?"
|
||||
params.append(year)
|
||||
|
||||
df = conn.execute(f"""
|
||||
SELECT
|
||||
j.tax_year,
|
||||
j.person_name,
|
||||
j.title,
|
||||
j.base_compensation,
|
||||
j.bonus_compensation,
|
||||
j.other_compensation,
|
||||
j.deferred_compensation,
|
||||
j.nontaxable_benefits,
|
||||
j.total_compensation,
|
||||
f.organization_name
|
||||
FROM raw_990_schedule_j j
|
||||
JOIN raw_990_filing f ON f.object_id = j.object_id
|
||||
{where}
|
||||
ORDER BY j.tax_year DESC, j.total_compensation DESC
|
||||
""", params).pl()
|
||||
|
||||
if df.height > 0:
|
||||
df = df.with_columns(
|
||||
pl.col("title").map_elements(
|
||||
normalize_title, return_dtype=pl.Utf8
|
||||
).alias("canonical_role")
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
def query_comp_by_role(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
|
||||
"""Compensation trends by canonical role across years."""
|
||||
df = conn.execute("""
|
||||
SELECT j.tax_year, j.person_name, j.title, j.total_compensation
|
||||
FROM raw_990_schedule_j j
|
||||
JOIN raw_990_filing f ON f.object_id = j.object_id
|
||||
WHERE j.total_compensation > 0
|
||||
ORDER BY j.tax_year, j.total_compensation DESC
|
||||
""").pl()
|
||||
|
||||
if df.height == 0:
|
||||
return df
|
||||
|
||||
df = df.with_columns(
|
||||
pl.col("title").map_elements(
|
||||
normalize_title, return_dtype=pl.Utf8
|
||||
).alias("canonical_role")
|
||||
)
|
||||
|
||||
# Keep highest-paid person per role per year
|
||||
return (
|
||||
df.sort("total_compensation", descending=True)
|
||||
.group_by(["tax_year", "canonical_role"])
|
||||
.first()
|
||||
.sort(["tax_year", "canonical_role"])
|
||||
)
|
||||
|
||||
|
||||
def query_comp_vs_cpi(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
|
||||
"""Compensation growth vs CPI growth, indexed to first available year = 100."""
|
||||
return conn.execute("""
|
||||
WITH yearly_max_comp AS (
|
||||
SELECT tax_year, MAX(total_compensation) AS top_comp
|
||||
FROM raw_990_schedule_j
|
||||
GROUP BY tax_year
|
||||
),
|
||||
annual_cpi AS (
|
||||
SELECT year, AVG(value) AS avg_cpi
|
||||
FROM raw_cpi_u GROUP BY year
|
||||
),
|
||||
base AS (
|
||||
SELECT c.top_comp AS base_comp, ac.avg_cpi AS base_cpi
|
||||
FROM yearly_max_comp c
|
||||
JOIN annual_cpi ac ON ac.year = c.tax_year
|
||||
ORDER BY c.tax_year LIMIT 1
|
||||
)
|
||||
SELECT
|
||||
c.tax_year AS year,
|
||||
c.top_comp,
|
||||
ac.avg_cpi,
|
||||
ROUND(c.top_comp * 100.0 / NULLIF((SELECT base_comp FROM base), 0), 1)
|
||||
AS comp_index,
|
||||
ROUND(ac.avg_cpi * 100.0 / NULLIF((SELECT base_cpi FROM base), 0), 1)
|
||||
AS cpi_index
|
||||
FROM yearly_max_comp c
|
||||
JOIN annual_cpi ac ON ac.year = c.tax_year
|
||||
ORDER BY year
|
||||
""").pl()
|
||||
|
||||
|
||||
def query_staff_composition(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
|
||||
"""Staff composition over time."""
|
||||
return conn.execute("""
|
||||
SELECT year, total_staff, faculty_total, management_total,
|
||||
total_staff - COALESCE(faculty_total, 0) - COALESCE(management_total, 0)
|
||||
AS other_staff
|
||||
FROM raw_ipeds_staff
|
||||
WHERE unitid = ?
|
||||
ORDER BY year
|
||||
""", [UD_UNITID]).pl()
|
||||
|
||||
|
||||
def query_student_staff_ratios(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
|
||||
"""Student-to-staff and student-to-faculty ratios."""
|
||||
return conn.execute("""
|
||||
SELECT s.year, e.total_enrollment, s.total_staff, s.faculty_total,
|
||||
ROUND(e.total_enrollment * 1.0 / NULLIF(s.total_staff, 0), 1)
|
||||
AS students_per_staff,
|
||||
ROUND(e.total_enrollment * 1.0 / NULLIF(s.faculty_total, 0), 1)
|
||||
AS students_per_faculty
|
||||
FROM raw_ipeds_staff s
|
||||
JOIN raw_ipeds_enrollment e ON e.unitid = s.unitid AND e.year = s.year
|
||||
WHERE s.unitid = ?
|
||||
ORDER BY s.year
|
||||
""", [UD_UNITID]).pl()
|
||||
|
||||
|
||||
def query_growth_index(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
|
||||
"""Management vs enrollment growth, indexed to first year = 100."""
|
||||
return conn.execute("""
|
||||
WITH base AS (
|
||||
SELECT s.management_total AS base_mgmt, e.total_enrollment AS base_enrl
|
||||
FROM raw_ipeds_staff s
|
||||
JOIN raw_ipeds_enrollment e ON e.unitid = s.unitid AND e.year = s.year
|
||||
WHERE s.unitid = ?
|
||||
ORDER BY s.year LIMIT 1
|
||||
)
|
||||
SELECT s.year,
|
||||
s.management_total,
|
||||
e.total_enrollment,
|
||||
ROUND(s.management_total * 100.0
|
||||
/ NULLIF((SELECT base_mgmt FROM base), 0), 1) AS mgmt_index,
|
||||
ROUND(e.total_enrollment * 100.0
|
||||
/ NULLIF((SELECT base_enrl FROM base), 0), 1) AS enrollment_index
|
||||
FROM raw_ipeds_staff s
|
||||
JOIN raw_ipeds_enrollment e ON e.unitid = s.unitid AND e.year = s.year
|
||||
WHERE s.unitid = ?
|
||||
ORDER BY s.year
|
||||
""", [UD_UNITID, UD_UNITID]).pl()
|
||||
|
||||
|
||||
def query_admin_headcount(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
|
||||
"""All scraped admin headcount entries."""
|
||||
return conn.execute("""
|
||||
SELECT unit, person_name, title, category, is_overhead, scrape_date
|
||||
FROM raw_admin_headcount
|
||||
ORDER BY unit, category, person_name
|
||||
""").pl()
|
||||
|
||||
|
||||
def query_headcount_summary(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
|
||||
"""Headcount summary by unit and category."""
|
||||
return conn.execute("""
|
||||
SELECT unit, category, is_overhead, COUNT(*) AS count
|
||||
FROM raw_admin_headcount
|
||||
GROUP BY unit, category, is_overhead
|
||||
ORDER BY unit, count DESC
|
||||
""").pl()
|
||||
|
|
@ -109,6 +109,19 @@ TABLES = {
|
|||
PRIMARY KEY (year, month)
|
||||
)
|
||||
""",
|
||||
"raw_admin_headcount": """
|
||||
CREATE SEQUENCE IF NOT EXISTS seq_admin_headcount START 1;
|
||||
CREATE TABLE IF NOT EXISTS raw_admin_headcount (
|
||||
id INTEGER PRIMARY KEY DEFAULT nextval('seq_admin_headcount'),
|
||||
scrape_date DATE NOT NULL,
|
||||
unit VARCHAR NOT NULL,
|
||||
person_name VARCHAR,
|
||||
title VARCHAR,
|
||||
email VARCHAR,
|
||||
category VARCHAR,
|
||||
is_overhead BOOLEAN
|
||||
)
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
import io
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
|
@ -57,11 +58,14 @@ def filter_index(
|
|||
def get_batch_list(year: int) -> list[str]:
|
||||
"""Get the list of available TEOS XML batch ZIP IDs for a year.
|
||||
|
||||
Tries common naming patterns: {YEAR}_TEOS_XML_{MM}{L} where
|
||||
MM is 01-12 and L is A-D.
|
||||
Two naming conventions exist:
|
||||
- 2021+: {YEAR}_TEOS_XML_{MM}{L}.zip (e.g. 2023_TEOS_XML_05A)
|
||||
- 2017-2020: download990xml_{YEAR}_{N}.zip (e.g. download990xml_2020_1)
|
||||
Also checks for _CT1 (correction/catchup) batches.
|
||||
"""
|
||||
batches = []
|
||||
with httpx.Client(follow_redirects=True, timeout=30.0) as client:
|
||||
# Modern pattern (2021+)
|
||||
for month in range(1, 13):
|
||||
for letter in "ABCD":
|
||||
batch_id = f"{year}_TEOS_XML_{month:02d}{letter}"
|
||||
|
|
@ -72,9 +76,61 @@ def get_batch_list(year: int) -> list[str]:
|
|||
batches.append(batch_id)
|
||||
except httpx.HTTPError:
|
||||
continue
|
||||
|
||||
# Legacy pattern (2017-2020)
|
||||
for n in range(1, 20):
|
||||
batch_id = f"download990xml_{year}_{n}"
|
||||
url = f"{config.IRS990_BASE_URL}/{year}/{batch_id}.zip"
|
||||
try:
|
||||
resp = client.head(url)
|
||||
if resp.status_code == 200:
|
||||
batches.append(batch_id)
|
||||
else:
|
||||
break # sequential numbering, stop at first miss
|
||||
except httpx.HTTPError:
|
||||
break
|
||||
|
||||
# Correction/catchup batch
|
||||
ct_id = f"{year}_TEOS_XML_CT1"
|
||||
url = f"{config.IRS990_BASE_URL}/{year}/{ct_id}.zip"
|
||||
try:
|
||||
resp = client.head(url)
|
||||
if resp.status_code == 200:
|
||||
batches.append(ct_id)
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
|
||||
return batches
|
||||
|
||||
|
||||
def _extract_with_system_unzip(
|
||||
zip_path: Path, target_name: str, dest: Path
|
||||
) -> bool:
|
||||
"""Extract a file from a ZIP using the system 7z or unzip command."""
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
# Try 7z first (handles more compression methods), then unzip
|
||||
for cmd in [
|
||||
["7z", "x", str(zip_path), f"-o{tmp_dir}", f"*{target_name}", "-y"],
|
||||
["unzip", "-o", "-j", str(zip_path), f"*{target_name}", "-d", tmp_dir],
|
||||
]:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, timeout=300
|
||||
)
|
||||
if result.returncode == 0:
|
||||
extracted = list(Path(tmp_dir).rglob(target_name))
|
||||
if extracted:
|
||||
import shutil
|
||||
shutil.move(str(extracted[0]), str(dest))
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _extract_xml_from_zip(
|
||||
zip_url: str, object_id: str, dest: Path
|
||||
) -> bool:
|
||||
|
|
@ -96,20 +152,27 @@ def _extract_xml_from_zip(
|
|||
for chunk in resp.iter_bytes(chunk_size=1024 * 1024):
|
||||
f.write(chunk)
|
||||
|
||||
with zipfile.ZipFile(tmp_path) as zf:
|
||||
names = zf.namelist()
|
||||
# Look for our target file (case-insensitive)
|
||||
match = None
|
||||
for name in names:
|
||||
if name.lower() == target_name.lower():
|
||||
match = name
|
||||
break
|
||||
if match is None:
|
||||
return False
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(match) as src, open(dest, "wb") as dst:
|
||||
dst.write(src.read())
|
||||
return True
|
||||
try:
|
||||
with zipfile.ZipFile(tmp_path) as zf:
|
||||
names = zf.namelist()
|
||||
# Look for our target file — may be in a subdirectory within the ZIP
|
||||
# e.g. "2024_TEOS_XML_04A/202421029349301032_public.xml"
|
||||
match = None
|
||||
for name in names:
|
||||
basename = name.rsplit("/", 1)[-1] if "/" in name else name
|
||||
if basename.lower() == target_name.lower():
|
||||
match = name
|
||||
break
|
||||
if match is None:
|
||||
return False
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(match) as src, open(dest, "wb") as dst:
|
||||
dst.write(src.read())
|
||||
return True
|
||||
except NotImplementedError:
|
||||
# Unsupported compression (e.g. Zstandard in newer ZIPs).
|
||||
# Fall back to system unzip.
|
||||
return _extract_with_system_unzip(tmp_path, target_name, dest)
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
|
|
@ -136,8 +199,9 @@ def download_filing_xml(
|
|||
return dest
|
||||
|
||||
if batch_id:
|
||||
# We know which ZIP to look in
|
||||
url = f"{config.IRS990_BASE_URL}/{year}/{batch_id}.zip"
|
||||
# We know which ZIP to look in. Index may have lowercase but URLs need uppercase.
|
||||
batch_id_upper = batch_id.strip().upper()
|
||||
url = f"{config.IRS990_BASE_URL}/{year}/{batch_id_upper}.zip"
|
||||
if _extract_xml_from_zip(url, object_id, dest):
|
||||
return dest
|
||||
return None
|
||||
|
|
|
|||
121
src/admin_analytics/scraper/classify.py
Normal file
121
src/admin_analytics/scraper/classify.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
"""Classify staff by title into functional categories.
|
||||
|
||||
Categories distinguish admin overhead from grant-funded, student-facing,
|
||||
and technical roles. This is critical because IPEDS/IRE "staff" counts
|
||||
lump everyone who isn't tenure-track faculty — including postdocs,
|
||||
research scientists, and lab technicians who are soft-funded through
|
||||
extramural research and are NOT administrative overhead.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
# Order matters — first match wins.
|
||||
CATEGORY_PATTERNS: list[tuple[str, re.Pattern]] = [
|
||||
# Leadership / structural overhead
|
||||
("LEADERSHIP", re.compile(
|
||||
r"\b(dean|chief of staff|associate dean|assistant dean)\b", re.I)),
|
||||
|
||||
# Faculty listed on staff page — not admin (must come before RESEARCH
|
||||
# to catch "Adjunct Professor NIST" as faculty, not research)
|
||||
("FACULTY", re.compile(
|
||||
r"\b(professor|lecturer|instructor|faculty|adjunct|affiliated)", re.I)),
|
||||
|
||||
# Grants administration — debatable; supports extramural funding
|
||||
("GRANTS_ADMIN", re.compile(
|
||||
r"\b(grants?\s+anal|pre-?award|post-?award|closeout\s+coord"
|
||||
r"|sponsored\s+program|grants?\s+admin|grants?\s+manag|grants?\s+coord)", re.I)),
|
||||
|
||||
# Research staff — soft-funded, NOT admin bloat
|
||||
("RESEARCH", re.compile(
|
||||
r"\b(research\s+(?:associate|assistant|scientist|scholar|fellow)"
|
||||
r"|postdoc|post-?doctoral|(?:associate\s+)?scientist\b)", re.I)),
|
||||
|
||||
# Academic / student-facing support
|
||||
("ACADEMIC_SUPPORT", re.compile(
|
||||
r"\b(academic\s+(?:advisor|analyst|program)|undergrad\w*\s+(?:recruit|advisor|affairs)"
|
||||
r"|graduate\s+(?:services|advisor)|student\s+(?:develop|support|services)"
|
||||
r"|program\s+(?:coordinator|manager))", re.I)),
|
||||
|
||||
# Advancement / development — revenue-generating (fundraising)
|
||||
("ADVANCEMENT", re.compile(
|
||||
r"\b(development|fundrais|advancement|alumni\s+relation|donor|giving)", re.I)),
|
||||
|
||||
# Finance / procurement
|
||||
("FINANCE", re.compile(
|
||||
r"\b(financial|fiscal|budget|procurement|business\s+(?:officer|admin)"
|
||||
r"|sr\.?\s+business)", re.I)),
|
||||
|
||||
# IT / computing
|
||||
("IT", re.compile(
|
||||
r"\b(computing|systems?\s+(?:prog|admin)|it\s+|information\s+tech"
|
||||
r"|support\s+specialist|service\s+desk|digital\s+tech)", re.I)),
|
||||
|
||||
# Communications / marketing (must come before DIRECTOR)
|
||||
("COMMUNICATIONS", re.compile(
|
||||
r"\b(communicat\w+|marketing|media\s+(?:specialist|coord|director)"
|
||||
r"|web\s+(?:develop|design|content)|event\s+(?:coord|plan|manag))", re.I)),
|
||||
|
||||
# Human resources
|
||||
("HR", re.compile(
|
||||
r"\b(human\s+resource|hr\s+analyst|talent|workforce)", re.I)),
|
||||
|
||||
# Facilities / space management
|
||||
("FACILITIES", re.compile(
|
||||
r"\b(facilit|building|space\s+(?:plan|manag)|safety|engineer\w+\s+facilit)", re.I)),
|
||||
|
||||
# Technical / lab operations — not admin bloat
|
||||
("TECHNICAL", re.compile(
|
||||
r"\b(machinist|lab\s+(?:manager|coord|tech)|equipment|technician"
|
||||
r"|instrument)", re.I)),
|
||||
|
||||
# Administrative support
|
||||
("ADMIN_SUPPORT", re.compile(
|
||||
r"\b(admin\w*\s+(?:assistant|specialist|support|secretary|coord)"
|
||||
r"|secretary|receptionist|office\s+(?:manager|coord))", re.I)),
|
||||
|
||||
# Director-level (catch remaining directors)
|
||||
("DIRECTOR", re.compile(
|
||||
r"\b(director|associate\s+director|sr\.?\s+director)\b", re.I)),
|
||||
]
|
||||
|
||||
# Which categories count as administrative overhead
|
||||
OVERHEAD_CATEGORIES = {
|
||||
"LEADERSHIP", "FINANCE", "IT", "COMMUNICATIONS", "HR",
|
||||
"FACILITIES", "ADMIN_SUPPORT", "DIRECTOR",
|
||||
}
|
||||
|
||||
# Debatable — could go either way depending on analysis
|
||||
DEBATABLE_CATEGORIES = {"GRANTS_ADMIN"}
|
||||
|
||||
# NOT overhead — these are mission-aligned or revenue-generating
|
||||
NON_OVERHEAD_CATEGORIES = {
|
||||
"RESEARCH", "ACADEMIC_SUPPORT", "ADVANCEMENT", "TECHNICAL", "FACULTY",
|
||||
}
|
||||
|
||||
|
||||
def classify_title(title: str | None) -> str:
|
||||
"""Classify a staff title into a functional category.
|
||||
|
||||
Returns the category string, or "UNKNOWN" if no pattern matches.
|
||||
"""
|
||||
if not title or not title.strip():
|
||||
return "UNKNOWN"
|
||||
|
||||
for category, pattern in CATEGORY_PATTERNS:
|
||||
if pattern.search(title):
|
||||
return category
|
||||
|
||||
return "UNKNOWN"
|
||||
|
||||
|
||||
def is_overhead(category: str) -> bool | None:
|
||||
"""Return True if the category is administrative overhead,
|
||||
False if not, None if debatable.
|
||||
"""
|
||||
if category in OVERHEAD_CATEGORIES:
|
||||
return True
|
||||
if category in NON_OVERHEAD_CATEGORIES:
|
||||
return False
|
||||
if category in DEBATABLE_CATEGORIES:
|
||||
return None
|
||||
return None
|
||||
292
src/admin_analytics/scraper/directory.py
Normal file
292
src/admin_analytics/scraper/directory.py
Normal file
|
|
@ -0,0 +1,292 @@
|
|||
"""Scrape UD staff directory pages for headcounts and title classification.
|
||||
|
||||
Two rendering modes:
|
||||
- Static (httpx): for pages that embed staff data in HTML source
|
||||
- Dynamic (Playwright): for JS-rendered pages (Divi/WordPress AJAX)
|
||||
|
||||
The scraper targets the Engineering management line:
|
||||
Department staff → COE Central → Provost → President
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from admin_analytics.scraper.classify import classify_title
|
||||
|
||||
|
||||
@dataclass
|
||||
class StaffEntry:
|
||||
name: str
|
||||
title: str | None
|
||||
email: str | None
|
||||
category: str = ""
|
||||
unit: str = ""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.category and self.title:
|
||||
self.category = classify_title(self.title)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DirectoryPage:
|
||||
"""Configuration for a single directory page to scrape."""
|
||||
unit: str # e.g. "COE Central", "ME", "Provost Office"
|
||||
url: str
|
||||
requires_js: bool = False # True = needs Playwright
|
||||
|
||||
|
||||
# UD College of Engineering management line + departments
|
||||
DIRECTORY_PAGES = [
|
||||
# COE Central admin (JS-rendered)
|
||||
DirectoryPage("COE Central", "https://engr.udel.edu/our-people/directory/", requires_js=True),
|
||||
# Engineering departments
|
||||
DirectoryPage("ME", "https://me.udel.edu/people/staff/", requires_js=True),
|
||||
DirectoryPage("CBE", "https://cbe.udel.edu/people/staff/", requires_js=False),
|
||||
DirectoryPage("ECE", "https://www.ece.udel.edu/people/staff/", requires_js=False),
|
||||
DirectoryPage("CIS", "https://www.cis.udel.edu/people/staff/", requires_js=False),
|
||||
DirectoryPage("BME", "https://bme.udel.edu/people/staff/", requires_js=True),
|
||||
DirectoryPage("CCEE", "https://www.ccee.udel.edu/people/staff/", requires_js=False),
|
||||
DirectoryPage("MSE", "https://mseg.udel.edu/people/staff/", requires_js=True),
|
||||
# Management line above Engineering
|
||||
DirectoryPage("Provost Office", "https://provost.udel.edu/about/our-staff/", requires_js=False),
|
||||
DirectoryPage("President Office", "https://www.udel.edu/about/leadership/", requires_js=False),
|
||||
]
|
||||
|
||||
|
||||
def _extract_from_rendered_text(text: str, unit: str) -> list[StaffEntry]:
|
||||
"""Extract staff entries from rendered page text.
|
||||
|
||||
The JS-rendered Divi pages display staff as:
|
||||
Name
|
||||
Title
|
||||
[Phone]
|
||||
email@udel.edu
|
||||
|
||||
We anchor on email addresses and look backwards for name/title.
|
||||
"""
|
||||
lines = [line.strip() for line in text.split("\n") if line.strip()]
|
||||
entries: list[StaffEntry] = []
|
||||
seen_emails: set[str] = set()
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
emails = re.findall(r"[\w.-]+@udel\.edu", line)
|
||||
for email in emails:
|
||||
if email in seen_emails:
|
||||
continue
|
||||
# Skip generic/departmental emails
|
||||
if any(k in email for k in ["-info", "dept", "help", "college", "engr@", "coe-"]):
|
||||
continue
|
||||
seen_emails.add(email)
|
||||
|
||||
# Look backwards for name and title
|
||||
name, title = _find_name_title(lines, i)
|
||||
if name:
|
||||
entries.append(StaffEntry(
|
||||
name=name, title=title, email=email, unit=unit,
|
||||
))
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def _find_name_title(lines: list[str], email_idx: int) -> tuple[str | None, str | None]:
|
||||
"""Look backwards from an email line to find the person's name and title."""
|
||||
name = None
|
||||
title = None
|
||||
|
||||
# Title keywords that distinguish a title line from a name line
|
||||
title_keywords = re.compile(
|
||||
r"(director|manager|specialist|advisor|analyst|coordinator|assistant|associate"
|
||||
r"|officer|chief|dean|admin|support|computing|systems|financial|grants?"
|
||||
r"|machinist|lab |academic|program|human|facilit|communicat|development"
|
||||
r"|procurement|sr\.|senior|recruit|budget|research|scientist|engineer"
|
||||
r"|technician|professor|lecturer)", re.I,
|
||||
)
|
||||
|
||||
for back in range(1, 6):
|
||||
idx = email_idx - back
|
||||
if idx < 0:
|
||||
break
|
||||
prev = lines[idx].strip()
|
||||
|
||||
# Stop if we hit another email (previous person's entry)
|
||||
if "@udel.edu" in prev:
|
||||
break
|
||||
|
||||
# Skip phone numbers
|
||||
if re.match(r"^[\d\-\(\)\s]{7,}$", prev):
|
||||
continue
|
||||
|
||||
# Is this a title or a name?
|
||||
if title is None and title_keywords.search(prev):
|
||||
title = prev
|
||||
elif name is None and re.match(r"^[A-Z][a-z]", prev) and len(prev) < 50:
|
||||
# Looks like a name (starts with capital, not too long)
|
||||
if not title_keywords.search(prev):
|
||||
name = prev
|
||||
elif title is None:
|
||||
# It has title keywords — it's actually a title
|
||||
title = prev
|
||||
else:
|
||||
name = prev
|
||||
|
||||
if name and title:
|
||||
break
|
||||
|
||||
return name, title
|
||||
|
||||
|
||||
def _extract_from_html(html_text: str, unit: str) -> list[StaffEntry]:
|
||||
"""Extract staff from static HTML source using email anchoring."""
|
||||
# Strip HTML tags to get plain text
|
||||
text = re.sub(r"<[^>]+>", " ", html_text)
|
||||
text = re.sub(r"&\w+;", " ", text)
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
entries: list[StaffEntry] = []
|
||||
seen_emails: set[str] = set()
|
||||
|
||||
for m in re.finditer(r"[\w.-]+@udel\.edu", text):
|
||||
email = m.group()
|
||||
if email in seen_emails:
|
||||
continue
|
||||
if any(k in email for k in ["-info", "dept", "help", "college", "engr@", "coe-"]):
|
||||
continue
|
||||
seen_emails.add(email)
|
||||
|
||||
# Get context before the email
|
||||
start = max(0, m.start() - 300)
|
||||
ctx = text[start:m.start()]
|
||||
|
||||
# Try to extract name + title from context
|
||||
name, title = _parse_context_for_name_title(ctx)
|
||||
if name:
|
||||
entries.append(StaffEntry(
|
||||
name=name, title=title, email=email, unit=unit,
|
||||
))
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def _parse_context_for_name_title(ctx: str) -> tuple[str | None, str | None]:
|
||||
"""Parse a text context block (before an email) for name and title."""
|
||||
# Pattern: Name Title [Location] [Phone]
|
||||
m = re.search(
|
||||
r"([A-Z][a-z]+(?:\s[A-Z]\.?)?\s(?:Mc|De|Van|La)?[A-Z][a-z]+(?:-[A-Z][a-z]+)?)\s+"
|
||||
r"([A-Za-z/,.& ]+?(?:I{1,3}|IV|V)?)\s+"
|
||||
r"(?:\d+\s+\w|\d{3}-)",
|
||||
ctx,
|
||||
)
|
||||
if m:
|
||||
return m.group(1).strip(), m.group(2).strip()
|
||||
|
||||
# Fallback: just find the last proper name in the context
|
||||
names = re.findall(
|
||||
r"([A-Z][a-z]+(?:\s[A-Z]\.?)?\s(?:Mc|De|Van|La)?[A-Z][a-z]+(?:-[A-Z][a-z]+)?)",
|
||||
ctx,
|
||||
)
|
||||
if names:
|
||||
return names[-1], None
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def _extract_provost_staff(html_text: str) -> list[StaffEntry]:
|
||||
"""Custom parser for the Provost office page which has a distinct structure."""
|
||||
from lxml import html as lxml_html
|
||||
|
||||
tree = lxml_html.fromstring(html_text)
|
||||
entries: list[StaffEntry] = []
|
||||
|
||||
# Leadership: h2 tags contain titles, strong tags contain names
|
||||
h2s = tree.xpath("//h2")
|
||||
strongs = tree.xpath("//strong")
|
||||
|
||||
strong_texts = [s.text_content().strip() for s in strongs if s.text_content().strip()]
|
||||
|
||||
for h2 in h2s:
|
||||
title = h2.text_content().strip()
|
||||
if not title or title in ("Administrative Staff", "Staff", "Quick Links", "Recent News"):
|
||||
continue
|
||||
# The name follows the h2 in a nearby strong tag or paragraph
|
||||
# Match by position — h2 title, then the next unmatched strong is the name
|
||||
if strong_texts:
|
||||
name = strong_texts.pop(0)
|
||||
entries.append(StaffEntry(
|
||||
name=name, title=title, email=None, unit="Provost Office",
|
||||
))
|
||||
|
||||
# Administrative staff section: "Title, Name" format in the remaining strongs
|
||||
for st in strong_texts:
|
||||
if "," in st:
|
||||
parts = st.split(",", 1)
|
||||
title = parts[0].strip()
|
||||
name = parts[1].strip()
|
||||
entries.append(StaffEntry(
|
||||
name=name, title=title, email=None, unit="Provost Office",
|
||||
))
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def scrape_static(page: DirectoryPage) -> list[StaffEntry]:
|
||||
"""Scrape a static HTML page using httpx."""
|
||||
with httpx.Client(
|
||||
follow_redirects=True,
|
||||
timeout=30.0,
|
||||
headers={"User-Agent": "Mozilla/5.0 (compatible; UDAdminAnalytics/0.1)"},
|
||||
) as client:
|
||||
resp = client.get(page.url)
|
||||
resp.raise_for_status()
|
||||
|
||||
if page.unit == "Provost Office":
|
||||
return _extract_provost_staff(resp.text)
|
||||
|
||||
return _extract_from_html(resp.text, page.unit)
|
||||
|
||||
|
||||
def scrape_dynamic(page: DirectoryPage) -> list[StaffEntry]:
|
||||
"""Scrape a JS-rendered page using Playwright."""
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
pw_page = browser.new_page()
|
||||
pw_page.goto(page.url, wait_until="networkidle", timeout=45000)
|
||||
text = pw_page.inner_text("body")
|
||||
browser.close()
|
||||
|
||||
return _extract_from_rendered_text(text, page.unit)
|
||||
|
||||
|
||||
def scrape_page(page: DirectoryPage) -> list[StaffEntry]:
|
||||
"""Scrape a single directory page, choosing the right method."""
|
||||
if page.requires_js:
|
||||
return scrape_dynamic(page)
|
||||
return scrape_static(page)
|
||||
|
||||
|
||||
def scrape_all(pages: list[DirectoryPage] | None = None) -> list[StaffEntry]:
|
||||
"""Scrape all configured directory pages.
|
||||
|
||||
Returns a flat list of all staff entries across all units.
|
||||
"""
|
||||
pages = pages or DIRECTORY_PAGES
|
||||
all_entries: list[StaffEntry] = []
|
||||
|
||||
for page in pages:
|
||||
method = "Playwright" if page.requires_js else "httpx"
|
||||
print(f" Scraping {page.unit} ({method})...")
|
||||
try:
|
||||
entries = scrape_page(page)
|
||||
print(f" Found {len(entries)} staff")
|
||||
all_entries.extend(entries)
|
||||
except Exception as e:
|
||||
print(f" Error: {e}")
|
||||
|
||||
return all_entries
|
||||
49
src/admin_analytics/scraper/loader.py
Normal file
49
src/admin_analytics/scraper/loader.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
"""Load scraped staff directory data into DuckDB."""
|
||||
|
||||
from datetime import date
|
||||
|
||||
import duckdb
|
||||
|
||||
from admin_analytics.scraper.classify import is_overhead
|
||||
from admin_analytics.scraper.directory import StaffEntry
|
||||
|
||||
|
||||
def load_scrape(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
entries: list[StaffEntry],
|
||||
scrape_date: date | None = None,
|
||||
) -> int:
|
||||
"""Load scraped staff entries into raw_admin_headcount.
|
||||
|
||||
Clears previous data for the same scrape_date before inserting,
|
||||
making re-runs idempotent.
|
||||
|
||||
Returns number of rows inserted.
|
||||
"""
|
||||
scrape_date = scrape_date or date.today()
|
||||
|
||||
conn.execute(
|
||||
"DELETE FROM raw_admin_headcount WHERE scrape_date = ?",
|
||||
[scrape_date],
|
||||
)
|
||||
|
||||
count = 0
|
||||
for entry in entries:
|
||||
overhead = is_overhead(entry.category)
|
||||
conn.execute(
|
||||
"""INSERT INTO raw_admin_headcount
|
||||
(scrape_date, unit, person_name, title, email, category, is_overhead)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
||||
[
|
||||
scrape_date,
|
||||
entry.unit,
|
||||
entry.name,
|
||||
entry.title,
|
||||
entry.email,
|
||||
entry.category,
|
||||
overhead,
|
||||
],
|
||||
)
|
||||
count += 1
|
||||
|
||||
return count
|
||||
168
src/admin_analytics/validation.py
Normal file
168
src/admin_analytics/validation.py
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
"""Data validation queries for the raw data layer."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import duckdb
|
||||
|
||||
from admin_analytics.config import DEFAULT_YEAR_RANGE
|
||||
from admin_analytics.db.schema import TABLES
|
||||
|
||||
# Key columns to check for NULLs per table
|
||||
KEY_COLUMNS: dict[str, list[str]] = {
|
||||
"raw_institution": ["unitid", "year", "institution_name"],
|
||||
"raw_ipeds_finance": ["unitid", "year", "total_expenses"],
|
||||
"raw_ipeds_staff": ["unitid", "year", "total_staff"],
|
||||
"raw_ipeds_enrollment": ["unitid", "year", "total_enrollment"],
|
||||
"raw_990_filing": ["ein", "tax_year", "total_revenue", "total_expenses"],
|
||||
"raw_990_part_vii": ["ein", "tax_year", "person_name", "reportable_comp_from_org"],
|
||||
"raw_990_schedule_j": ["ein", "tax_year", "person_name", "total_compensation"],
|
||||
"raw_cpi_u": ["year", "month", "value"],
|
||||
"raw_admin_headcount": ["unit", "person_name", "category"],
|
||||
}
|
||||
|
||||
# Year column name per table (some use tax_year, some have no year)
|
||||
YEAR_COLUMN: dict[str, str] = {
|
||||
"raw_institution": "year",
|
||||
"raw_ipeds_finance": "year",
|
||||
"raw_ipeds_staff": "year",
|
||||
"raw_ipeds_enrollment": "year",
|
||||
"raw_990_filing": "tax_year",
|
||||
"raw_990_part_vii": "tax_year",
|
||||
"raw_990_schedule_j": "tax_year",
|
||||
"raw_cpi_u": "year",
|
||||
}
|
||||
|
||||
|
||||
def validate_row_counts(conn: duckdb.DuckDBPyConnection) -> dict[str, int]:
|
||||
"""Return row count per raw table."""
|
||||
counts = {}
|
||||
for table in TABLES:
|
||||
row = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()
|
||||
counts[table] = row[0]
|
||||
return counts
|
||||
|
||||
|
||||
def validate_null_rates(conn: duckdb.DuckDBPyConnection) -> dict[str, dict[str, float]]:
|
||||
"""Return NULL percentage for key columns per table.
|
||||
|
||||
Only tables with rows are included. Returns {table: {column: pct}}.
|
||||
"""
|
||||
results: dict[str, dict[str, float]] = {}
|
||||
for table, columns in KEY_COLUMNS.items():
|
||||
total = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
|
||||
if total == 0:
|
||||
continue
|
||||
rates: dict[str, float] = {}
|
||||
for col in columns:
|
||||
null_count = conn.execute(
|
||||
f"SELECT COUNT(*) FROM {table} WHERE {col} IS NULL"
|
||||
).fetchone()[0]
|
||||
rates[col] = round(null_count * 100.0 / total, 1)
|
||||
results[table] = rates
|
||||
return results
|
||||
|
||||
|
||||
def validate_year_coverage(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Return year coverage info per table.
|
||||
|
||||
Returns {table: {"years": [...], "gaps": [...]}}.
|
||||
"""
|
||||
results: dict[str, dict[str, Any]] = {}
|
||||
expected = set(DEFAULT_YEAR_RANGE)
|
||||
|
||||
for table, col in YEAR_COLUMN.items():
|
||||
rows = conn.execute(
|
||||
f"SELECT DISTINCT {col} FROM {table} ORDER BY {col}"
|
||||
).fetchall()
|
||||
years = [r[0] for r in rows]
|
||||
if not years:
|
||||
results[table] = {"years": [], "gaps": sorted(expected)}
|
||||
continue
|
||||
present = set(years)
|
||||
gaps = sorted(expected - present)
|
||||
results[table] = {"years": years, "gaps": gaps}
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def validate_cross_source_consistency(
|
||||
conn: duckdb.DuckDBPyConnection,
|
||||
) -> dict[str, Any]:
|
||||
"""Check year overlap across IPEDS, IRS 990, and CPI sources."""
|
||||
sources = {
|
||||
"ipeds_finance": ("raw_ipeds_finance", "year"),
|
||||
"irs990": ("raw_990_filing", "tax_year"),
|
||||
"cpi": ("raw_cpi_u", "year"),
|
||||
}
|
||||
|
||||
year_sets: dict[str, set[int]] = {}
|
||||
for key, (table, col) in sources.items():
|
||||
rows = conn.execute(
|
||||
f"SELECT DISTINCT {col} FROM {table}"
|
||||
).fetchall()
|
||||
year_sets[key] = {r[0] for r in rows}
|
||||
|
||||
non_empty = {k: v for k, v in year_sets.items() if v}
|
||||
if len(non_empty) < 2:
|
||||
all_years: set[int] = set()
|
||||
else:
|
||||
all_years = set.intersection(*non_empty.values())
|
||||
|
||||
return {
|
||||
"source_years": {k: sorted(v) for k, v in year_sets.items()},
|
||||
"years_in_all_sources": sorted(all_years),
|
||||
}
|
||||
|
||||
|
||||
def format_report(conn: duckdb.DuckDBPyConnection) -> str:
|
||||
"""Generate a full text validation report."""
|
||||
lines: list[str] = []
|
||||
|
||||
# Row counts
|
||||
lines.append("=== Row Counts ===")
|
||||
counts = validate_row_counts(conn)
|
||||
for table, count in counts.items():
|
||||
lines.append(f" {table:<30s} {count:>8d}")
|
||||
|
||||
# NULL rates
|
||||
lines.append("\n=== NULL Rates (%) ===")
|
||||
nulls = validate_null_rates(conn)
|
||||
if not nulls:
|
||||
lines.append(" (no data loaded)")
|
||||
for table, rates in nulls.items():
|
||||
lines.append(f" {table}:")
|
||||
for col, pct in rates.items():
|
||||
flag = " !" if pct > 0 else ""
|
||||
lines.append(f" {col:<35s} {pct:>5.1f}%{flag}")
|
||||
|
||||
# Year coverage
|
||||
lines.append("\n=== Year Coverage ===")
|
||||
coverage = validate_year_coverage(conn)
|
||||
for table, info in coverage.items():
|
||||
years = info["years"]
|
||||
gaps = info["gaps"]
|
||||
if not years:
|
||||
lines.append(f" {table:<30s} no data")
|
||||
continue
|
||||
yr_range = f"{min(years)}-{max(years)} ({len(years)} years)"
|
||||
lines.append(f" {table:<30s} {yr_range}")
|
||||
if gaps:
|
||||
lines.append(f" gaps: {gaps}")
|
||||
|
||||
# Cross-source
|
||||
lines.append("\n=== Cross-Source Consistency ===")
|
||||
cs = validate_cross_source_consistency(conn)
|
||||
for source, years in cs["source_years"].items():
|
||||
if years:
|
||||
lines.append(f" {source:<20s} {min(years)}-{max(years)} ({len(years)} years)")
|
||||
else:
|
||||
lines.append(f" {source:<20s} no data")
|
||||
overlap = cs["years_in_all_sources"]
|
||||
if overlap:
|
||||
lines.append(f" Years in all sources: {min(overlap)}-{max(overlap)} ({len(overlap)} years)")
|
||||
else:
|
||||
lines.append(" Years in all sources: none")
|
||||
|
||||
return "\n".join(lines)
|
||||
Loading…
Add table
Add a link
Reference in a new issue