Updates after testing

This commit is contained in:
emfurst 2026-03-30 20:42:08 -04:00
commit 04095c9970
15 changed files with 238 additions and 33 deletions

View file

@ -23,7 +23,7 @@ def download_cpi_file(force: bool = False) -> Path:
with httpx.Client(
follow_redirects=True,
timeout=120.0,
headers={"User-Agent": "admin-analytics/0.1 (research)"},
headers={"User-Agent": "admin-analytics/0.1 (research@udel.edu)"},
) as client:
resp = client.get(BLS_CPI_URL)
resp.raise_for_status()

View file

@ -28,11 +28,12 @@ def load_cpi(conn: duckdb.DuckDBPyConnection, file_path: Path) -> int:
# Normalize column names (BLS has trailing spaces in headers too)
df = df.rename({col: col.strip() for col in df.columns})
# Filter to CPI-U series and monthly periods only
# Filter to CPI-U series and monthly periods only, drop missing values
df = df.filter(
(pl.col("series_id") == "CUUR0000SA0")
& (pl.col("period").str.starts_with("M"))
& (pl.col("period") != "M13")
& (pl.col("value") != "-")
)
# Extract month from period (M01 -> 1, M12 -> 12)

View file

@ -80,8 +80,8 @@ def ipeds(
@ingest_app.command()
def irs990(
year_range: Annotated[
str, typer.Option(help="Year range for index files, e.g. 2019-2024")
] = "2019-2024",
str, typer.Option(help="Year range for index files, e.g. 2019-2025")
] = "2019-2025",
force: Annotated[
bool, typer.Option("--force", help="Re-download even if files exist")
] = False,
@ -163,6 +163,7 @@ def scrape() -> None:
@app.command()
def dashboard(
host: Annotated[str, typer.Option(help="Host to bind to (0.0.0.0 for network access)")] = "127.0.0.1",
port: Annotated[int, typer.Option(help="Port to serve on")] = 8050,
debug: Annotated[bool, typer.Option(help="Enable Dash debug mode")] = True,
) -> None:
@ -170,8 +171,8 @@ def dashboard(
from admin_analytics.dashboard.app import create_app
dash_app = create_app()
typer.echo(f"Starting dashboard at http://localhost:{port}/")
dash_app.run(debug=debug, port=port)
typer.echo(f"Starting dashboard at http://{host}:{port}/")
dash_app.run(host=host, debug=debug, port=port)
@app.command()

View file

@ -4,15 +4,13 @@ import dash
from dash import dcc, html, Input, Output
from admin_analytics.db.connection import get_connection
from admin_analytics.db.schema import ensure_schema
from admin_analytics.dashboard.pages import overview, compensation, staffing, headcount
def create_app() -> dash.Dash:
"""Create and configure the Dash application."""
app = dash.Dash(__name__, suppress_callback_exceptions=True)
conn = get_connection()
ensure_schema(conn)
conn = get_connection(read_only=True)
app.layout = html.Div(
[
@ -22,10 +20,10 @@ def create_app() -> dash.Dash:
),
dcc.Tabs(
id="tabs",
value="overview",
value="compensation",
children=[
dcc.Tab(label="Admin Cost Overview", value="overview"),
dcc.Tab(label="Executive Compensation", value="compensation"),
dcc.Tab(label="Admin Cost Overview", value="overview"),
dcc.Tab(label="Staffing & Enrollment", value="staffing"),
dcc.Tab(label="Current Headcount", value="headcount"),
],

View file

@ -113,6 +113,4 @@ def layout(conn: duckdb.DuckDBPyConnection):
style={"display": "flex", "gap": "16px"},
),
dcc.Graph(figure=cat_fig),
html.H3("Staff Directory Detail", style={"marginTop": "24px"}),
table,
])

View file

@ -73,6 +73,11 @@ def layout(conn: duckdb.DuckDBPyConnection):
mode="lines+markers", name="Management Growth",
line={"color": "#E07A5F"},
))
growth_fig.add_trace(go.Scatter(
x=growth_pd["year"], y=growth_pd["faculty_index"],
mode="lines+markers", name="Faculty Growth",
line={"color": "#7FB069"},
))
growth_fig.add_trace(go.Scatter(
x=growth_pd["year"], y=growth_pd["enrollment_index"],
mode="lines+markers", name="Enrollment Growth",
@ -80,7 +85,7 @@ def layout(conn: duckdb.DuckDBPyConnection):
))
growth_fig.add_hline(y=100, line_dash="dot", line_color="#ccc")
growth_fig.update_layout(
title="Management vs Enrollment Growth (Indexed, Base Year = 100)",
title="Management vs Faculty vs Enrollment Growth (Indexed, Base Year = 100)",
xaxis_title="Year", yaxis_title="Index",
template="plotly_white", height=380,
)

View file

@ -221,10 +221,12 @@ def query_student_staff_ratios(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
def query_growth_index(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
"""Management vs enrollment growth, indexed to first year = 100."""
"""Management, faculty, and enrollment growth, indexed to first year = 100."""
return conn.execute("""
WITH base AS (
SELECT s.management_total AS base_mgmt, e.total_enrollment AS base_enrl
SELECT s.management_total AS base_mgmt,
s.faculty_total AS base_fac,
e.total_enrollment AS base_enrl
FROM raw_ipeds_staff s
JOIN raw_ipeds_enrollment e ON e.unitid = s.unitid AND e.year = s.year
WHERE s.unitid = ?
@ -232,9 +234,12 @@ def query_growth_index(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame:
)
SELECT s.year,
s.management_total,
s.faculty_total,
e.total_enrollment,
ROUND(s.management_total * 100.0
/ NULLIF((SELECT base_mgmt FROM base), 0), 1) AS mgmt_index,
ROUND(s.faculty_total * 100.0
/ NULLIF((SELECT base_fac FROM base), 0), 1) AS faculty_index,
ROUND(e.total_enrollment * 100.0
/ NULLIF((SELECT base_enrl FROM base), 0), 1) AS enrollment_index
FROM raw_ipeds_staff s

View file

@ -5,8 +5,10 @@ import duckdb
from admin_analytics.config import DB_PATH
def get_connection(path: Path | None = None) -> duckdb.DuckDBPyConnection:
def get_connection(
path: Path | None = None, read_only: bool = False
) -> duckdb.DuckDBPyConnection:
"""Return a DuckDB connection, creating the parent directory if needed."""
path = path or DB_PATH
path.parent.mkdir(parents=True, exist_ok=True)
return duckdb.connect(str(path))
return duckdb.connect(str(path), read_only=read_only)

View file

@ -22,10 +22,14 @@ def load_staff(
"""Load IPEDS Fall Staff data into raw_ipeds_staff.
The S_OC file has multiple rows per institution (by occupational category).
OCCUPCAT definitions:
100 = All staff
200 = Instructional, research and public service staff (faculty)
300 = Management
We extract:
- total_staff: FTPT=2, OCCUPCAT=100 (full-time total, all occupations)
- management_total: FTPT=2, OCCUPCAT=200 (full-time management)
- faculty_total: FTPT=2, OCCUPCAT=250 (full-time instruction/research)
- total_staff: FTPT=2, OCCUPCAT=100 (full-time total)
- faculty_total: FTPT=2, OCCUPCAT=200 (full-time instruction/research/public service)
- management_total: FTPT=2, OCCUPCAT=300 (full-time management)
"""
total = 0
for year in year_range:
@ -62,8 +66,8 @@ def load_staff(
# Pivot occupational categories into columns per UNITID
total_staff = ft.filter(pl.col("_occupcat") == "100").select("unitid", pl.col("_count").alias("total_staff"))
management = ft.filter(pl.col("_occupcat") == "200").select("unitid", pl.col("_count").alias("management_total"))
faculty = ft.filter(pl.col("_occupcat") == "250").select("unitid", pl.col("_count").alias("faculty_total"))
faculty = ft.filter(pl.col("_occupcat") == "200").select("unitid", pl.col("_count").alias("faculty_total"))
management = ft.filter(pl.col("_occupcat") == "300").select("unitid", pl.col("_count").alias("management_total"))
result = total_staff.join(management, on="unitid", how="left").join(faculty, on="unitid", how="left")
elif stafftot_col:

View file

@ -230,6 +230,7 @@ def parse_schedule_j(root: etree._Element) -> list[dict]:
"BaseCompensation",
]),
"bonus_compensation": _find_first_int(grp, [
"BonusFilingOrganizationAmount",
"BonusFilingOrgAmt",
"Bonus",
]),