diff --git a/CLAUDE.md b/CLAUDE.md index b0941d4..05a6021 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,7 @@ Currently in planning. Phase 1 (Data Acquisition) is planned but not yet built. |--------|--------|-----------------| | IRS 990 bulk XML | XML (versioned schemas) | Filing financials, Part VII compensation, Schedule J detailed compensation | | IPEDS | CSV bulk downloads | Institution directory (HD), finance by function (F1A/F2), staffing (S/SAL), enrollment (EF) | +| EADA (Equity in Athletics) | Excel via ope.ed.gov/athletics API | Athletics expenses by sport, coaching salaries, participants (keyed by UNITID, FY2007+) | | BLS CPI-U | Flat file or API | Consumer Price Index for inflation-adjusted compensation analysis | | Admin office web pages (stretch) | HTML scraping | Staff directory headcounts | @@ -55,6 +56,7 @@ Not yet implemented. When built, the CLI will support: admin-analytics ingest ipeds --year-range 2005-2024 admin-analytics ingest irs990 --year-range 2005-2024 admin-analytics ingest cpi +admin-analytics ingest eada --year-range 2007-2025 admin-analytics ingest all ``` diff --git a/src/admin_analytics/cli.py b/src/admin_analytics/cli.py index 8d550c8..f8080c8 100644 --- a/src/admin_analytics/cli.py +++ b/src/admin_analytics/cli.py @@ -2,7 +2,7 @@ from typing import Annotated import typer -from admin_analytics.config import DEFAULT_YEAR_RANGE +from admin_analytics.config import DEFAULT_YEAR_RANGE, EADA_YEAR_RANGE from admin_analytics.db.connection import get_connection from admin_analytics.db.schema import ensure_schema @@ -130,6 +130,35 @@ def cpi( conn.close() +@ingest_app.command() +def eada( + year_range: Annotated[ + str, typer.Option(help="Academic-year-end range, e.g. 2007-2025 (2025 = 2024-25)") + ] = f"{EADA_YEAR_RANGE.start}-{EADA_YEAR_RANGE.stop - 1}", + force: Annotated[ + bool, typer.Option("--force", help="Re-download even if files exist") + ] = False, +) -> None: + """Ingest EADA athletics finance data for the University of Delaware.""" + years = _parse_year_range(year_range) + conn = get_connection() + ensure_schema(conn) + + from admin_analytics.eada.download import download_all + from admin_analytics.eada.loader import load_eada + + typer.echo(f"Downloading EADA data for {years.start}-{years.stop - 1}...") + download_all(years, force=force) + + typer.echo("Loading EADA data into database...") + totals = load_eada(conn, years) + typer.echo( + f"EADA ingestion complete: {totals['institution']} institution rows, " + f"{totals['sport']} sport rows." + ) + conn.close() + + @ingest_app.command() def scrape() -> None: """Scrape UD staff directory pages for admin headcounts.""" @@ -202,4 +231,5 @@ def ingest_all( ipeds(year_range=year_range, component="all", force=force) irs990(year_range=year_range, force=force) cpi(force=force) + eada(force=force) scrape() diff --git a/src/admin_analytics/config.py b/src/admin_analytics/config.py index 16cb959..eabda44 100644 --- a/src/admin_analytics/config.py +++ b/src/admin_analytics/config.py @@ -18,6 +18,13 @@ DEFAULT_YEAR_RANGE = range(2005, 2025) # 2005-2024 inclusive BLS_CPI_URL = "https://download.bls.gov/pub/time.series/cu/cu.data.0.Current" BLS_DATA_DIR = DATA_DIR / "bls" +# EADA (Equity in Athletics Disclosure Act), keyed by UNITID. +# Years are academic-year end (2024 = 2023-24), matching IPEDS fiscal years. +# Pre-2007 files use coded column names and are not supported. +EADA_API_URL = "https://ope.ed.gov/athletics/api/dataFiles" +EADA_DATA_DIR = DATA_DIR / "eada" +EADA_YEAR_RANGE = range(2007, 2026) # 2006-07 through 2024-25 + # IRS 990 UD_EINS = [516000297, 516017306] # UD + UD Research Foundation IRS990_BASE_URL = "https://apps.irs.gov/pub/epostcard/990/xml" diff --git a/src/admin_analytics/dashboard/app.py b/src/admin_analytics/dashboard/app.py index 55e6909..7dc8c7a 100644 --- a/src/admin_analytics/dashboard/app.py +++ b/src/admin_analytics/dashboard/app.py @@ -5,7 +5,7 @@ from dash import dcc, html, Input, Output from admin_analytics.db.connection import get_connection from admin_analytics.dashboard.pages import ( - overview, compensation, staffing, headcount, endowment, philanthropy, about, + overview, compensation, staffing, headcount, endowment, philanthropy, athletics, about, ) @@ -18,7 +18,7 @@ def create_app() -> dash.Dash: [ html.H1( "University of Delaware — Administrative Analytics", - style={"textAlign": "center", "padding": "20px", "color": "#00539F"}, + className="app-title", ), dcc.Tabs( id="tabs", @@ -29,14 +29,16 @@ def create_app() -> dash.Dash: dcc.Tab(label="Staffing & Enrollment", value="staffing"), dcc.Tab(label="Endowment", value="endowment"), dcc.Tab(label="Philanthropy", value="philanthropy"), + dcc.Tab(label="Athletics", value="athletics"), dcc.Tab(label="Current Headcount", value="headcount"), dcc.Tab(label="About", value="about"), ], - style={"marginBottom": "20px"}, + className="tabs-bar", + mobile_breakpoint=0, ), - html.Div(id="tab-content", style={"padding": "0 20px 20px 20px"}), + html.Div(id="tab-content", className="tab-content"), ], - style={"fontFamily": "system-ui, -apple-system, sans-serif", "maxWidth": "1400px", "margin": "0 auto"}, + className="app", ) @app.callback(Output("tab-content", "children"), Input("tabs", "value")) @@ -51,6 +53,8 @@ def create_app() -> dash.Dash: return endowment.layout(conn) elif tab == "philanthropy": return philanthropy.layout(conn) + elif tab == "athletics": + return athletics.layout(conn) elif tab == "headcount": return headcount.layout(conn) elif tab == "about": diff --git a/src/admin_analytics/dashboard/assets/style.css b/src/admin_analytics/dashboard/assets/style.css new file mode 100644 index 0000000..0be1abf --- /dev/null +++ b/src/admin_analytics/dashboard/assets/style.css @@ -0,0 +1,159 @@ +/* Served automatically by Dash from the assets/ folder. */ + +body { + margin: 0; + background: #fff; +} + +.app { + font-family: system-ui, -apple-system, sans-serif; + max-width: 1400px; + margin: 0 auto; +} + +.app-title { + text-align: center; + color: #00539F; + padding: 20px; + margin: 0; + font-size: 2rem; +} + +/* Tabs: one horizontally scrollable row at every width, instead of the + vertical stack dcc.Tabs switches to on small screens. */ +.tabs-bar { + display: flex !important; + flex-wrap: nowrap; + overflow-x: auto; + -webkit-overflow-scrolling: touch; + scrollbar-width: thin; + margin-bottom: 20px; +} + +.tabs-bar .tab { + flex: 1 0 auto !important; + width: auto !important; + white-space: nowrap; + padding: 12px 16px !important; +} + +.tab-content { + padding: 0 20px 20px 20px; +} + +.no-data { + text-align: center; + padding: 40px; + color: #888; +} + +/* KPI cards wrap onto multiple lines rather than squeezing. */ +.kpi-row { + display: flex; + flex-wrap: wrap; + gap: 16px; + margin-bottom: 24px; +} + +.kpi-card { + flex: 1 1 200px; + padding: 20px; + background-color: #f8f9fa; + border-radius: 8px; + text-align: center; +} + +.kpi-card h4 { + margin: 0; + color: #666; + font-size: 14px; +} + +.kpi-card h2 { + margin: 5px 0; + color: #00539F; +} + +.kpi-card p { + margin: 0; + color: #999; + font-size: 12px; +} + +/* Side-by-side charts stack once each would be narrower than ~450px. */ +.chart-row { + display: flex; + flex-wrap: wrap; + gap: 16px; +} + +.chart-row > * { + flex: 1 1 450px; + min-width: 0; +} + +.chart { + margin-bottom: 16px; +} + +.chart-title { + margin: 16px 0 0 0; + font-size: 17px; + font-weight: 500; + color: #2a3f5f; +} + +.filter-bar { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; + margin-bottom: 16px; +} + +.filter-bar .dash-dropdown { + width: 200px; +} + +.note { + color: #555; + font-size: 14px; + line-height: 1.5; +} + +@media (max-width: 600px) { + .app-title { + font-size: 1.25rem; + padding: 12px; + } + + .tab-content { + padding: 0 12px 12px 12px; + } + + .kpi-row { + gap: 8px; + } + + .kpi-card { + flex-basis: 140px; + padding: 12px; + } + + .kpi-card h2 { + font-size: 1.3rem; + } + + /* Charts are view-only on phones; the toolbar overlaps titles. */ + .modebar-container { + display: none; + } +} + +/* Markdown tables (About page) scroll within themselves instead of + widening the page. */ +.tab-content table { + display: block; + overflow-x: auto; + max-width: 100%; +} diff --git a/src/admin_analytics/dashboard/components.py b/src/admin_analytics/dashboard/components.py new file mode 100644 index 0000000..c761d6b --- /dev/null +++ b/src/admin_analytics/dashboard/components.py @@ -0,0 +1,69 @@ +"""Shared layout components for dashboard pages. + +Layout classes (kpi-row, chart-row, chart-title, ...) are styled in +assets/style.css, which handles the responsive wrapping for narrow screens. +""" + +import plotly.graph_objects as go +from dash import html, dcc + +GRAPH_CONFIG = {"displaylogo": False, "responsive": True} + + +def no_data(message: str) -> html.Div: + return html.Div(message, className="no-data") + + +def kpi_card(title: str, value: str, subtitle: str = "") -> html.Div: + children = [html.H4(title), html.H2(value)] + if subtitle: + children.append(html.P(subtitle)) + return html.Div(children, className="kpi-card") + + +def kpi_row(cards: list) -> html.Div: + return html.Div(cards, className="kpi-row") + + +def chart_row(*items) -> html.Div: + """Place charts side by side on wide screens; they stack on narrow ones.""" + return html.Div(list(items), className="chart-row") + + +def style_figure(fig: go.Figure) -> go.Figure: + """Apply layout defaults that read well at both phone and desktop widths. + + The legend sits horizontally above the plot area so it never steals width + from the data, and drag-to-zoom is off so swiping on a phone scrolls the + page instead of zooming the chart (zoom remains available in the modebar). + """ + fig.update_layout( + legend={"orientation": "h", "yanchor": "bottom", "y": 1.02, "xanchor": "left", "x": 0}, + margin={"l": 50, "r": 16, "t": 24, "b": 40}, + dragmode=False, + ) + fig.update_yaxes(automargin=True, title_standoff=8) + fig.update_xaxes(automargin=True) + return fig + + +def graph_style(height: int | None) -> dict: + """Pin the graph container to the figure height. + + With responsive=True the plot fills its container, so the container needs + an explicit height or flex layouts collapse it to nothing. + """ + return {"height": f"{height or 450}px"} + + +def graph(fig: go.Figure, **kwargs) -> html.Div: + """Render a figure with its title as an HTML heading. + + HTML headings wrap on narrow screens; Plotly titles get clipped instead. + """ + title = fig.layout.title.text + fig.update_layout(title=None) + style_figure(fig) + children = [html.H3(title, className="chart-title")] if title else [] + children.append(dcc.Graph(figure=fig, config=GRAPH_CONFIG, style=graph_style(fig.layout.height), **kwargs)) + return html.Div(children, className="chart") diff --git a/src/admin_analytics/dashboard/pages/about.py b/src/admin_analytics/dashboard/pages/about.py index 8663cc2..af539fd 100644 --- a/src/admin_analytics/dashboard/pages/about.py +++ b/src/admin_analytics/dashboard/pages/about.py @@ -21,6 +21,7 @@ All data is drawn from public, open-access sources: |--------|-----------|-------------|----------| | **IPEDS** | U.S. Dept. of Education, NCES | Institutional directory, expenses by function, staffing by occupation, enrollment, endowment, philanthropic gifts | 2005-2024 | | **IRS Form 990** | Internal Revenue Service | Executive compensation (Schedule J), filing financials for UD and UD Research Foundation | Tax years 2015-2025 | +| **EADA** | U.S. Dept. of Education, OPE | Athletics expenses by sport, participants, coaching salaries, full-time undergraduate enrollment | FY 2007-2025 | | **BLS CPI-U** | Bureau of Labor Statistics | Consumer Price Index for inflation adjustment (series CUUR0000SA0) | Full history | | **UD Staff Directories** | University of Delaware public web pages | Administrative office headcounts (College of Engineering line management, Provost's Office) | Current snapshot | @@ -53,6 +54,14 @@ year. For 2005-2019, only beginning and end-of-year values are reported. contracts. The compensation-vs-giving comparison indexes the President and VP of Development compensation against total philanthropic revenue. +**Athletics** uses Equity in Athletics Disclosure Act (EADA) filings, keyed by +the same IPEDS UNITID. Expenses are split into football, basketball, other +sports, and spending not allocated to a single team. Per-student figures use +EADA's full-time undergraduate enrollment. Head coach and athletic director +compensation comes from Schedule J. EADA reports athletics revenue equal to +expenses, so it measures the cost of athletics, not the institutional +subsidy. + **Inflation Adjustment** uses the BLS CPI-U annual average (all items, U.S. city average, not seasonally adjusted). CPI-adjusted values are expressed in the most recent available year's dollars. diff --git a/src/admin_analytics/dashboard/pages/athletics.py b/src/admin_analytics/dashboard/pages/athletics.py new file mode 100644 index 0000000..480d760 --- /dev/null +++ b/src/admin_analytics/dashboard/pages/athletics.py @@ -0,0 +1,232 @@ +"""Page: Intercollegiate Athletics.""" + +import duckdb +from dash import html, dcc +import plotly.graph_objects as go + +from admin_analytics.dashboard.components import ( + graph, + kpi_card, + kpi_row, + chart_row, + no_data, +) +from admin_analytics.dashboard.queries import ( + CONSISTENT_REPORTING_YEAR, + FBS_TRANSITION_YEAR, + query_athletics_expenses, + query_athletics_vs_university, + query_athletics_comp, +) + +_NO_DATA = no_data("No athletics data loaded. Run: admin-analytics ingest eada") + +_ROLE_COLORS = { + "Head Football Coach": "#00539F", + "Head Basketball Coach": "#E07A5F", + "Athletic Director": "#7FB069", +} + +_CONTEXT = """ +#### Context + +**University finances.** In October 2024, President Assanis projected a FY2025 +shortfall of $20–40 million, citing inflation, salary competition, financial +aid, and health insurance premiums. The response included a partial hiring +freeze, delayed capital projects, $11 million in payroll reductions, and $14 +million saved by "reducing travel and other expenses" +([Newark Post, Oct. 4, 2024](https://www.newarkpostonline.com/news/assanis-on-uds-financial-woes-were-not-out-of-the-woods-yet/article_81015f48-8226-11ef-a864-db72de6d8f73.html)). + +**Athletics.** The same fiscal year (2024–25) was football's transition to the +Football Bowl Subdivision ahead of full Conference USA membership on July 1, +2025. The Athletic Director told the Faculty Senate in December 2024 that the +department had cut institutional support by 24% over five years while growing +self-generated revenue and "gradually rais\\[ing\\] student fees," and that the $5 +million FBS reclassification fee was covered by philanthropy +([UDaily](https://udel.edu/udaily/2025/february/faculty-senate-meeting-december-2024)). + +#### Reading these numbers + +- **EADA reports cost, not subsidy.** UD reports athletics revenue equal to + expenses every year: institutional support and student fees are counted as + revenue to balance the books. These figures show what athletics *costs*, + not how much of that cost the university or students cover. Subsidy figures + from UD's NCAA financial reports are published by the + [Knight-Newhouse College Athletics Database](https://knightnewhousedata.org/fcs/university-delaware). +- **"Not allocated by sport"** is spending not assigned to a single team: + administration, facilities, conference and reclassification fees, and + similar. It doubled in 2024–25. Some of that may be one-time transition + costs, and some may be paid by gifts. +- **2011–12 drop.** Total and unallocated expenses fall sharply between + 2010–11 and 2011–12 while per-sport spending keeps rising, which suggests a + change in how UD reported shared costs rather than a real cut. The + growth comparison therefore starts in FY 2012. +- **Years** are fiscal years ending June 30 (2025 = 2024–25). University-wide + comparisons stop where IPEDS finance data ends. +""" + + +def _transition_marker(fig: go.Figure) -> None: + fig.add_vline( + x=FBS_TRANSITION_YEAR, line_dash="dot", line_color="#999", + annotation_text="FBS transition", annotation_position="top left", + annotation_font_size=11, annotation_font_color="#666", + ) + + +def layout(conn: duckdb.DuckDBPyConnection): + df = query_athletics_expenses(conn) + if df.height == 0: + return _NO_DATA + + exp = df.to_pandas() + latest = exp.iloc[-1] + prior = exp.iloc[-2] if len(exp) > 1 else None + year = int(latest["year"]) + + vs_df = query_athletics_vs_university(conn) + vs = vs_df.to_pandas() if vs_df.height > 0 else None + growth_df = query_athletics_vs_university(conn, since=CONSISTENT_REPORTING_YEAR) + growth = growth_df.to_pandas() if growth_df.height > 0 else None + + # KPI cards + cards = [ + kpi_card( + "Athletics Expenses", + f"${latest['total_expenses'] / 1e6:.1f}M", + f"FY {year} ({year - 1}–{str(year)[2:]})", + ), + ] + if prior is not None: + change = latest["total_expenses"] - prior["total_expenses"] + cards.append(kpi_card( + "One-Year Change", + f"{change / prior['total_expenses'] * 100:+.0f}%", + f"{'+' if change >= 0 else '−'}${abs(change) / 1e6:.1f}M vs FY {int(prior['year'])}", + )) + cards.append(kpi_card( + "Per Full-Time Undergraduate", + f"${int(latest['per_student']):,}", + f"FY {year}", + )) + if vs is not None: + last_vs = vs.iloc[-1] + cards.append(kpi_card( + "Share of University Expenses", + f"{last_vs['athletics_pct']:.1f}%", + f"FY {int(last_vs['year'])} (latest IPEDS year)", + )) + kpis = kpi_row(cards) + + # Expenses by category, stacked + by_cat_fig = go.Figure() + for col, label, color in [ + ("football", "Football", "#00539F"), + ("basketball", "Basketball", "#E07A5F"), + ("other_sports", "Other sports", "#7FB069"), + ("unallocated_expenses", "Not allocated by sport", "#B8B8B8"), + ]: + by_cat_fig.add_trace(go.Bar( + x=exp["year"], y=exp[col].astype(float) / 1e6, + name=label, marker_color=color, + )) + by_cat_fig.add_trace(go.Scatter( + x=exp["year"], y=exp["total_expenses_cpi"] / 1e6, + mode="lines", name="Total, CPI-adjusted", + line={"color": "#333", "dash": "dash"}, + )) + _transition_marker(by_cat_fig) + by_cat_fig.update_layout( + title="Athletics Expenses by Category (Millions $)", + xaxis_title="Fiscal Year", yaxis_title="Millions $", + barmode="stack", template="plotly_white", height=440, + ) + + # Growth vs university spending + growth_fig = go.Figure() + if growth is not None: + for col, label, color, dash in [ + ("athletics_index", "Athletics", "#00539F", None), + ("university_index", "Total university", "#E07A5F", None), + ("instruction_index", "Instruction", "#7FB069", None), + ("cpi_index", "CPI-U", "#FFD200", "dash"), + ]: + growth_fig.add_trace(go.Scatter( + x=growth["year"], y=growth[col], mode="lines+markers", name=label, + line={"color": color, "dash": dash}, + )) + growth_fig.add_hline(y=100, line_dash="dot", line_color="#ccc") + base_year = int(growth.iloc[0]["year"]) + else: + base_year = None + growth_fig.update_layout( + title=f"Spending Growth (Indexed, FY {base_year} = 100)" if base_year else "Spending Growth", + xaxis_title="Fiscal Year", yaxis_title="Index", + template="plotly_white", height=380, + ) + + # Per-student cost + per_student_fig = go.Figure() + per_student_fig.add_trace(go.Scatter( + x=exp["year"], y=exp["per_student"], + mode="lines+markers", name="Nominal", line={"color": "#00539F"}, + )) + per_student_fig.add_trace(go.Scatter( + x=exp["year"], y=exp["per_student_cpi"], + mode="lines+markers", name="CPI-adjusted", line={"color": "#FFD200", "dash": "dash"}, + )) + _transition_marker(per_student_fig) + per_student_fig.update_layout( + title="Athletics Expenses per Full-Time Undergraduate", + xaxis_title="Fiscal Year", yaxis_title="$ per student", + template="plotly_white", height=380, + ) + + # Share of university expenses + share_fig = go.Figure() + if vs is not None: + share_fig.add_trace(go.Scatter( + x=vs["year"], y=vs["athletics_pct"], + mode="lines+markers", name="Athletics share", line={"color": "#00539F"}, + )) + share_fig.update_layout( + title="Athletics as % of Total University Expenses", + xaxis_title="Fiscal Year", yaxis_title="%", + template="plotly_white", height=380, + ) + + # Leadership compensation (Schedule J) + comp_df = query_athletics_comp(conn) + comp_fig = go.Figure() + if comp_df.height > 0: + comp = comp_df.to_pandas() + for role, color in _ROLE_COLORS.items(): + subset = comp[comp["role"] == role] + if len(subset) == 0: + continue + # Reindex to every year so missing years show as gaps, not bridges + years = range(int(comp["tax_year"].min()), int(comp["tax_year"].max()) + 1) + subset = subset.set_index("tax_year").reindex(years) + comp_fig.add_trace(go.Scatter( + x=list(years), y=subset["total_compensation"], + mode="lines+markers", name=role, line={"color": color}, + customdata=subset["person_name"].fillna(""), + hovertemplate="%{customdata}
$%{y:,.0f}" + role + "", + )) + comp_fig.update_layout( + title="Athletics Leadership Compensation (IRS 990 Schedule J)", + xaxis_title="Tax Year", yaxis_title="Total compensation ($)", + template="plotly_white", height=400, + ) + + children = [ + kpis, + graph(by_cat_fig), + chart_row(graph(growth_fig), graph(per_student_fig)), + ] + if vs is not None: + children.append(chart_row(graph(share_fig), graph(comp_fig))) + else: + children.append(graph(comp_fig)) + children.append(dcc.Markdown(_CONTEXT, className="note", link_target="_blank")) + return html.Div(children) diff --git a/src/admin_analytics/dashboard/pages/compensation.py b/src/admin_analytics/dashboard/pages/compensation.py index 08a370e..df18a63 100644 --- a/src/admin_analytics/dashboard/pages/compensation.py +++ b/src/admin_analytics/dashboard/pages/compensation.py @@ -6,6 +6,16 @@ from dash import html, dcc, Input, Output, dash_table import plotly.express as px import plotly.graph_objects as go +from admin_analytics.dashboard.components import ( + GRAPH_CONFIG, + graph, + graph_style, + kpi_card, + kpi_row, + chart_row, + no_data, + style_figure, +) from admin_analytics.dashboard.queries import ( query_top_earners, query_comp_by_role, @@ -15,33 +25,12 @@ from admin_analytics.dashboard.queries import ( query_aggregate_comp_cagr, ) -_NO_DATA = html.Div( - "No IRS 990 data loaded. Run: admin-analytics ingest irs990", - style={"textAlign": "center", "padding": "40px", "color": "#888"}, -) +_NO_DATA = no_data("No IRS 990 data loaded. Run: admin-analytics ingest irs990") # Roles to highlight in trend chart _KEY_ROLES = ["PRESIDENT", "PROVOST", "VP_FINANCE", "VP_RESEARCH", "VP_ADVANCEMENT", "CFO"] -def _kpi_card(title: str, value: str, subtitle: str = "") -> html.Div: - return html.Div( - [ - html.H4(title, style={"margin": "0", "color": "#666", "fontSize": "14px"}), - html.H2(value, style={"margin": "5px 0", "color": "#00539F"}), - html.P(subtitle, style={"margin": "0", "color": "#999", "fontSize": "12px"}), - ], - style={ - "flex": "1", - "padding": "20px", - "backgroundColor": "#f8f9fa", - "borderRadius": "8px", - "textAlign": "center", - "margin": "0 8px", - }, - ) - - def layout(conn: duckdb.DuckDBPyConnection): all_earners = query_top_earners(conn) if all_earners.height == 0: @@ -57,28 +46,28 @@ def layout(conn: duckdb.DuckDBPyConnection): agg_cagr = query_aggregate_comp_cagr(conn) kpi_cards = [] if cagr: - kpi_cards.append(_kpi_card( + kpi_cards.append(kpi_card( "President Compensation", f"${cagr['end_comp']:,}", f"Tax year {cagr['end_year']}", )) - kpi_cards.append(_kpi_card( + kpi_cards.append(kpi_card( "President CAGR", f"{cagr['cagr_pct']}%", f"Annualized growth, {cagr['start_year']}-{cagr['end_year']}", )) if agg_cagr: - kpi_cards.append(_kpi_card( + kpi_cards.append(kpi_card( "Top-10 Total Compensation", f"${agg_cagr['end_comp']:,}", f"Tax year {agg_cagr['end_year']}", )) - kpi_cards.append(_kpi_card( + kpi_cards.append(kpi_card( "Top-10 CAGR", f"{agg_cagr['cagr_pct']}%", f"Annualized growth, {agg_cagr['start_year']}-{agg_cagr['end_year']}", )) - kpi_row = html.Div(kpi_cards, style={"display": "flex", "marginBottom": "24px"}) if kpi_cards else html.Div() + kpis = kpi_row(kpi_cards) if kpi_cards else html.Div() # Compensation by role trend role_df = query_comp_by_role(conn) @@ -127,7 +116,7 @@ def layout(conn: duckdb.DuckDBPyConnection): ) return html.Div([ - kpi_row, + kpis, html.Div( [ html.Label("Filter by Tax Year: ", style={"fontWeight": "bold"}), @@ -135,10 +124,10 @@ def layout(conn: duckdb.DuckDBPyConnection): id="comp-year-dropdown", options=year_options, value="all", - style={"width": "200px", "display": "inline-block"}, + clearable=False, ), ], - style={"marginBottom": "16px"}, + className="filter-bar", ), dash_table.DataTable( id="comp-table", @@ -162,14 +151,17 @@ def layout(conn: duckdb.DuckDBPyConnection): style_cell={"textAlign": "left", "padding": "8px", "fontSize": "13px"}, style_header={"fontWeight": "bold", "backgroundColor": "#f0f0f0"}, ), - html.Div( - [ - html.Div(dcc.Graph(id="comp-breakdown-chart", config={"displaylogo": False}), style={"flex": "1"}), - html.Div(dcc.Graph(figure=cpi_fig, config={"displaylogo": False}), style={"flex": "1"}), - ], - style={"display": "flex", "gap": "16px", "marginTop": "16px"}, + chart_row( + html.Div( + [ + html.H3(id="comp-breakdown-title", className="chart-title"), + dcc.Graph(id="comp-breakdown-chart", config=GRAPH_CONFIG, style=graph_style(380)), + ], + className="chart", + ), + graph(cpi_fig), ), - dcc.Graph(figure=role_fig, config={"displaylogo": False}), + graph(role_fig), ]) @@ -177,7 +169,11 @@ def register_callbacks(app: dash.Dash, conn: duckdb.DuckDBPyConnection) -> None: """Register interactive callbacks for the compensation page.""" @app.callback( - [Output("comp-table", "data"), Output("comp-breakdown-chart", "figure")], + [ + Output("comp-table", "data"), + Output("comp-breakdown-chart", "figure"), + Output("comp-breakdown-title", "children"), + ], Input("comp-year-dropdown", "value"), ) def update_compensation(year_value): @@ -218,9 +214,9 @@ def register_callbacks(app: dash.Dash, conn: duckdb.DuckDBPyConnection) -> None: title_suffix = f" ({year})" if year else " (All Years)" breakdown_fig.update_layout( - title=f"Compensation Breakdown — Top 10{title_suffix}", xaxis_title="", yaxis_title="$", template="plotly_white", height=380, ) + style_figure(breakdown_fig) - return table_data, breakdown_fig + return table_data, breakdown_fig, f"Compensation Breakdown — Top 10{title_suffix}" diff --git a/src/admin_analytics/dashboard/pages/endowment.py b/src/admin_analytics/dashboard/pages/endowment.py index d2418e5..209b4e4 100644 --- a/src/admin_analytics/dashboard/pages/endowment.py +++ b/src/admin_analytics/dashboard/pages/endowment.py @@ -1,35 +1,15 @@ """Page: Endowment Performance.""" import duckdb -from dash import html, dcc +from dash import html import plotly.graph_objects as go +from admin_analytics.dashboard.components import graph, kpi_card, kpi_row, no_data from admin_analytics.dashboard.queries import ( query_endowment, query_endowment_per_student, query_cio_vs_endowment, ) -_NO_DATA = html.Div( - "No endowment data loaded. Run: admin-analytics ingest ipeds --component finance", - style={"textAlign": "center", "padding": "40px", "color": "#888"}, -) - - -def _kpi_card(title: str, value: str, subtitle: str = "") -> html.Div: - return html.Div( - [ - html.H4(title, style={"margin": "0", "color": "#666", "fontSize": "14px"}), - html.H2(value, style={"margin": "5px 0", "color": "#00539F"}), - html.P(subtitle, style={"margin": "0", "color": "#999", "fontSize": "12px"}), - ], - style={ - "flex": "1", - "padding": "20px", - "backgroundColor": "#f8f9fa", - "borderRadius": "8px", - "textAlign": "center", - "margin": "0 8px", - }, - ) +_NO_DATA = no_data("No endowment data loaded. Run: admin-analytics ingest ipeds --component finance") def layout(conn: duckdb.DuckDBPyConnection): @@ -62,29 +42,29 @@ def layout(conn: duckdb.DuckDBPyConnection): # KPI cards — single row kpi_cards = [ - _kpi_card( + kpi_card( "Endowment Value", f"${latest['endowment_eoy'] / 1e9:.2f}B" if latest["endowment_eoy"] else "N/A", f"End of FY {int(latest['year'])}", ), - _kpi_card( + kpi_card( "Endowment CAGR", f"{endow_cagr}%" if endow_cagr is not None else "N/A", f"FY {start_year}-{end_year}" if start_year else "", ), ] if latest_eps is not None and latest_eps["endowment_per_student"]: - kpi_cards.append(_kpi_card( + kpi_cards.append(kpi_card( "Endowment per Student", f"${int(latest_eps['endowment_per_student']):,}", f"FY {int(latest_eps['year'])}", )) - kpi_cards.append(_kpi_card( + kpi_cards.append(kpi_card( "New Gifts to Endowment", f"${latest['new_gifts'] / 1e6:.1f}M" if latest["new_gifts"] else "N/A", f"FY {int(latest['year'])}", )) - kpi_row = html.Div(kpi_cards, style={"display": "flex", "marginBottom": "24px"}) + kpis = kpi_row(kpi_cards) # Endowment value trend value_fig = go.Figure() @@ -189,13 +169,13 @@ def layout(conn: duckdb.DuckDBPyConnection): ) charts = [ - kpi_row, - dcc.Graph(figure=value_fig, config={"displaylogo": False}), - dcc.Graph(figure=eps_fig, config={"displaylogo": False}), - dcc.Graph(figure=components_fig, config={"displaylogo": False}), - dcc.Graph(figure=rate_fig, config={"displaylogo": False}), + kpis, + graph(value_fig), + graph(eps_fig), + graph(components_fig), + graph(rate_fig), ] if cio_df.height > 1: - charts.append(dcc.Graph(figure=cio_fig, config={"displaylogo": False})) + charts.append(graph(cio_fig)) return html.Div(charts) diff --git a/src/admin_analytics/dashboard/pages/headcount.py b/src/admin_analytics/dashboard/pages/headcount.py index 3adf03e..73aec2f 100644 --- a/src/admin_analytics/dashboard/pages/headcount.py +++ b/src/admin_analytics/dashboard/pages/headcount.py @@ -1,36 +1,17 @@ """Page 4: Current Admin Headcount (from scraper).""" import duckdb -from dash import html, dcc, dash_table +from dash import html, dash_table import plotly.express as px import plotly.graph_objects as go +from admin_analytics.dashboard.components import graph, kpi_card, kpi_row, chart_row, no_data from admin_analytics.dashboard.queries import ( query_admin_headcount, query_headcount_summary, ) -_NO_DATA = html.Div( - "No headcount data loaded. Run: admin-analytics ingest scrape", - style={"textAlign": "center", "padding": "40px", "color": "#888"}, -) - - -def _kpi_card(title: str, value: str) -> html.Div: - return html.Div( - [ - html.H4(title, style={"margin": "0", "color": "#666", "fontSize": "14px"}), - html.H2(value, style={"margin": "5px 0", "color": "#00539F"}), - ], - style={ - "flex": "1", - "padding": "20px", - "backgroundColor": "#f8f9fa", - "borderRadius": "8px", - "textAlign": "center", - "margin": "0 8px", - }, - ) +_NO_DATA = no_data("No headcount data loaded. Run: admin-analytics ingest scrape") def layout(conn: duckdb.DuckDBPyConnection): @@ -47,13 +28,12 @@ def layout(conn: duckdb.DuckDBPyConnection): overhead_pct = round(overhead_count * 100 / total, 1) if total > 0 else 0 # KPI cards - kpi_row = html.Div( + kpis = kpi_row( [ - _kpi_card("Total Staff Scraped", str(total)), - _kpi_card("Overhead Staff", str(overhead_count)), - _kpi_card("Overhead %", f"{overhead_pct}%"), + kpi_card("Total Staff Scraped", str(total)), + kpi_card("Overhead Staff", str(overhead_count)), + kpi_card("Overhead %", f"{overhead_pct}%"), ], - style={"display": "flex", "marginBottom": "24px"}, ) # Staff by unit bar chart @@ -104,13 +84,7 @@ def layout(conn: duckdb.DuckDBPyConnection): ) return html.Div([ - kpi_row, - html.Div( - [ - html.Div(dcc.Graph(figure=unit_fig, config={"displaylogo": False}), style={"flex": "1"}), - html.Div(dcc.Graph(figure=pie_fig, config={"displaylogo": False}), style={"flex": "1"}), - ], - style={"display": "flex", "gap": "16px"}, - ), - dcc.Graph(figure=cat_fig, config={"displaylogo": False}), + kpis, + chart_row(graph(unit_fig), graph(pie_fig)), + graph(cat_fig), ]) diff --git a/src/admin_analytics/dashboard/pages/overview.py b/src/admin_analytics/dashboard/pages/overview.py index cf46de2..fdce580 100644 --- a/src/admin_analytics/dashboard/pages/overview.py +++ b/src/admin_analytics/dashboard/pages/overview.py @@ -1,10 +1,11 @@ """Page 1: Administrative Cost Overview.""" import duckdb -from dash import html, dcc +from dash import html import plotly.express as px import plotly.graph_objects as go +from admin_analytics.dashboard.components import graph, kpi_card, kpi_row, chart_row, no_data from admin_analytics.dashboard.queries import ( query_admin_cost_ratio, query_expense_breakdown, @@ -12,28 +13,7 @@ from admin_analytics.dashboard.queries import ( query_admin_faculty_ratio, ) -_NO_DATA = html.Div( - "No data loaded. Run: admin-analytics ingest all", - style={"textAlign": "center", "padding": "40px", "color": "#888"}, -) - - -def _kpi_card(title: str, value: str, subtitle: str = "") -> html.Div: - return html.Div( - [ - html.H4(title, style={"margin": "0", "color": "#666", "fontSize": "14px"}), - html.H2(value, style={"margin": "5px 0", "color": "#00539F"}), - html.P(subtitle, style={"margin": "0", "color": "#999", "fontSize": "12px"}), - ], - style={ - "flex": "1", - "padding": "20px", - "backgroundColor": "#f8f9fa", - "borderRadius": "8px", - "textAlign": "center", - "margin": "0 8px", - }, - ) +_NO_DATA = no_data("No data loaded. Run: admin-analytics ingest all") def layout(conn: duckdb.DuckDBPyConnection): @@ -56,25 +36,24 @@ def layout(conn: duckdb.DuckDBPyConnection): latest_afr = afr_pd.iloc[-1] if afr_pd is not None else None # KPI cards - kpi_row = html.Div( + kpis = kpi_row( [ - _kpi_card( + kpi_card( "Admin Cost Ratio", f"{latest['admin_cost_pct']:.1f}%", f"Institutional Support / Total Expenses ({int(latest['year'])})", ), - _kpi_card( + kpi_card( "Admin Cost per Student", f"${int(latest_aps['admin_per_student']):,}" if latest_aps is not None else "N/A", f"CPI-adjusted: ${int(latest_aps['admin_per_student_cpi']):,}" if latest_aps is not None else "", ), - _kpi_card( + kpi_card( "Admin-to-Faculty Ratio", f"{latest_afr['admin_faculty_ratio']:.2f}" if latest_afr is not None else "N/A", f"Management / Faculty ({int(latest_afr['year'])})" if latest_afr is not None else "", ), ], - style={"display": "flex", "marginBottom": "24px"}, ) # Admin cost ratio trend @@ -155,14 +134,8 @@ def layout(conn: duckdb.DuckDBPyConnection): ) return html.Div([ - kpi_row, - dcc.Graph(figure=ratio_fig, config={"displaylogo": False}), - dcc.Graph(figure=breakdown_fig, config={"displaylogo": False}), - html.Div( - [ - html.Div(dcc.Graph(figure=aps_fig, config={"displaylogo": False}), style={"flex": "1"}), - html.Div(dcc.Graph(figure=afr_fig, config={"displaylogo": False}), style={"flex": "1"}), - ], - style={"display": "flex", "gap": "16px"}, - ), + kpis, + graph(ratio_fig), + graph(breakdown_fig), + chart_row(graph(aps_fig), graph(afr_fig)), ]) diff --git a/src/admin_analytics/dashboard/pages/philanthropy.py b/src/admin_analytics/dashboard/pages/philanthropy.py index 0b77d9e..afb16ce 100644 --- a/src/admin_analytics/dashboard/pages/philanthropy.py +++ b/src/admin_analytics/dashboard/pages/philanthropy.py @@ -1,33 +1,13 @@ """Page: Philanthropic Giving.""" import duckdb -from dash import html, dcc +from dash import html import plotly.graph_objects as go +from admin_analytics.dashboard.components import graph, kpi_card, kpi_row, no_data from admin_analytics.dashboard.queries import query_philanthropy, query_comp_vs_philanthropy -_NO_DATA = html.Div( - "No philanthropy data loaded. Run: admin-analytics ingest ipeds --component finance", - style={"textAlign": "center", "padding": "40px", "color": "#888"}, -) - - -def _kpi_card(title: str, value: str, subtitle: str = "") -> html.Div: - return html.Div( - [ - html.H4(title, style={"margin": "0", "color": "#666", "fontSize": "14px"}), - html.H2(value, style={"margin": "5px 0", "color": "#00539F"}), - html.P(subtitle, style={"margin": "0", "color": "#999", "fontSize": "12px"}), - ], - style={ - "flex": "1", - "padding": "20px", - "backgroundColor": "#f8f9fa", - "borderRadius": "8px", - "textAlign": "center", - "margin": "0 8px", - }, - ) +_NO_DATA = no_data("No philanthropy data loaded. Run: admin-analytics ingest ipeds --component finance") def layout(conn: duckdb.DuckDBPyConnection): @@ -39,20 +19,19 @@ def layout(conn: duckdb.DuckDBPyConnection): latest = pd.iloc[-1] # KPI cards - kpi_row = html.Div( + kpis = kpi_row( [ - _kpi_card( + kpi_card( "Total Private Gifts & Grants", f"${latest['total_private_gifts'] / 1e6:.1f}M" if latest["total_private_gifts"] else "N/A", f"FY {int(latest['year'])}", ), - _kpi_card( + kpi_card( "Gifts to Endowment", f"${latest['endowment_gifts'] / 1e6:.1f}M" if latest["endowment_gifts"] else "N/A", f"FY {int(latest['year'])}", ), ], - style={"display": "flex", "marginBottom": "24px"}, ) # Private gifts trend (nominal and CPI-adjusted) @@ -122,11 +101,11 @@ def layout(conn: duckdb.DuckDBPyConnection): ) charts = [ - kpi_row, - dcc.Graph(figure=gifts_fig, config={"displaylogo": False}), - dcc.Graph(figure=split_fig, config={"displaylogo": False}), + kpis, + graph(gifts_fig), + graph(split_fig), ] if cvp_df.height > 1: - charts.append(dcc.Graph(figure=cvp_fig, config={"displaylogo": False})) + charts.append(graph(cvp_fig)) return html.Div(charts) diff --git a/src/admin_analytics/dashboard/pages/staffing.py b/src/admin_analytics/dashboard/pages/staffing.py index 5050960..18f2930 100644 --- a/src/admin_analytics/dashboard/pages/staffing.py +++ b/src/admin_analytics/dashboard/pages/staffing.py @@ -1,19 +1,17 @@ """Page 3: Staffing & Enrollment.""" import duckdb -from dash import html, dcc +from dash import html import plotly.graph_objects as go +from admin_analytics.dashboard.components import graph, chart_row, no_data from admin_analytics.dashboard.queries import ( query_staff_composition, query_student_staff_ratios, query_growth_index, ) -_NO_DATA = html.Div( - "No IPEDS staff data loaded. Run: admin-analytics ingest ipeds", - style={"textAlign": "center", "padding": "40px", "color": "#888"}, -) +_NO_DATA = no_data("No IPEDS staff data loaded. Run: admin-analytics ingest ipeds") def layout(conn: duckdb.DuckDBPyConnection): @@ -91,12 +89,6 @@ def layout(conn: duckdb.DuckDBPyConnection): ) return html.Div([ - dcc.Graph(figure=comp_fig, config={"displaylogo": False}), - html.Div( - [ - html.Div(dcc.Graph(figure=ratio_fig, config={"displaylogo": False}), style={"flex": "1"}), - html.Div(dcc.Graph(figure=growth_fig, config={"displaylogo": False}), style={"flex": "1"}), - ], - style={"display": "flex", "gap": "16px"}, - ), + graph(comp_fig), + chart_row(graph(ratio_fig), graph(growth_fig)), ]) diff --git a/src/admin_analytics/dashboard/queries.py b/src/admin_analytics/dashboard/queries.py index d3dc65b..a8ea2cf 100644 --- a/src/admin_analytics/dashboard/queries.py +++ b/src/admin_analytics/dashboard/queries.py @@ -553,3 +553,137 @@ def query_headcount_summary(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame: GROUP BY unit, category, is_overhead ORDER BY unit, count DESC """).pl() + + +# --- Athletics (EADA) --- + +# Year the football program spent transitioning to FBS (2024-25); UD joined +# Conference USA as a full member on July 1, 2025 (FY2026). +FBS_TRANSITION_YEAR = 2025 + +# UD's unallocated athletics expenses drop by ~60% between FY2011 and FY2012 +# while per-sport spending keeps rising, consistent with a change in how +# shared costs were reported. Growth comparisons start here. +CONSISTENT_REPORTING_YEAR = 2012 + + +def query_athletics_expenses(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame: + """Athletics expenses by year, split by sport, with CPI and per-student values. + + "Not allocated" is spending EADA does not assign to any single team + (administration, facilities, conference fees, etc.). + """ + return conn.execute(f""" + {_CPI_CTE}, + sports AS ( + SELECT year, + SUM(total_expenses) FILTER (WHERE sport = 'Football') AS football, + SUM(total_expenses) FILTER (WHERE sport = 'Basketball') AS basketball, + SUM(total_expenses) FILTER (WHERE sport NOT IN ('Football', 'Basketball')) + AS other_sports + FROM raw_eada_sport + WHERE unitid = ? + GROUP BY year + ) + SELECT + i.year, + i.classification, + i.total_expenses, + ROUND(i.total_expenses * (SELECT avg_cpi FROM latest_cpi) / ac.avg_cpi, 0) + AS total_expenses_cpi, + s.football, + s.basketball, + s.other_sports, + i.unallocated_expenses, + i.student_aid, + i.participants_men + i.participants_women AS participants, + i.ft_undergrad_enrollment, + ROUND(i.total_expenses * 1.0 / NULLIF(i.ft_undergrad_enrollment, 0), 0) + AS per_student, + ROUND(i.total_expenses * (SELECT avg_cpi FROM latest_cpi) / ac.avg_cpi + / NULLIF(i.ft_undergrad_enrollment, 0), 0) AS per_student_cpi + FROM raw_eada_institution i + LEFT JOIN sports s ON s.year = i.year + LEFT JOIN annual_cpi ac ON ac.year = i.year + WHERE i.unitid = ? + ORDER BY i.year + """, [UD_UNITID, UD_UNITID]).pl() + + +def query_athletics_vs_university( + conn: duckdb.DuckDBPyConnection, since: int | None = None +) -> pl.DataFrame: + """Athletics spending against university-wide IPEDS spending. + + Restricted to years present in both EADA and IPEDS (and >= since, if + given), indexed to the first such year = 100. + """ + return conn.execute(""" + WITH annual_cpi AS ( + SELECT year, AVG(value) AS avg_cpi FROM raw_cpi_u GROUP BY year + ), + joined AS ( + SELECT a.year, + a.total_expenses AS athletics, + f.total_expenses AS university, + f.instruction_expenses AS instruction, + f.institutional_support_expenses AS inst_support, + ac.avg_cpi + FROM raw_eada_institution a + JOIN raw_ipeds_finance f ON f.unitid = a.unitid AND f.year = a.year + LEFT JOIN annual_cpi ac ON ac.year = a.year + WHERE a.unitid = ? AND a.year >= ? + ), + base AS (SELECT * FROM joined ORDER BY year LIMIT 1) + SELECT j.year, j.athletics, j.university, + ROUND(j.athletics * 100.0 / NULLIF(j.university, 0), 2) AS athletics_pct, + ROUND(j.athletics * 100.0 / NULLIF(b.athletics, 0), 1) AS athletics_index, + ROUND(j.university * 100.0 / NULLIF(b.university, 0), 1) AS university_index, + ROUND(j.instruction * 100.0 / NULLIF(b.instruction, 0), 1) AS instruction_index, + ROUND(j.inst_support * 100.0 / NULLIF(b.inst_support, 0), 1) AS inst_support_index, + ROUND(j.avg_cpi * 100.0 / NULLIF(b.avg_cpi, 0), 1) AS cpi_index + FROM joined j CROSS JOIN base b + ORDER BY j.year + """, [UD_UNITID, since or 0]).pl() + + +_ATHLETICS_ROLES = [ + ("Head Football Coach", r"FOOTBALL"), + ("Head Basketball Coach", r"BASKETBALL"), + # Titles abbreviate and misspell freely: "DIR, INTERCOLLEGIATE ATHLCS", + # "DIRECTOR, INTERCOLLEGIATE ATHELTICS", "ATHLETIC DIRECTOR". + ("Athletic Director", r"ATHLETIC DIRECTOR|INTERCOLLEGIATE ATH"), +] + + +def query_athletics_comp(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame: + """Schedule J compensation for head coaches and the athletic director. + + When a role changed hands mid-year (e.g. a departing coach and a new one), + the highest-paid person in that role is kept for the year. + """ + df = conn.execute(""" + SELECT tax_year, person_name, title, total_compensation + FROM raw_990_schedule_j + WHERE regexp_matches(upper(title), 'COACH|ATHL|ATHEL') + AND NOT regexp_matches(upper(title), 'FORMER|ASSISTANT|ASSOCIATE') + """).pl() + if df.height == 0: + return pl.DataFrame(schema={ + "tax_year": pl.Int32, "role": pl.Utf8, + "person_name": pl.Utf8, "total_compensation": pl.Int64, + }) + role_expr = pl.lit(None, dtype=pl.Utf8) + for role, pattern in reversed(_ATHLETICS_ROLES): + role_expr = ( + pl.when(pl.col("title").str.to_uppercase().str.contains(pattern)) + .then(pl.lit(role)).otherwise(role_expr) + ) + return ( + df.with_columns(role_expr.alias("role")) + .filter(pl.col("role").is_not_null()) + .sort("total_compensation", descending=True) + .unique(subset=["tax_year", "role"], keep="first") + .select("tax_year", "role", "person_name", "total_compensation") + .sort(["role", "tax_year"]) + ) diff --git a/src/admin_analytics/db/schema.py b/src/admin_analytics/db/schema.py index 7654451..f81f697 100644 --- a/src/admin_analytics/db/schema.py +++ b/src/admin_analytics/db/schema.py @@ -116,6 +116,39 @@ TABLES = { PRIMARY KEY (unitid, year) ) """, + "raw_eada_institution": """ + CREATE TABLE IF NOT EXISTS raw_eada_institution ( + unitid INTEGER NOT NULL, + year INTEGER NOT NULL, + classification VARCHAR, + total_revenue BIGINT, + total_expenses BIGINT, + unallocated_revenue BIGINT, + unallocated_expenses BIGINT, + student_aid BIGINT, + recruiting_expenses BIGINT, + head_coach_salary_men BIGINT, + head_coach_salary_women BIGINT, + asst_coach_salary_men BIGINT, + asst_coach_salary_women BIGINT, + participants_men INTEGER, + participants_women INTEGER, + ft_undergrad_enrollment INTEGER, + PRIMARY KEY (unitid, year) + ) + """, + "raw_eada_sport": """ + CREATE TABLE IF NOT EXISTS raw_eada_sport ( + unitid INTEGER NOT NULL, + year INTEGER NOT NULL, + sport VARCHAR NOT NULL, + participants_men INTEGER, + participants_women INTEGER, + total_revenue BIGINT, + total_expenses BIGINT, + PRIMARY KEY (unitid, year, sport) + ) + """, "raw_cpi_u": """ CREATE TABLE IF NOT EXISTS raw_cpi_u ( year INTEGER NOT NULL, diff --git a/src/admin_analytics/eada/__init__.py b/src/admin_analytics/eada/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/admin_analytics/eada/download.py b/src/admin_analytics/eada/download.py new file mode 100644 index 0000000..37ac356 --- /dev/null +++ b/src/admin_analytics/eada/download.py @@ -0,0 +1,57 @@ +"""Download EADA athletics data files from the Department of Education.""" + +import io +import zipfile +from pathlib import Path + +import httpx + +from admin_analytics import config + +# Only the spreadsheets are needed; the zips also carry SAS/SPSS/Word copies. +_SPREADSHEET_SUFFIXES = (".xls", ".xlsx") + + +def list_data_files(client: httpx.Client) -> dict[int, str]: + """Return {year: filename} for the per-year EADA data zips. + + The API also lists "All_Data_Combined" zips with one record per + institution; those duplicate the per-year institution file and are skipped. + """ + resp = client.get(f"{config.EADA_API_URL}/fileList") + resp.raise_for_status() + return { + int(entry["Year"]): entry["FileName"] + for entry in resp.json() + if "Combined" not in entry["FileName"] + } + + +def _dest_dir(year: int) -> Path: + return config.EADA_DATA_DIR / str(year) + + +def download_year(client: httpx.Client, year: int, filename: str, force: bool = False) -> Path: + """Download and extract the spreadsheets for one EADA year.""" + dest = _dest_dir(year) + if not force and dest.exists() and any(dest.glob("*.xls*")): + return dest + dest.mkdir(parents=True, exist_ok=True) + resp = client.get(f"{config.EADA_API_URL}/file", params={"fileName": filename}) + resp.raise_for_status() + with zipfile.ZipFile(io.BytesIO(resp.content)) as zf: + for name in zf.namelist(): + if name.lower().endswith(_SPREADSHEET_SUFFIXES): + zf.extract(name, dest) + return dest + + +def download_all(year_range: range, force: bool = False) -> None: + """Download all available EADA years in the range.""" + with httpx.Client(follow_redirects=True, timeout=600.0) as client: + available = list_data_files(client) + for year in year_range: + if year not in available: + print(f" Warning: EADA {year} not available") + continue + download_year(client, year, available[year], force=force) diff --git a/src/admin_analytics/eada/loader.py b/src/admin_analytics/eada/loader.py new file mode 100644 index 0000000..c30caff --- /dev/null +++ b/src/admin_analytics/eada/loader.py @@ -0,0 +1,133 @@ +"""Load EADA athletics spreadsheets into raw_eada_* tables. + +Each EADA year ships two spreadsheets: an institution-level file (one row per +school) and a schools file (one row per sport per school). Filenames and +column-name casing vary across years, so both are resolved loosely. + +Note that EADA revenue equals expenses for most public institutions: the +institutional subsidy is reported as revenue to balance the books, so EADA +measures athletics *cost* but does not separate out the subsidy. +""" + +from pathlib import Path + +import duckdb +import pandas as pd +import polars as pl + +from admin_analytics import config +from admin_analytics.config import UD_UNITID + +# Canonical column -> candidate EADA columns, first match wins (case-insensitive). +INSTITUTION_VARIANTS = { + "unitid": ["UNITID"], + "classification": ["CLASSIFICATION_NAME", "VALUE_NAME"], + "total_revenue": ["GRND_TOTAL_REVENUE"], + "total_expenses": ["GRND_TOTAL_EXPENSE"], + "unallocated_revenue": ["TOT_REVENUE_ALL_NOTALLOC"], + "unallocated_expenses": ["TOT_EXPENSE_ALL_NOTALLOC"], + "student_aid": ["STUDENTAID_TOTAL"], + "recruiting_expenses": ["RECRUITEXP_TOTAL"], + "head_coach_salary_men": ["HDCOACH_SALARY_MEN"], + "head_coach_salary_women": ["HDCOACH_SALARY_WOMEN"], + "asst_coach_salary_men": ["ASCOACH_SALARY_MEN"], + "asst_coach_salary_women": ["ASCOACH_SALARY_WOMEN"], + "participants_men": ["UNDUP_CT_PARTIC_MEN", "IL_SUM_PARTIC_MEN"], + "participants_women": ["UNDUP_CT_PARTIC_WOMEN", "IL_SUM_PARTIC_WOMEN"], + "ft_undergrad_enrollment": ["EFTOTALCOUNT"], +} + +SPORT_VARIANTS = { + "unitid": ["UNITID"], + "sport": ["SPORTS"], + "participants_men": ["SUM_PARTIC_MEN", "PARTIC_MEN"], + "participants_women": ["SUM_PARTIC_WOMEN", "PARTIC_WOMEN"], + "total_revenue": ["TOTAL_REVENUE_ALL"], + "total_expenses": ["TOTAL_EXPENSE_ALL"], +} + +INSTITUTION_COLUMNS = ["unitid", "year"] + [c for c in INSTITUTION_VARIANTS if c != "unitid"] +SPORT_COLUMNS = ["unitid", "year"] + [c for c in SPORT_VARIANTS if c != "unitid"] +_TEXT_COLUMNS = {"classification", "sport"} +_INT32_COLUMNS = {"participants_men", "participants_women", "ft_undergrad_enrollment"} + + +def _find_file(year_dir: Path, kind: str) -> Path | None: + """Find the institution ("inst") or per-sport ("school") spreadsheet.""" + for path in sorted(year_dir.glob("*.xls*")): + if kind in path.name.lower(): + return path + return None + + +def _read(path: Path, variants: dict, unitid_filter: int | None) -> pl.DataFrame | None: + df = pd.read_excel(path) + upper_cols = {str(c).strip().upper(): c for c in df.columns} + col_map = {} + for canonical, candidates in variants.items(): + for candidate in candidates: + if candidate in upper_cols: + col_map[canonical] = upper_cols[candidate] + break + if "unitid" not in col_map: + return None + + df = df[list(col_map.values())].rename(columns={v: k for k, v in col_map.items()}) + if unitid_filter is not None: + df = df[df["unitid"] == unitid_filter] + return pl.from_pandas(df) + + +def _normalize(df: pl.DataFrame, year: int, columns: list[str]) -> pl.DataFrame: + df = df.with_columns(pl.lit(year).alias("year")) + for col in columns: + if col not in df.columns: + df = df.with_columns(pl.lit(None).alias(col)) + if col in _TEXT_COLUMNS: + df = df.with_columns(pl.col(col).cast(pl.Utf8).str.strip_chars()) + elif col in _INT32_COLUMNS or col in ("unitid", "year"): + df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False).cast(pl.Int32, strict=False)) + else: + df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False).round(0).cast(pl.Int64, strict=False)) + return df.select(columns) + + +def _insert(conn: duckdb.DuckDBPyConnection, table: str, df: pl.DataFrame) -> int: + if df.height == 0: + return 0 + conn.register("_tmp_eada", df.to_arrow()) + conn.execute(f"INSERT INTO {table} SELECT * FROM _tmp_eada") + conn.unregister("_tmp_eada") + return df.height + + +def load_eada( + conn: duckdb.DuckDBPyConnection, + year_range: range, + unitid_filter: int | None = UD_UNITID, +) -> dict[str, int]: + """Load EADA institution and per-sport data for each year in the range.""" + totals = {"institution": 0, "sport": 0} + for year in year_range: + year_dir = config.EADA_DATA_DIR / str(year) + conn.execute("DELETE FROM raw_eada_institution WHERE year = ?", [year]) + conn.execute("DELETE FROM raw_eada_sport WHERE year = ?", [year]) + + inst_path = _find_file(year_dir, "inst") + if inst_path is None: + print(f" No EADA institution file for {year}, skipping") + continue + inst = _read(inst_path, INSTITUTION_VARIANTS, unitid_filter) + if inst is not None: + totals["institution"] += _insert( + conn, "raw_eada_institution", _normalize(inst, year, INSTITUTION_COLUMNS) + ) + + sport_path = _find_file(year_dir, "school") + if sport_path is not None: + sports = _read(sport_path, SPORT_VARIANTS, unitid_filter) + if sports is not None: + sports = _normalize(sports, year, SPORT_COLUMNS).filter(pl.col("sport").is_not_null()) + totals["sport"] += _insert(conn, "raw_eada_sport", sports) + + return totals diff --git a/tests/test_eada.py b/tests/test_eada.py new file mode 100644 index 0000000..0bd6ad4 --- /dev/null +++ b/tests/test_eada.py @@ -0,0 +1,172 @@ +"""Tests for EADA athletics download, loader, and dashboard queries.""" + +import io +import zipfile + +import httpx +import pandas as pd +import respx + +import admin_analytics.config as config +from admin_analytics.config import UD_UNITID +from admin_analytics.dashboard.queries import ( + query_athletics_comp, + query_athletics_expenses, + query_athletics_vs_university, +) +from admin_analytics.eada.download import download_all +from admin_analytics.eada.loader import load_eada + +OTHER_UNITID = 999999 + + +def _write_year(year_dir, inst_name="instLevel.xlsx", school_name="Schools.xlsx", lower=False): + """Write minimal EADA institution and per-sport spreadsheets.""" + year_dir.mkdir(parents=True, exist_ok=True) + inst = pd.DataFrame({ + "unitid": [UD_UNITID, OTHER_UNITID], + "classification_name": ["NCAA Division I-FCS", "NCAA Division II"], + "EFTotalCount": [18000, 5000], + "STUDENTAID_TOTAL": [15_000_000, 1_000_000], + "HDCOACH_SALARY_MEN": [240_000.0, 80_000.0], + "UNDUP_CT_PARTIC_MEN": [300, 100], + "UNDUP_CT_PARTIC_WOMEN": [320, 110], + "TOT_EXPENSE_ALL_NOTALLOC": [14_000_000, 2_000_000], + "GRND_TOTAL_REVENUE": [48_000_000, 9_000_000], + "GRND_TOTAL_EXPENSE": [48_000_000, 9_000_000], + }) + if lower: + inst.columns = [c.lower() for c in inst.columns] + inst.to_excel(year_dir / inst_name, index=False) + pd.DataFrame({ + "unitid": [UD_UNITID, UD_UNITID, UD_UNITID, OTHER_UNITID], + "SUM_PARTIC_MEN": [110, 15, 30, 50], + "SUM_PARTIC_WOMEN": [None, 16, 28, 40], + "TOTAL_REVENUE_ALL": [9_000_000, 6_000_000, 2_000_000, 1_000_000], + "TOTAL_EXPENSE_ALL": [9_000_000, 6_000_000, 2_000_000, 1_000_000], + "Sports": ["Football", "Basketball", "Soccer", "Football"], + }).to_excel(year_dir / school_name, index=False) + + +class TestLoader: + def test_loads_ud_only(self, db_conn, tmp_path, monkeypatch): + monkeypatch.setattr(config, "EADA_DATA_DIR", tmp_path) + _write_year(tmp_path / "2024") + totals = load_eada(db_conn, range(2024, 2025)) + assert totals == {"institution": 1, "sport": 3} + row = db_conn.execute( + "SELECT total_expenses, unallocated_expenses, participants_men, " + "ft_undergrad_enrollment, classification FROM raw_eada_institution" + ).fetchone() + assert row == (48_000_000, 14_000_000, 300, 18000, "NCAA Division I-FCS") + + def test_resolves_column_and_filename_variants(self, db_conn, tmp_path, monkeypatch): + monkeypatch.setattr(config, "EADA_DATA_DIR", tmp_path) + _write_year(tmp_path / "2010", inst_name="instLevelData.xlsx", + school_name="schools.xlsx", lower=True) + totals = load_eada(db_conn, range(2010, 2011)) + assert totals["institution"] == 1 + assert db_conn.execute( + "SELECT total_expenses FROM raw_eada_institution WHERE year = 2010" + ).fetchone()[0] == 48_000_000 + + def test_reload_replaces_year(self, db_conn, tmp_path, monkeypatch): + monkeypatch.setattr(config, "EADA_DATA_DIR", tmp_path) + _write_year(tmp_path / "2024") + load_eada(db_conn, range(2024, 2025)) + load_eada(db_conn, range(2024, 2025)) + assert db_conn.execute("SELECT COUNT(*) FROM raw_eada_sport").fetchone()[0] == 3 + + def test_missing_year_skipped(self, db_conn, tmp_path, monkeypatch): + monkeypatch.setattr(config, "EADA_DATA_DIR", tmp_path) + assert load_eada(db_conn, range(2024, 2025)) == {"institution": 0, "sport": 0} + + +class TestDownload: + @respx.mock + def test_downloads_spreadsheets_only(self, tmp_path, monkeypatch): + monkeypatch.setattr(config, "EADA_DATA_DIR", tmp_path) + respx.get(f"{config.EADA_API_URL}/fileList").mock(return_value=httpx.Response(200, json=[ + {"Year": 2024, "FileName": "EADA_2023-2024.zip"}, + {"Year": 2024, "FileName": "EADA_All_Data_Combined_2023-2024_SAS_SPSS_EXCEL.zip"}, + ])) + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("instLevel.xlsx", b"x") + zf.writestr("schools.sav", b"x") + route = respx.get(f"{config.EADA_API_URL}/file").mock( + return_value=httpx.Response(200, content=buf.getvalue()) + ) + download_all(range(2024, 2025)) + assert route.calls.last.request.url.params["fileName"] == "EADA_2023-2024.zip" + assert sorted(p.name for p in (tmp_path / "2024").iterdir()) == ["instLevel.xlsx"] + + +def _seed(conn): + for year, total, unalloc in [(2023, 40_000_000, 10_000_000), (2024, 50_000_000, 20_000_000)]: + conn.execute( + "INSERT INTO raw_eada_institution (unitid, year, total_expenses, unallocated_expenses, " + "participants_men, participants_women, ft_undergrad_enrollment) VALUES (?, ?, ?, ?, 300, 300, 20000)", + [UD_UNITID, year, total, unalloc], + ) + for sport, amount in [("Football", 10_000_000), ("Basketball", 5_000_000), ("Soccer", 3_000_000)]: + conn.execute( + "INSERT INTO raw_eada_sport (unitid, year, sport, total_expenses) VALUES (?, ?, ?, ?)", + [UD_UNITID, year, sport, amount], + ) + conn.execute( + "INSERT INTO raw_ipeds_finance (unitid, year, total_expenses, instruction_expenses, " + "institutional_support_expenses) VALUES (?, 2023, 1000000000, 400000000, 100000000)", + [UD_UNITID], + ) + for year, value in [(2023, 300.0), (2024, 310.0)]: + conn.execute( + "INSERT INTO raw_cpi_u (year, month, value, series_id) VALUES (?, 1, ?, 'CUUR0000SA0')", + [year, value], + ) + + +class TestQueries: + def test_expenses_split_and_per_student(self, db_conn): + _seed(db_conn) + df = query_athletics_expenses(db_conn) + assert df["year"].to_list() == [2023, 2024] + latest = df.row(1, named=True) + assert latest["football"] == 10_000_000 + assert latest["other_sports"] == 3_000_000 + assert latest["per_student"] == 2500 + # Latest year is the CPI base, so adjusted equals nominal + assert latest["total_expenses_cpi"] == 50_000_000 + + def test_vs_university_only_overlapping_years(self, db_conn): + _seed(db_conn) + df = query_athletics_vs_university(db_conn) + assert df["year"].to_list() == [2023] + assert df["athletics_pct"][0] == 4.0 + assert df["athletics_index"][0] == 100.0 + assert query_athletics_vs_university(db_conn, since=2024).height == 0 + + def test_comp_matches_abbreviated_titles(self, db_conn): + for year, name, title, comp in [ + (2022, "A", "DIR, INTERCOLLEGIATE ATHLC & REC SR", 700_000), + (2023, "A", "DIRECTOR, INTERCOLLEGIATE ATHELTICS & RECREATION", 710_000), + (2023, "B", "HEAD FOOTBALL COACH", 680_000), + (2023, "C", "HEAD FOOTBALL COACH TO 1/1/23", 500_000), + (2023, "D", "FORMER ATHLETIC DIRECTOR", 400_000), + (2023, "E", "PROVOST", 800_000), + ]: + db_conn.execute( + "INSERT INTO raw_990_schedule_j (tax_year, person_name, title, total_compensation) " + "VALUES (?, ?, ?, ?)", + [year, name, title, comp], + ) + df = query_athletics_comp(db_conn) + assert set(zip(df["tax_year"], df["role"], df["person_name"])) == { + (2022, "Athletic Director", "A"), + (2023, "Athletic Director", "A"), + (2023, "Head Football Coach", "B"), + } + + def test_empty(self, db_conn): + assert query_athletics_expenses(db_conn).height == 0 + assert query_athletics_comp(db_conn).height == 0