diff --git a/CLAUDE.md b/CLAUDE.md index b0941d4..05a6021 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,7 @@ Currently in planning. Phase 1 (Data Acquisition) is planned but not yet built. |--------|--------|-----------------| | IRS 990 bulk XML | XML (versioned schemas) | Filing financials, Part VII compensation, Schedule J detailed compensation | | IPEDS | CSV bulk downloads | Institution directory (HD), finance by function (F1A/F2), staffing (S/SAL), enrollment (EF) | +| EADA (Equity in Athletics) | Excel via ope.ed.gov/athletics API | Athletics expenses by sport, coaching salaries, participants (keyed by UNITID, FY2007+) | | BLS CPI-U | Flat file or API | Consumer Price Index for inflation-adjusted compensation analysis | | Admin office web pages (stretch) | HTML scraping | Staff directory headcounts | @@ -55,6 +56,7 @@ Not yet implemented. When built, the CLI will support: admin-analytics ingest ipeds --year-range 2005-2024 admin-analytics ingest irs990 --year-range 2005-2024 admin-analytics ingest cpi +admin-analytics ingest eada --year-range 2007-2025 admin-analytics ingest all ``` diff --git a/src/admin_analytics/cli.py b/src/admin_analytics/cli.py index 8d550c8..f8080c8 100644 --- a/src/admin_analytics/cli.py +++ b/src/admin_analytics/cli.py @@ -2,7 +2,7 @@ from typing import Annotated import typer -from admin_analytics.config import DEFAULT_YEAR_RANGE +from admin_analytics.config import DEFAULT_YEAR_RANGE, EADA_YEAR_RANGE from admin_analytics.db.connection import get_connection from admin_analytics.db.schema import ensure_schema @@ -130,6 +130,35 @@ def cpi( conn.close() +@ingest_app.command() +def eada( + year_range: Annotated[ + str, typer.Option(help="Academic-year-end range, e.g. 2007-2025 (2025 = 2024-25)") + ] = f"{EADA_YEAR_RANGE.start}-{EADA_YEAR_RANGE.stop - 1}", + force: Annotated[ + bool, typer.Option("--force", help="Re-download even if files exist") + ] = False, +) -> None: + """Ingest EADA athletics finance data for the University of Delaware.""" + years = _parse_year_range(year_range) + conn = get_connection() + ensure_schema(conn) + + from admin_analytics.eada.download import download_all + from admin_analytics.eada.loader import load_eada + + typer.echo(f"Downloading EADA data for {years.start}-{years.stop - 1}...") + download_all(years, force=force) + + typer.echo("Loading EADA data into database...") + totals = load_eada(conn, years) + typer.echo( + f"EADA ingestion complete: {totals['institution']} institution rows, " + f"{totals['sport']} sport rows." + ) + conn.close() + + @ingest_app.command() def scrape() -> None: """Scrape UD staff directory pages for admin headcounts.""" @@ -202,4 +231,5 @@ def ingest_all( ipeds(year_range=year_range, component="all", force=force) irs990(year_range=year_range, force=force) cpi(force=force) + eada(force=force) scrape() diff --git a/src/admin_analytics/config.py b/src/admin_analytics/config.py index 16cb959..eabda44 100644 --- a/src/admin_analytics/config.py +++ b/src/admin_analytics/config.py @@ -18,6 +18,13 @@ DEFAULT_YEAR_RANGE = range(2005, 2025) # 2005-2024 inclusive BLS_CPI_URL = "https://download.bls.gov/pub/time.series/cu/cu.data.0.Current" BLS_DATA_DIR = DATA_DIR / "bls" +# EADA (Equity in Athletics Disclosure Act), keyed by UNITID. +# Years are academic-year end (2024 = 2023-24), matching IPEDS fiscal years. +# Pre-2007 files use coded column names and are not supported. +EADA_API_URL = "https://ope.ed.gov/athletics/api/dataFiles" +EADA_DATA_DIR = DATA_DIR / "eada" +EADA_YEAR_RANGE = range(2007, 2026) # 2006-07 through 2024-25 + # IRS 990 UD_EINS = [516000297, 516017306] # UD + UD Research Foundation IRS990_BASE_URL = "https://apps.irs.gov/pub/epostcard/990/xml" diff --git a/src/admin_analytics/dashboard/app.py b/src/admin_analytics/dashboard/app.py index a068783..7dc8c7a 100644 --- a/src/admin_analytics/dashboard/app.py +++ b/src/admin_analytics/dashboard/app.py @@ -5,7 +5,7 @@ from dash import dcc, html, Input, Output from admin_analytics.db.connection import get_connection from admin_analytics.dashboard.pages import ( - overview, compensation, staffing, headcount, endowment, philanthropy, about, + overview, compensation, staffing, headcount, endowment, philanthropy, athletics, about, ) @@ -29,6 +29,7 @@ def create_app() -> dash.Dash: dcc.Tab(label="Staffing & Enrollment", value="staffing"), dcc.Tab(label="Endowment", value="endowment"), dcc.Tab(label="Philanthropy", value="philanthropy"), + dcc.Tab(label="Athletics", value="athletics"), dcc.Tab(label="Current Headcount", value="headcount"), dcc.Tab(label="About", value="about"), ], @@ -52,6 +53,8 @@ def create_app() -> dash.Dash: return endowment.layout(conn) elif tab == "philanthropy": return philanthropy.layout(conn) + elif tab == "athletics": + return athletics.layout(conn) elif tab == "headcount": return headcount.layout(conn) elif tab == "about": diff --git a/src/admin_analytics/dashboard/pages/about.py b/src/admin_analytics/dashboard/pages/about.py index 8663cc2..af539fd 100644 --- a/src/admin_analytics/dashboard/pages/about.py +++ b/src/admin_analytics/dashboard/pages/about.py @@ -21,6 +21,7 @@ All data is drawn from public, open-access sources: |--------|-----------|-------------|----------| | **IPEDS** | U.S. Dept. of Education, NCES | Institutional directory, expenses by function, staffing by occupation, enrollment, endowment, philanthropic gifts | 2005-2024 | | **IRS Form 990** | Internal Revenue Service | Executive compensation (Schedule J), filing financials for UD and UD Research Foundation | Tax years 2015-2025 | +| **EADA** | U.S. Dept. of Education, OPE | Athletics expenses by sport, participants, coaching salaries, full-time undergraduate enrollment | FY 2007-2025 | | **BLS CPI-U** | Bureau of Labor Statistics | Consumer Price Index for inflation adjustment (series CUUR0000SA0) | Full history | | **UD Staff Directories** | University of Delaware public web pages | Administrative office headcounts (College of Engineering line management, Provost's Office) | Current snapshot | @@ -53,6 +54,14 @@ year. For 2005-2019, only beginning and end-of-year values are reported. contracts. The compensation-vs-giving comparison indexes the President and VP of Development compensation against total philanthropic revenue. +**Athletics** uses Equity in Athletics Disclosure Act (EADA) filings, keyed by +the same IPEDS UNITID. Expenses are split into football, basketball, other +sports, and spending not allocated to a single team. Per-student figures use +EADA's full-time undergraduate enrollment. Head coach and athletic director +compensation comes from Schedule J. EADA reports athletics revenue equal to +expenses, so it measures the cost of athletics, not the institutional +subsidy. + **Inflation Adjustment** uses the BLS CPI-U annual average (all items, U.S. city average, not seasonally adjusted). CPI-adjusted values are expressed in the most recent available year's dollars. diff --git a/src/admin_analytics/dashboard/pages/athletics.py b/src/admin_analytics/dashboard/pages/athletics.py new file mode 100644 index 0000000..480d760 --- /dev/null +++ b/src/admin_analytics/dashboard/pages/athletics.py @@ -0,0 +1,232 @@ +"""Page: Intercollegiate Athletics.""" + +import duckdb +from dash import html, dcc +import plotly.graph_objects as go + +from admin_analytics.dashboard.components import ( + graph, + kpi_card, + kpi_row, + chart_row, + no_data, +) +from admin_analytics.dashboard.queries import ( + CONSISTENT_REPORTING_YEAR, + FBS_TRANSITION_YEAR, + query_athletics_expenses, + query_athletics_vs_university, + query_athletics_comp, +) + +_NO_DATA = no_data("No athletics data loaded. Run: admin-analytics ingest eada") + +_ROLE_COLORS = { + "Head Football Coach": "#00539F", + "Head Basketball Coach": "#E07A5F", + "Athletic Director": "#7FB069", +} + +_CONTEXT = """ +#### Context + +**University finances.** In October 2024, President Assanis projected a FY2025 +shortfall of $20–40 million, citing inflation, salary competition, financial +aid, and health insurance premiums. The response included a partial hiring +freeze, delayed capital projects, $11 million in payroll reductions, and $14 +million saved by "reducing travel and other expenses" +([Newark Post, Oct. 4, 2024](https://www.newarkpostonline.com/news/assanis-on-uds-financial-woes-were-not-out-of-the-woods-yet/article_81015f48-8226-11ef-a864-db72de6d8f73.html)). + +**Athletics.** The same fiscal year (2024–25) was football's transition to the +Football Bowl Subdivision ahead of full Conference USA membership on July 1, +2025. The Athletic Director told the Faculty Senate in December 2024 that the +department had cut institutional support by 24% over five years while growing +self-generated revenue and "gradually rais\\[ing\\] student fees," and that the $5 +million FBS reclassification fee was covered by philanthropy +([UDaily](https://udel.edu/udaily/2025/february/faculty-senate-meeting-december-2024)). + +#### Reading these numbers + +- **EADA reports cost, not subsidy.** UD reports athletics revenue equal to + expenses every year: institutional support and student fees are counted as + revenue to balance the books. These figures show what athletics *costs*, + not how much of that cost the university or students cover. Subsidy figures + from UD's NCAA financial reports are published by the + [Knight-Newhouse College Athletics Database](https://knightnewhousedata.org/fcs/university-delaware). +- **"Not allocated by sport"** is spending not assigned to a single team: + administration, facilities, conference and reclassification fees, and + similar. It doubled in 2024–25. Some of that may be one-time transition + costs, and some may be paid by gifts. +- **2011–12 drop.** Total and unallocated expenses fall sharply between + 2010–11 and 2011–12 while per-sport spending keeps rising, which suggests a + change in how UD reported shared costs rather than a real cut. The + growth comparison therefore starts in FY 2012. +- **Years** are fiscal years ending June 30 (2025 = 2024–25). University-wide + comparisons stop where IPEDS finance data ends. +""" + + +def _transition_marker(fig: go.Figure) -> None: + fig.add_vline( + x=FBS_TRANSITION_YEAR, line_dash="dot", line_color="#999", + annotation_text="FBS transition", annotation_position="top left", + annotation_font_size=11, annotation_font_color="#666", + ) + + +def layout(conn: duckdb.DuckDBPyConnection): + df = query_athletics_expenses(conn) + if df.height == 0: + return _NO_DATA + + exp = df.to_pandas() + latest = exp.iloc[-1] + prior = exp.iloc[-2] if len(exp) > 1 else None + year = int(latest["year"]) + + vs_df = query_athletics_vs_university(conn) + vs = vs_df.to_pandas() if vs_df.height > 0 else None + growth_df = query_athletics_vs_university(conn, since=CONSISTENT_REPORTING_YEAR) + growth = growth_df.to_pandas() if growth_df.height > 0 else None + + # KPI cards + cards = [ + kpi_card( + "Athletics Expenses", + f"${latest['total_expenses'] / 1e6:.1f}M", + f"FY {year} ({year - 1}–{str(year)[2:]})", + ), + ] + if prior is not None: + change = latest["total_expenses"] - prior["total_expenses"] + cards.append(kpi_card( + "One-Year Change", + f"{change / prior['total_expenses'] * 100:+.0f}%", + f"{'+' if change >= 0 else '−'}${abs(change) / 1e6:.1f}M vs FY {int(prior['year'])}", + )) + cards.append(kpi_card( + "Per Full-Time Undergraduate", + f"${int(latest['per_student']):,}", + f"FY {year}", + )) + if vs is not None: + last_vs = vs.iloc[-1] + cards.append(kpi_card( + "Share of University Expenses", + f"{last_vs['athletics_pct']:.1f}%", + f"FY {int(last_vs['year'])} (latest IPEDS year)", + )) + kpis = kpi_row(cards) + + # Expenses by category, stacked + by_cat_fig = go.Figure() + for col, label, color in [ + ("football", "Football", "#00539F"), + ("basketball", "Basketball", "#E07A5F"), + ("other_sports", "Other sports", "#7FB069"), + ("unallocated_expenses", "Not allocated by sport", "#B8B8B8"), + ]: + by_cat_fig.add_trace(go.Bar( + x=exp["year"], y=exp[col].astype(float) / 1e6, + name=label, marker_color=color, + )) + by_cat_fig.add_trace(go.Scatter( + x=exp["year"], y=exp["total_expenses_cpi"] / 1e6, + mode="lines", name="Total, CPI-adjusted", + line={"color": "#333", "dash": "dash"}, + )) + _transition_marker(by_cat_fig) + by_cat_fig.update_layout( + title="Athletics Expenses by Category (Millions $)", + xaxis_title="Fiscal Year", yaxis_title="Millions $", + barmode="stack", template="plotly_white", height=440, + ) + + # Growth vs university spending + growth_fig = go.Figure() + if growth is not None: + for col, label, color, dash in [ + ("athletics_index", "Athletics", "#00539F", None), + ("university_index", "Total university", "#E07A5F", None), + ("instruction_index", "Instruction", "#7FB069", None), + ("cpi_index", "CPI-U", "#FFD200", "dash"), + ]: + growth_fig.add_trace(go.Scatter( + x=growth["year"], y=growth[col], mode="lines+markers", name=label, + line={"color": color, "dash": dash}, + )) + growth_fig.add_hline(y=100, line_dash="dot", line_color="#ccc") + base_year = int(growth.iloc[0]["year"]) + else: + base_year = None + growth_fig.update_layout( + title=f"Spending Growth (Indexed, FY {base_year} = 100)" if base_year else "Spending Growth", + xaxis_title="Fiscal Year", yaxis_title="Index", + template="plotly_white", height=380, + ) + + # Per-student cost + per_student_fig = go.Figure() + per_student_fig.add_trace(go.Scatter( + x=exp["year"], y=exp["per_student"], + mode="lines+markers", name="Nominal", line={"color": "#00539F"}, + )) + per_student_fig.add_trace(go.Scatter( + x=exp["year"], y=exp["per_student_cpi"], + mode="lines+markers", name="CPI-adjusted", line={"color": "#FFD200", "dash": "dash"}, + )) + _transition_marker(per_student_fig) + per_student_fig.update_layout( + title="Athletics Expenses per Full-Time Undergraduate", + xaxis_title="Fiscal Year", yaxis_title="$ per student", + template="plotly_white", height=380, + ) + + # Share of university expenses + share_fig = go.Figure() + if vs is not None: + share_fig.add_trace(go.Scatter( + x=vs["year"], y=vs["athletics_pct"], + mode="lines+markers", name="Athletics share", line={"color": "#00539F"}, + )) + share_fig.update_layout( + title="Athletics as % of Total University Expenses", + xaxis_title="Fiscal Year", yaxis_title="%", + template="plotly_white", height=380, + ) + + # Leadership compensation (Schedule J) + comp_df = query_athletics_comp(conn) + comp_fig = go.Figure() + if comp_df.height > 0: + comp = comp_df.to_pandas() + for role, color in _ROLE_COLORS.items(): + subset = comp[comp["role"] == role] + if len(subset) == 0: + continue + # Reindex to every year so missing years show as gaps, not bridges + years = range(int(comp["tax_year"].min()), int(comp["tax_year"].max()) + 1) + subset = subset.set_index("tax_year").reindex(years) + comp_fig.add_trace(go.Scatter( + x=list(years), y=subset["total_compensation"], + mode="lines+markers", name=role, line={"color": color}, + customdata=subset["person_name"].fillna(""), + hovertemplate="%{customdata}
$%{y:,.0f}" + role + "", + )) + comp_fig.update_layout( + title="Athletics Leadership Compensation (IRS 990 Schedule J)", + xaxis_title="Tax Year", yaxis_title="Total compensation ($)", + template="plotly_white", height=400, + ) + + children = [ + kpis, + graph(by_cat_fig), + chart_row(graph(growth_fig), graph(per_student_fig)), + ] + if vs is not None: + children.append(chart_row(graph(share_fig), graph(comp_fig))) + else: + children.append(graph(comp_fig)) + children.append(dcc.Markdown(_CONTEXT, className="note", link_target="_blank")) + return html.Div(children) diff --git a/src/admin_analytics/dashboard/queries.py b/src/admin_analytics/dashboard/queries.py index d3dc65b..a8ea2cf 100644 --- a/src/admin_analytics/dashboard/queries.py +++ b/src/admin_analytics/dashboard/queries.py @@ -553,3 +553,137 @@ def query_headcount_summary(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame: GROUP BY unit, category, is_overhead ORDER BY unit, count DESC """).pl() + + +# --- Athletics (EADA) --- + +# Year the football program spent transitioning to FBS (2024-25); UD joined +# Conference USA as a full member on July 1, 2025 (FY2026). +FBS_TRANSITION_YEAR = 2025 + +# UD's unallocated athletics expenses drop by ~60% between FY2011 and FY2012 +# while per-sport spending keeps rising, consistent with a change in how +# shared costs were reported. Growth comparisons start here. +CONSISTENT_REPORTING_YEAR = 2012 + + +def query_athletics_expenses(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame: + """Athletics expenses by year, split by sport, with CPI and per-student values. + + "Not allocated" is spending EADA does not assign to any single team + (administration, facilities, conference fees, etc.). + """ + return conn.execute(f""" + {_CPI_CTE}, + sports AS ( + SELECT year, + SUM(total_expenses) FILTER (WHERE sport = 'Football') AS football, + SUM(total_expenses) FILTER (WHERE sport = 'Basketball') AS basketball, + SUM(total_expenses) FILTER (WHERE sport NOT IN ('Football', 'Basketball')) + AS other_sports + FROM raw_eada_sport + WHERE unitid = ? + GROUP BY year + ) + SELECT + i.year, + i.classification, + i.total_expenses, + ROUND(i.total_expenses * (SELECT avg_cpi FROM latest_cpi) / ac.avg_cpi, 0) + AS total_expenses_cpi, + s.football, + s.basketball, + s.other_sports, + i.unallocated_expenses, + i.student_aid, + i.participants_men + i.participants_women AS participants, + i.ft_undergrad_enrollment, + ROUND(i.total_expenses * 1.0 / NULLIF(i.ft_undergrad_enrollment, 0), 0) + AS per_student, + ROUND(i.total_expenses * (SELECT avg_cpi FROM latest_cpi) / ac.avg_cpi + / NULLIF(i.ft_undergrad_enrollment, 0), 0) AS per_student_cpi + FROM raw_eada_institution i + LEFT JOIN sports s ON s.year = i.year + LEFT JOIN annual_cpi ac ON ac.year = i.year + WHERE i.unitid = ? + ORDER BY i.year + """, [UD_UNITID, UD_UNITID]).pl() + + +def query_athletics_vs_university( + conn: duckdb.DuckDBPyConnection, since: int | None = None +) -> pl.DataFrame: + """Athletics spending against university-wide IPEDS spending. + + Restricted to years present in both EADA and IPEDS (and >= since, if + given), indexed to the first such year = 100. + """ + return conn.execute(""" + WITH annual_cpi AS ( + SELECT year, AVG(value) AS avg_cpi FROM raw_cpi_u GROUP BY year + ), + joined AS ( + SELECT a.year, + a.total_expenses AS athletics, + f.total_expenses AS university, + f.instruction_expenses AS instruction, + f.institutional_support_expenses AS inst_support, + ac.avg_cpi + FROM raw_eada_institution a + JOIN raw_ipeds_finance f ON f.unitid = a.unitid AND f.year = a.year + LEFT JOIN annual_cpi ac ON ac.year = a.year + WHERE a.unitid = ? AND a.year >= ? + ), + base AS (SELECT * FROM joined ORDER BY year LIMIT 1) + SELECT j.year, j.athletics, j.university, + ROUND(j.athletics * 100.0 / NULLIF(j.university, 0), 2) AS athletics_pct, + ROUND(j.athletics * 100.0 / NULLIF(b.athletics, 0), 1) AS athletics_index, + ROUND(j.university * 100.0 / NULLIF(b.university, 0), 1) AS university_index, + ROUND(j.instruction * 100.0 / NULLIF(b.instruction, 0), 1) AS instruction_index, + ROUND(j.inst_support * 100.0 / NULLIF(b.inst_support, 0), 1) AS inst_support_index, + ROUND(j.avg_cpi * 100.0 / NULLIF(b.avg_cpi, 0), 1) AS cpi_index + FROM joined j CROSS JOIN base b + ORDER BY j.year + """, [UD_UNITID, since or 0]).pl() + + +_ATHLETICS_ROLES = [ + ("Head Football Coach", r"FOOTBALL"), + ("Head Basketball Coach", r"BASKETBALL"), + # Titles abbreviate and misspell freely: "DIR, INTERCOLLEGIATE ATHLCS", + # "DIRECTOR, INTERCOLLEGIATE ATHELTICS", "ATHLETIC DIRECTOR". + ("Athletic Director", r"ATHLETIC DIRECTOR|INTERCOLLEGIATE ATH"), +] + + +def query_athletics_comp(conn: duckdb.DuckDBPyConnection) -> pl.DataFrame: + """Schedule J compensation for head coaches and the athletic director. + + When a role changed hands mid-year (e.g. a departing coach and a new one), + the highest-paid person in that role is kept for the year. + """ + df = conn.execute(""" + SELECT tax_year, person_name, title, total_compensation + FROM raw_990_schedule_j + WHERE regexp_matches(upper(title), 'COACH|ATHL|ATHEL') + AND NOT regexp_matches(upper(title), 'FORMER|ASSISTANT|ASSOCIATE') + """).pl() + if df.height == 0: + return pl.DataFrame(schema={ + "tax_year": pl.Int32, "role": pl.Utf8, + "person_name": pl.Utf8, "total_compensation": pl.Int64, + }) + role_expr = pl.lit(None, dtype=pl.Utf8) + for role, pattern in reversed(_ATHLETICS_ROLES): + role_expr = ( + pl.when(pl.col("title").str.to_uppercase().str.contains(pattern)) + .then(pl.lit(role)).otherwise(role_expr) + ) + return ( + df.with_columns(role_expr.alias("role")) + .filter(pl.col("role").is_not_null()) + .sort("total_compensation", descending=True) + .unique(subset=["tax_year", "role"], keep="first") + .select("tax_year", "role", "person_name", "total_compensation") + .sort(["role", "tax_year"]) + ) diff --git a/src/admin_analytics/db/schema.py b/src/admin_analytics/db/schema.py index 7654451..f81f697 100644 --- a/src/admin_analytics/db/schema.py +++ b/src/admin_analytics/db/schema.py @@ -116,6 +116,39 @@ TABLES = { PRIMARY KEY (unitid, year) ) """, + "raw_eada_institution": """ + CREATE TABLE IF NOT EXISTS raw_eada_institution ( + unitid INTEGER NOT NULL, + year INTEGER NOT NULL, + classification VARCHAR, + total_revenue BIGINT, + total_expenses BIGINT, + unallocated_revenue BIGINT, + unallocated_expenses BIGINT, + student_aid BIGINT, + recruiting_expenses BIGINT, + head_coach_salary_men BIGINT, + head_coach_salary_women BIGINT, + asst_coach_salary_men BIGINT, + asst_coach_salary_women BIGINT, + participants_men INTEGER, + participants_women INTEGER, + ft_undergrad_enrollment INTEGER, + PRIMARY KEY (unitid, year) + ) + """, + "raw_eada_sport": """ + CREATE TABLE IF NOT EXISTS raw_eada_sport ( + unitid INTEGER NOT NULL, + year INTEGER NOT NULL, + sport VARCHAR NOT NULL, + participants_men INTEGER, + participants_women INTEGER, + total_revenue BIGINT, + total_expenses BIGINT, + PRIMARY KEY (unitid, year, sport) + ) + """, "raw_cpi_u": """ CREATE TABLE IF NOT EXISTS raw_cpi_u ( year INTEGER NOT NULL, diff --git a/src/admin_analytics/eada/__init__.py b/src/admin_analytics/eada/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/admin_analytics/eada/download.py b/src/admin_analytics/eada/download.py new file mode 100644 index 0000000..37ac356 --- /dev/null +++ b/src/admin_analytics/eada/download.py @@ -0,0 +1,57 @@ +"""Download EADA athletics data files from the Department of Education.""" + +import io +import zipfile +from pathlib import Path + +import httpx + +from admin_analytics import config + +# Only the spreadsheets are needed; the zips also carry SAS/SPSS/Word copies. +_SPREADSHEET_SUFFIXES = (".xls", ".xlsx") + + +def list_data_files(client: httpx.Client) -> dict[int, str]: + """Return {year: filename} for the per-year EADA data zips. + + The API also lists "All_Data_Combined" zips with one record per + institution; those duplicate the per-year institution file and are skipped. + """ + resp = client.get(f"{config.EADA_API_URL}/fileList") + resp.raise_for_status() + return { + int(entry["Year"]): entry["FileName"] + for entry in resp.json() + if "Combined" not in entry["FileName"] + } + + +def _dest_dir(year: int) -> Path: + return config.EADA_DATA_DIR / str(year) + + +def download_year(client: httpx.Client, year: int, filename: str, force: bool = False) -> Path: + """Download and extract the spreadsheets for one EADA year.""" + dest = _dest_dir(year) + if not force and dest.exists() and any(dest.glob("*.xls*")): + return dest + dest.mkdir(parents=True, exist_ok=True) + resp = client.get(f"{config.EADA_API_URL}/file", params={"fileName": filename}) + resp.raise_for_status() + with zipfile.ZipFile(io.BytesIO(resp.content)) as zf: + for name in zf.namelist(): + if name.lower().endswith(_SPREADSHEET_SUFFIXES): + zf.extract(name, dest) + return dest + + +def download_all(year_range: range, force: bool = False) -> None: + """Download all available EADA years in the range.""" + with httpx.Client(follow_redirects=True, timeout=600.0) as client: + available = list_data_files(client) + for year in year_range: + if year not in available: + print(f" Warning: EADA {year} not available") + continue + download_year(client, year, available[year], force=force) diff --git a/src/admin_analytics/eada/loader.py b/src/admin_analytics/eada/loader.py new file mode 100644 index 0000000..c30caff --- /dev/null +++ b/src/admin_analytics/eada/loader.py @@ -0,0 +1,133 @@ +"""Load EADA athletics spreadsheets into raw_eada_* tables. + +Each EADA year ships two spreadsheets: an institution-level file (one row per +school) and a schools file (one row per sport per school). Filenames and +column-name casing vary across years, so both are resolved loosely. + +Note that EADA revenue equals expenses for most public institutions: the +institutional subsidy is reported as revenue to balance the books, so EADA +measures athletics *cost* but does not separate out the subsidy. +""" + +from pathlib import Path + +import duckdb +import pandas as pd +import polars as pl + +from admin_analytics import config +from admin_analytics.config import UD_UNITID + +# Canonical column -> candidate EADA columns, first match wins (case-insensitive). +INSTITUTION_VARIANTS = { + "unitid": ["UNITID"], + "classification": ["CLASSIFICATION_NAME", "VALUE_NAME"], + "total_revenue": ["GRND_TOTAL_REVENUE"], + "total_expenses": ["GRND_TOTAL_EXPENSE"], + "unallocated_revenue": ["TOT_REVENUE_ALL_NOTALLOC"], + "unallocated_expenses": ["TOT_EXPENSE_ALL_NOTALLOC"], + "student_aid": ["STUDENTAID_TOTAL"], + "recruiting_expenses": ["RECRUITEXP_TOTAL"], + "head_coach_salary_men": ["HDCOACH_SALARY_MEN"], + "head_coach_salary_women": ["HDCOACH_SALARY_WOMEN"], + "asst_coach_salary_men": ["ASCOACH_SALARY_MEN"], + "asst_coach_salary_women": ["ASCOACH_SALARY_WOMEN"], + "participants_men": ["UNDUP_CT_PARTIC_MEN", "IL_SUM_PARTIC_MEN"], + "participants_women": ["UNDUP_CT_PARTIC_WOMEN", "IL_SUM_PARTIC_WOMEN"], + "ft_undergrad_enrollment": ["EFTOTALCOUNT"], +} + +SPORT_VARIANTS = { + "unitid": ["UNITID"], + "sport": ["SPORTS"], + "participants_men": ["SUM_PARTIC_MEN", "PARTIC_MEN"], + "participants_women": ["SUM_PARTIC_WOMEN", "PARTIC_WOMEN"], + "total_revenue": ["TOTAL_REVENUE_ALL"], + "total_expenses": ["TOTAL_EXPENSE_ALL"], +} + +INSTITUTION_COLUMNS = ["unitid", "year"] + [c for c in INSTITUTION_VARIANTS if c != "unitid"] +SPORT_COLUMNS = ["unitid", "year"] + [c for c in SPORT_VARIANTS if c != "unitid"] +_TEXT_COLUMNS = {"classification", "sport"} +_INT32_COLUMNS = {"participants_men", "participants_women", "ft_undergrad_enrollment"} + + +def _find_file(year_dir: Path, kind: str) -> Path | None: + """Find the institution ("inst") or per-sport ("school") spreadsheet.""" + for path in sorted(year_dir.glob("*.xls*")): + if kind in path.name.lower(): + return path + return None + + +def _read(path: Path, variants: dict, unitid_filter: int | None) -> pl.DataFrame | None: + df = pd.read_excel(path) + upper_cols = {str(c).strip().upper(): c for c in df.columns} + col_map = {} + for canonical, candidates in variants.items(): + for candidate in candidates: + if candidate in upper_cols: + col_map[canonical] = upper_cols[candidate] + break + if "unitid" not in col_map: + return None + + df = df[list(col_map.values())].rename(columns={v: k for k, v in col_map.items()}) + if unitid_filter is not None: + df = df[df["unitid"] == unitid_filter] + return pl.from_pandas(df) + + +def _normalize(df: pl.DataFrame, year: int, columns: list[str]) -> pl.DataFrame: + df = df.with_columns(pl.lit(year).alias("year")) + for col in columns: + if col not in df.columns: + df = df.with_columns(pl.lit(None).alias(col)) + if col in _TEXT_COLUMNS: + df = df.with_columns(pl.col(col).cast(pl.Utf8).str.strip_chars()) + elif col in _INT32_COLUMNS or col in ("unitid", "year"): + df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False).cast(pl.Int32, strict=False)) + else: + df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False).round(0).cast(pl.Int64, strict=False)) + return df.select(columns) + + +def _insert(conn: duckdb.DuckDBPyConnection, table: str, df: pl.DataFrame) -> int: + if df.height == 0: + return 0 + conn.register("_tmp_eada", df.to_arrow()) + conn.execute(f"INSERT INTO {table} SELECT * FROM _tmp_eada") + conn.unregister("_tmp_eada") + return df.height + + +def load_eada( + conn: duckdb.DuckDBPyConnection, + year_range: range, + unitid_filter: int | None = UD_UNITID, +) -> dict[str, int]: + """Load EADA institution and per-sport data for each year in the range.""" + totals = {"institution": 0, "sport": 0} + for year in year_range: + year_dir = config.EADA_DATA_DIR / str(year) + conn.execute("DELETE FROM raw_eada_institution WHERE year = ?", [year]) + conn.execute("DELETE FROM raw_eada_sport WHERE year = ?", [year]) + + inst_path = _find_file(year_dir, "inst") + if inst_path is None: + print(f" No EADA institution file for {year}, skipping") + continue + inst = _read(inst_path, INSTITUTION_VARIANTS, unitid_filter) + if inst is not None: + totals["institution"] += _insert( + conn, "raw_eada_institution", _normalize(inst, year, INSTITUTION_COLUMNS) + ) + + sport_path = _find_file(year_dir, "school") + if sport_path is not None: + sports = _read(sport_path, SPORT_VARIANTS, unitid_filter) + if sports is not None: + sports = _normalize(sports, year, SPORT_COLUMNS).filter(pl.col("sport").is_not_null()) + totals["sport"] += _insert(conn, "raw_eada_sport", sports) + + return totals diff --git a/tests/test_eada.py b/tests/test_eada.py new file mode 100644 index 0000000..0bd6ad4 --- /dev/null +++ b/tests/test_eada.py @@ -0,0 +1,172 @@ +"""Tests for EADA athletics download, loader, and dashboard queries.""" + +import io +import zipfile + +import httpx +import pandas as pd +import respx + +import admin_analytics.config as config +from admin_analytics.config import UD_UNITID +from admin_analytics.dashboard.queries import ( + query_athletics_comp, + query_athletics_expenses, + query_athletics_vs_university, +) +from admin_analytics.eada.download import download_all +from admin_analytics.eada.loader import load_eada + +OTHER_UNITID = 999999 + + +def _write_year(year_dir, inst_name="instLevel.xlsx", school_name="Schools.xlsx", lower=False): + """Write minimal EADA institution and per-sport spreadsheets.""" + year_dir.mkdir(parents=True, exist_ok=True) + inst = pd.DataFrame({ + "unitid": [UD_UNITID, OTHER_UNITID], + "classification_name": ["NCAA Division I-FCS", "NCAA Division II"], + "EFTotalCount": [18000, 5000], + "STUDENTAID_TOTAL": [15_000_000, 1_000_000], + "HDCOACH_SALARY_MEN": [240_000.0, 80_000.0], + "UNDUP_CT_PARTIC_MEN": [300, 100], + "UNDUP_CT_PARTIC_WOMEN": [320, 110], + "TOT_EXPENSE_ALL_NOTALLOC": [14_000_000, 2_000_000], + "GRND_TOTAL_REVENUE": [48_000_000, 9_000_000], + "GRND_TOTAL_EXPENSE": [48_000_000, 9_000_000], + }) + if lower: + inst.columns = [c.lower() for c in inst.columns] + inst.to_excel(year_dir / inst_name, index=False) + pd.DataFrame({ + "unitid": [UD_UNITID, UD_UNITID, UD_UNITID, OTHER_UNITID], + "SUM_PARTIC_MEN": [110, 15, 30, 50], + "SUM_PARTIC_WOMEN": [None, 16, 28, 40], + "TOTAL_REVENUE_ALL": [9_000_000, 6_000_000, 2_000_000, 1_000_000], + "TOTAL_EXPENSE_ALL": [9_000_000, 6_000_000, 2_000_000, 1_000_000], + "Sports": ["Football", "Basketball", "Soccer", "Football"], + }).to_excel(year_dir / school_name, index=False) + + +class TestLoader: + def test_loads_ud_only(self, db_conn, tmp_path, monkeypatch): + monkeypatch.setattr(config, "EADA_DATA_DIR", tmp_path) + _write_year(tmp_path / "2024") + totals = load_eada(db_conn, range(2024, 2025)) + assert totals == {"institution": 1, "sport": 3} + row = db_conn.execute( + "SELECT total_expenses, unallocated_expenses, participants_men, " + "ft_undergrad_enrollment, classification FROM raw_eada_institution" + ).fetchone() + assert row == (48_000_000, 14_000_000, 300, 18000, "NCAA Division I-FCS") + + def test_resolves_column_and_filename_variants(self, db_conn, tmp_path, monkeypatch): + monkeypatch.setattr(config, "EADA_DATA_DIR", tmp_path) + _write_year(tmp_path / "2010", inst_name="instLevelData.xlsx", + school_name="schools.xlsx", lower=True) + totals = load_eada(db_conn, range(2010, 2011)) + assert totals["institution"] == 1 + assert db_conn.execute( + "SELECT total_expenses FROM raw_eada_institution WHERE year = 2010" + ).fetchone()[0] == 48_000_000 + + def test_reload_replaces_year(self, db_conn, tmp_path, monkeypatch): + monkeypatch.setattr(config, "EADA_DATA_DIR", tmp_path) + _write_year(tmp_path / "2024") + load_eada(db_conn, range(2024, 2025)) + load_eada(db_conn, range(2024, 2025)) + assert db_conn.execute("SELECT COUNT(*) FROM raw_eada_sport").fetchone()[0] == 3 + + def test_missing_year_skipped(self, db_conn, tmp_path, monkeypatch): + monkeypatch.setattr(config, "EADA_DATA_DIR", tmp_path) + assert load_eada(db_conn, range(2024, 2025)) == {"institution": 0, "sport": 0} + + +class TestDownload: + @respx.mock + def test_downloads_spreadsheets_only(self, tmp_path, monkeypatch): + monkeypatch.setattr(config, "EADA_DATA_DIR", tmp_path) + respx.get(f"{config.EADA_API_URL}/fileList").mock(return_value=httpx.Response(200, json=[ + {"Year": 2024, "FileName": "EADA_2023-2024.zip"}, + {"Year": 2024, "FileName": "EADA_All_Data_Combined_2023-2024_SAS_SPSS_EXCEL.zip"}, + ])) + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("instLevel.xlsx", b"x") + zf.writestr("schools.sav", b"x") + route = respx.get(f"{config.EADA_API_URL}/file").mock( + return_value=httpx.Response(200, content=buf.getvalue()) + ) + download_all(range(2024, 2025)) + assert route.calls.last.request.url.params["fileName"] == "EADA_2023-2024.zip" + assert sorted(p.name for p in (tmp_path / "2024").iterdir()) == ["instLevel.xlsx"] + + +def _seed(conn): + for year, total, unalloc in [(2023, 40_000_000, 10_000_000), (2024, 50_000_000, 20_000_000)]: + conn.execute( + "INSERT INTO raw_eada_institution (unitid, year, total_expenses, unallocated_expenses, " + "participants_men, participants_women, ft_undergrad_enrollment) VALUES (?, ?, ?, ?, 300, 300, 20000)", + [UD_UNITID, year, total, unalloc], + ) + for sport, amount in [("Football", 10_000_000), ("Basketball", 5_000_000), ("Soccer", 3_000_000)]: + conn.execute( + "INSERT INTO raw_eada_sport (unitid, year, sport, total_expenses) VALUES (?, ?, ?, ?)", + [UD_UNITID, year, sport, amount], + ) + conn.execute( + "INSERT INTO raw_ipeds_finance (unitid, year, total_expenses, instruction_expenses, " + "institutional_support_expenses) VALUES (?, 2023, 1000000000, 400000000, 100000000)", + [UD_UNITID], + ) + for year, value in [(2023, 300.0), (2024, 310.0)]: + conn.execute( + "INSERT INTO raw_cpi_u (year, month, value, series_id) VALUES (?, 1, ?, 'CUUR0000SA0')", + [year, value], + ) + + +class TestQueries: + def test_expenses_split_and_per_student(self, db_conn): + _seed(db_conn) + df = query_athletics_expenses(db_conn) + assert df["year"].to_list() == [2023, 2024] + latest = df.row(1, named=True) + assert latest["football"] == 10_000_000 + assert latest["other_sports"] == 3_000_000 + assert latest["per_student"] == 2500 + # Latest year is the CPI base, so adjusted equals nominal + assert latest["total_expenses_cpi"] == 50_000_000 + + def test_vs_university_only_overlapping_years(self, db_conn): + _seed(db_conn) + df = query_athletics_vs_university(db_conn) + assert df["year"].to_list() == [2023] + assert df["athletics_pct"][0] == 4.0 + assert df["athletics_index"][0] == 100.0 + assert query_athletics_vs_university(db_conn, since=2024).height == 0 + + def test_comp_matches_abbreviated_titles(self, db_conn): + for year, name, title, comp in [ + (2022, "A", "DIR, INTERCOLLEGIATE ATHLC & REC SR", 700_000), + (2023, "A", "DIRECTOR, INTERCOLLEGIATE ATHELTICS & RECREATION", 710_000), + (2023, "B", "HEAD FOOTBALL COACH", 680_000), + (2023, "C", "HEAD FOOTBALL COACH TO 1/1/23", 500_000), + (2023, "D", "FORMER ATHLETIC DIRECTOR", 400_000), + (2023, "E", "PROVOST", 800_000), + ]: + db_conn.execute( + "INSERT INTO raw_990_schedule_j (tax_year, person_name, title, total_compensation) " + "VALUES (?, ?, ?, ?)", + [year, name, title, comp], + ) + df = query_athletics_comp(db_conn) + assert set(zip(df["tax_year"], df["role"], df["person_name"])) == { + (2022, "Athletic Director", "A"), + (2023, "Athletic Director", "A"), + (2023, "Head Football Coach", "B"), + } + + def test_empty(self, db_conn): + assert query_athletics_expenses(db_conn).height == 0 + assert query_athletics_comp(db_conn).height == 0