Skip to content

Commit c1a66ec

Browse files
SpelkingtonSpelkington
authored
feat: spark extension (#4)
* feat: init spark * fix: spark namespace rename * fix: continued progress on spark port * fix: jar build added * fix: upgraded scala to lts * fix: spark tests from dotnet working * feat: more spark functions wrapped * fix: start spark catalog system * fix: spark tests * fix: resolve issues with DI-version of spark executor * ci: codecov * feat: spark pipeline test * fix: resolved spark pipeline issues * fix: misc noise from JVM * fix: sand down API surface * ci: resolve ci dependency issues * fix: minor preference changes in spark starter * fix: extended spark expression support * fix: agg issue in spark * feat: spark and dataframe analyzers * feat: analyzers for spark and dataframes * docs: add vstest to docs * feat: additional analyzer tests * fix: make dataframe analyzer error funnier * fix: log issue on example test run output * ci: fix with affected targets * ci: fix with affected targets
1 parent 6d8573b commit c1a66ec

332 files changed

Lines changed: 70512 additions & 213 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.config/dotnet-tools.json‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,7 @@
55
"csharpier": {
66
"version": "0.29.2",
77
"commands": [
8-
"dotnet-csharpier",
9-
"reportgenerator"
8+
"dotnet-csharpier"
109
],
1110
"rollForward": false
1211
},
@@ -25,4 +24,4 @@
2524
"rollForward": false
2625
}
2726
}
28-
}
27+
}

‎.github/actions/setup/action.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: Setup Flowthru CI Environment
2+
description: >
3+
Installs all non-Node toolchains (.NET, Node, Python, uv, pnpm, Java, Spark)
4+
and runs pnpm install (which also runs dotnet restore via the postinstall hook).
5+
6+
runs:
7+
using: composite
8+
steps:
9+
- name: Setup .NET
10+
uses: actions/setup-dotnet@v4
11+
with:
12+
dotnet-version: "10.0.x"
13+
dotnet-quality: "preview"
14+
env:
15+
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "true"
16+
DOTNET_NOLOGO: "true"
17+
DOTNET_CLI_TELEMETRY_OPTOUT: "true"
18+
19+
- name: Setup Node.js
20+
uses: actions/setup-node@v4
21+
with:
22+
node-version: "22"
23+
24+
- name: Setup Python
25+
uses: actions/setup-python@v5
26+
with:
27+
python-version: "3.10"
28+
29+
- name: Setup UV
30+
uses: astral-sh/setup-uv@v5
31+
32+
- name: Setup pnpm
33+
uses: pnpm/action-setup@v4
34+
with:
35+
version: 10.6.3
36+
37+
- name: Setup Java
38+
uses: actions/setup-java@v4
39+
with:
40+
distribution: temurin
41+
java-version: "21"
42+
43+
- name: Setup Apache Spark 4.1.1
44+
shell: bash
45+
run: |
46+
SPARK_VERSION=4.1.1
47+
HADOOP_VERSION=3
48+
INSTALL_DIR=/opt/spark
49+
50+
curl -fsSL \
51+
"https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" \
52+
| tar -xz -C /tmp
53+
54+
sudo mv "/tmp/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" "${INSTALL_DIR}"
55+
echo "SPARK_HOME=${INSTALL_DIR}" >> $GITHUB_ENV
56+
echo "${INSTALL_DIR}/bin" >> $GITHUB_PATH
57+
58+
- name: Cache NuGet packages
59+
uses: actions/cache@v4
60+
with:
61+
path: ~/.nuget/packages
62+
key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json', '**/*.csproj') }}
63+
restore-keys: |
64+
${{ runner.os }}-nuget-
65+
66+
- name: Install Node dependencies
67+
shell: bash
68+
run: pnpm install

‎.github/hooks/scripts/stop-build.js‎

Lines changed: 83 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,72 @@
1313
const { spawnSync } = require('child_process');
1414
const path = require('path');
1515

16+
/**
17+
* Parse dotnet test output to extract test counts.
18+
* Aggregates counts across all test runs.
19+
* Returns: { passed, failed, skipped, inconclusive, total }
20+
*/
21+
function parseTestCounts(output) {
22+
// Match patterns like: "Failed: 6, Passed: 156, Skipped: 0, Total: 162"
23+
const regex = /Failed:\s*(\d+),\s*Passed:\s*(\d+),\s*Skipped:\s*(\d+),\s*Total:\s*(\d+)/g;
24+
const counts = { passed: 0, failed: 0, skipped: 0, total: 0 };
25+
26+
let match;
27+
while ((match = regex.exec(output)) !== null) {
28+
counts.failed += parseInt(match[1], 10);
29+
counts.passed += parseInt(match[2], 10);
30+
counts.skipped += parseInt(match[3], 10);
31+
counts.total += parseInt(match[4], 10);
32+
}
33+
34+
// Inconclusive = Total - Passed - Failed - Skipped
35+
counts.inconclusive = counts.total - counts.passed - counts.failed - counts.skipped;
36+
37+
return counts;
38+
}
39+
40+
/**
41+
* Format test counts for display.
42+
*/
43+
function formatTestCounts(counts) {
44+
return [
45+
`Passed: ${counts.passed}`,
46+
`Failed: ${counts.failed}`,
47+
`Skipped: ${counts.skipped}`,
48+
`Inconclusive: ${counts.inconclusive}`,
49+
`Total: ${counts.total}`,
50+
].join('\n');
51+
}
52+
53+
/**
54+
* Extract full failure blocks from dotnet test output.
55+
* A block starts with an indented "Failed <TestName>" line and continues
56+
* until the next dotnet test summary line or end of output.
57+
*/
58+
function extractFailureBlocks(output) {
59+
const lines = output.split('\n');
60+
const resultLines = [];
61+
let capturing = false;
62+
63+
for (const line of lines) {
64+
if (/^\s+Failed\s+\S/.test(line)) {
65+
// Start of a new failure block — blank separator between blocks.
66+
if (resultLines.length > 0) resultLines.push('');
67+
capturing = true;
68+
resultLines.push(line);
69+
} else if (capturing) {
70+
// Stop at the dotnet test run summary line (e.g. "Failed! - Failed: 6, Passed: ...")
71+
if (/^(Failed!|Passed!)\s+-\s+Failed:/.test(line.trim())) {
72+
capturing = false;
73+
} else {
74+
resultLines.push(line);
75+
}
76+
}
77+
}
78+
79+
return resultLines.join('\n').trim();
80+
}
81+
1682
// Read and parse stdin to detect re-entry.
1783
let hookInput = {};
1884
try {
@@ -66,14 +132,14 @@ const stdout = (result.stdout || '').trim();
66132
const stderr = (result.stderr || '').trim();
67133
const combined = [stdout, stderr].filter(Boolean).join('\n');
68134

69-
if (result.status !== 0) {
70-
// Extract failed test lines for a focused summary.
71-
const failureLines = combined
72-
.split('\n')
73-
.filter(line => /failed|error|FAILED|ERROR/i.test(line))
74-
.slice(0, 40); // cap at 40 lines to avoid overwhelming the agent
135+
// Parse test counts from output.
136+
const testCounts = parseTestCounts(combined);
137+
const countsDisplay = formatTestCounts(testCounts);
75138

76-
const summary = failureLines.length > 0 ? failureLines.join('\n') : combined;
139+
// NX does not always propagate the dotnet exit code — fall back to parsed counts.
140+
if (result.status !== 0 || testCounts.failed > 0) {
141+
const failureBlocks = extractFailureBlocks(combined);
142+
const summary = failureBlocks.length > 0 ? failureBlocks : combined;
77143

78144
process.stdout.write(JSON.stringify({
79145
hookSpecificOutput: {
@@ -82,12 +148,21 @@ if (result.status !== 0) {
82148
reason: [
83149
`nx affected test FAILED (affected: ${affectedProjects.join(', ')}) — address these failures before concluding.`,
84150
'',
151+
'Test Summary:',
152+
countsDisplay,
153+
'',
85154
summary,
86155
].join('\n'),
87156
},
88157
}));
158+
process.exit(1);
89159
} else {
90160
process.stdout.write(JSON.stringify({
91-
systemMessage: `nx affected test: passed (${affectedProjects.length} project(s): ${affectedProjects.join(', ')}).`,
161+
systemMessage: [
162+
`nx affected test: passed (${affectedProjects.length} project(s): ${affectedProjects.join(', ')}).`,
163+
'',
164+
'Test Summary:',
165+
countsDisplay,
166+
].join('\n'),
92167
}));
93168
}

‎.github/instructions/flowthru.instructions.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
---
2+
description: Guidelines for routing Flowthru sessions based on development focus.
23
applyTo: "**"
34
---
45

‎.github/workflows/pr-tests.yml‎

Lines changed: 2 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -26,44 +26,8 @@ jobs:
2626
with:
2727
fetch-depth: 0
2828

29-
- name: Setup .NET
30-
uses: actions/setup-dotnet@v4
31-
with:
32-
dotnet-version: "10.0.x"
33-
dotnet-quality: "preview"
34-
env:
35-
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "true"
36-
DOTNET_NOLOGO: "true"
37-
DOTNET_CLI_TELEMETRY_OPTOUT: "true"
38-
39-
- name: Setup Node.js
40-
uses: actions/setup-node@v4
41-
with:
42-
node-version: "22"
43-
44-
- name: Setup Python
45-
uses: actions/setup-python@v5
46-
with:
47-
python-version: "3.10"
48-
49-
- name: Setup UV
50-
uses: astral-sh/setup-uv@v5
51-
52-
- name: Setup pnpm
53-
uses: pnpm/action-setup@v4
54-
with:
55-
version: 10.6.3
56-
57-
- name: Cache NuGet packages
58-
uses: actions/cache@v4
59-
with:
60-
path: ~/.nuget/packages
61-
key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json', '**/*.csproj') }}
62-
restore-keys: |
63-
${{ runner.os }}-nuget-
64-
65-
- name: Install Node dependencies
66-
run: pnpm install
29+
- name: Setup CI environment
30+
uses: ./.github/actions/setup
6731

6832
- name: Run affected tests
6933
env:

0 commit comments

Comments
 (0)