Skip to content

Commit 569816e

Browse files
SpelkingtonSpelkington
committed
feat: better analyzer for missing pipeline requirements
1 parent a3de8af commit 569816e

16 files changed

Lines changed: 485 additions & 439 deletions

File tree

‎examples/advanced/KedroSpaceflights.Custom/Program.cs‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -66,36 +66,33 @@ private static void ConfigureServices(IServiceCollection services, string basePa
6666
});
6767

6868
flowthru
69-
.RegisterPipeline<Catalog>(label: "DataProcessing", pipeline: DataProcessingPipeline.Create)
69+
.RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create)
7070
.WithDescription("Preprocesses raw data and creates model input table");
7171

7272
flowthru
73-
.RegisterPipelineWithConfiguration<Catalog, DataSciencePipeline.Params>(
73+
.RegisterPipeline(
7474
label: "DataScience",
7575
pipeline: DataSciencePipeline.Create,
7676
configurationSection: "Flowthru:Pipelines:DataScience"
7777
)
7878
.WithDescription("Trains ML model");
7979

8080
flowthru
81-
.RegisterPipeline<Catalog>(
82-
label: "DataDiagnostics",
83-
pipeline: DataDiagnosticsPipeline.Create
84-
)
81+
.RegisterPipeline(label: "DataDiagnostics", pipeline: DataDiagnosticsPipeline.Create)
8582
.WithDescription(
8683
"Validates pipeline outputs against Kedro reference and exports diagnostic data"
8784
);
8885

8986
flowthru
90-
.RegisterPipelineWithConfiguration<Catalog, DataEvaluationPipeline.Params>(
87+
.RegisterPipeline(
9188
label: "DataEvaluation",
9289
pipeline: DataEvaluationPipeline.Create,
9390
configurationSection: "Flowthru:Pipelines:DataEvaluation"
9491
)
9592
.WithDescription("Evaluates ML model performance and cross-validation");
9693

9794
flowthru
98-
.RegisterPipeline<Catalog>(label: "Reporting", pipeline: ReportingPipeline.Create)
95+
.RegisterPipeline(label: "Reporting", pipeline: ReportingPipeline.Create)
9996
.WithDescription("Generates reports and visualizations");
10097
});
10198

‎examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed/Program.cs‎

Lines changed: 23 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public static Task<int> Main(string[] args) =>
3535
);
3636

3737
/// <summary>
38-
/// Configures services for the application. Used by test infrastructure.
38+
/// Configures services for the application.
3939
/// </summary>
4040
public static IServiceProvider ConfigureServices(string? basePath = null)
4141
{
@@ -56,62 +56,48 @@ private static void ConfigureServices(IServiceCollection services, string basePa
5656
// Each library owns its own catalog. Registered by concrete type so each
5757
// pipeline factory receives the correct strongly-typed instance.
5858

59-
flowthru.UseCatalog<DataProcessingCatalog>(_ => new DataProcessingCatalog(dataPath));
60-
flowthru.UseCatalog<DataScienceCatalog>(_ => new DataScienceCatalog(dataPath));
61-
flowthru.UseCatalog<ReportingCatalog>(_ => new ReportingCatalog(dataPath));
62-
63-
// ─── Metadata Providers ───────────────────────────────────────────────
64-
65-
flowthru.ConfigureMetadata(meta =>
66-
{
67-
var metadataPath = Path.Combine(basePath, "Metadata");
68-
meta.AddProvider<JsonMetadataProvider, JsonMetadataProviderBuilder>(json =>
69-
json.WithOutputDirectory(metadataPath)
70-
)
71-
.AddProvider<MermaidMetadataProvider, MermaidMetadataProviderBuilder>(mermaid =>
72-
mermaid.WithOutputDirectory(metadataPath)
73-
);
74-
});
59+
flowthru.UseCatalog(_ => new DataProcessingCatalog(dataPath));
60+
flowthru.UseCatalog(_ => new DataScienceCatalog(dataPath));
61+
flowthru.UseCatalog(_ => new ReportingCatalog(dataPath));
7562

7663
// ─── Pipeline Registration ─────────────────────────────────────────────
77-
// Single-catalog pipeline: reads from and writes to DataProcessingCatalog only.
64+
// Each pipeline's Create method signature IS the cross-catalog contract.
65+
// The framework resolves catalogs and config automatically from DI.
66+
7867
flowthru
79-
.RegisterPipeline<DataProcessingCatalog>(
80-
label: "DataProcessing",
81-
pipeline: DataProcessingPipeline.Create
82-
)
68+
.RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create)
8369
.WithDescription("Preprocesses companies and shuttles data into a model input table");
8470

85-
// 2-catalog pipeline: reads model input from DataProcessing, writes model artifacts to DataScience.
86-
// The two-catalog signature makes the cross-domain dependency an explicit compile-time contract.
8771
flowthru
88-
.RegisterPipelineWithConfiguration<
89-
DataProcessingCatalog,
90-
DataScienceCatalog,
91-
DataSciencePipeline.Params
92-
>(
72+
.RegisterPipeline(
9373
label: "DataScience",
9474
pipeline: DataSciencePipeline.Create,
9575
configurationSection: "Flowthru:Pipelines:DataScience"
9676
)
9777
.WithDescription("Trains linear regression model for shuttle price prediction");
9878

99-
// 3-catalog pipeline: reads preprocessed shuttles (DataProcessing) and model predictions
100-
// (DataScience), writes all reports and charts to ReportingCatalog.
10179
flowthru
102-
.RegisterPipelineWithConfiguration<
103-
DataProcessingCatalog,
104-
DataScienceCatalog,
105-
ReportingCatalog,
106-
ReportingPipeline.Params
107-
>(
80+
.RegisterPipeline(
10881
label: "Reporting",
10982
pipeline: ReportingPipeline.Create,
11083
configurationSection: "Flowthru:Pipelines:Reporting"
11184
)
11285
.WithDescription(
11386
"Generates passenger capacity reports and confusion matrix visualizations"
11487
);
88+
89+
// ─── Metadata Providers ───────────────────────────────────────────────
90+
91+
flowthru.ConfigureMetadata(meta =>
92+
{
93+
var metadataPath = Path.Combine(basePath, "Metadata");
94+
meta.AddProvider<JsonMetadataProvider, JsonMetadataProviderBuilder>(json =>
95+
json.WithOutputDirectory(metadataPath)
96+
)
97+
.AddProvider<MermaidMetadataProvider, MermaidMetadataProviderBuilder>(mermaid =>
98+
mermaid.WithOutputDirectory(metadataPath)
99+
);
100+
});
115101
});
116102

117103
services.AddLogging(logging =>

‎examples/advanced/SpaceflightsPythonEFCore/Program.cs‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -101,23 +101,15 @@ private static void ConfigureServices(IServiceCollection services, string basePa
101101
tempProvider.GetRequiredService<Flowthru.Extensions.Python.Execution.IPythonExecutor>();
102102

103103
flowthru
104-
.RegisterPipeline<Catalog>(label: "DataProcessing", pipeline: DataProcessingPipeline.Create)
104+
.RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create)
105105
.WithDescription("Preprocesses companies and shuttles (C#), stores in EFCore");
106106

107107
flowthru
108-
.RegisterPipeline<Catalog, Flowthru.Extensions.Python.Execution.IPythonExecutor>(
109-
label: "DataScience",
110-
pipeline: DataSciencePipeline.Create,
111-
parameters: executor
112-
)
108+
.RegisterPipeline(label: "DataScience", pipeline: DataSciencePipeline.Create)
113109
.WithDescription("Trains and evaluates regression model (Python); reads/writes EFCore");
114110

115111
flowthru
116-
.RegisterPipeline<Catalog, Flowthru.Extensions.Python.Execution.IPythonExecutor>(
117-
label: "Reporting",
118-
pipeline: ReportingPipeline.Create,
119-
parameters: executor
120-
)
112+
.RegisterPipeline(label: "Reporting", pipeline: ReportingPipeline.Create)
121113
.WithDescription(
122114
"Generates visualizations (Python); reads PreprocessedShuttles and ModelPredictions from EFCore"
123115
);

‎examples/archived/UmapReferenceComparisons/Program.cs‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,16 +51,13 @@ private static void ConfigureServices(IServiceCollection services, string basePa
5151

5252
// Register comparison pipelines
5353
flowthru
54-
.RegisterPipeline<Catalog>(
55-
label: "IrisComparisonPipeline",
56-
pipeline: IrisComparisonPipeline.Create
57-
)
54+
.RegisterPipeline(label: "IrisComparisonPipeline", pipeline: IrisComparisonPipeline.Create)
5855
.WithDescription(
5956
"Compare C# UMAP against Python reference for Iris dataset (150 samples, 4 features)"
6057
);
6158

6259
flowthru
63-
.RegisterPipeline<Catalog>(
60+
.RegisterPipeline(
6461
label: "DigitsComparisonPipeline",
6562
pipeline: DigitsComparisonPipeline.Create
6663
)
@@ -71,7 +68,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa
7168
// Fashion-MNIST pipeline disabled to keep repo lean (70K samples = ~12MB)
7269
// Uncomment when large dataset support is needed
7370
// flowthru
74-
// .RegisterPipeline<Catalog>(
71+
// .RegisterPipeline(
7572
// label: "FashionComparisonPipeline",
7673
// pipeline: Pipelines.FashionComparison.FashionComparisonPipeline.Create
7774
// )

‎examples/starter/KedroIris/Program.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa
6060

6161
// Register data engineering pipeline with configuration parameters
6262
flowthru
63-
.RegisterPipelineWithConfiguration<Catalog, DataEngineeringPipeline.Params>(
63+
.RegisterPipeline(
6464
label: "DataEngineering",
6565
pipeline: DataEngineeringPipeline.Create,
6666
configurationSection: "Flowthru:Pipelines:DataEngineering"
@@ -69,7 +69,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa
6969

7070
// Register data science pipeline with configuration parameters
7171
flowthru
72-
.RegisterPipelineWithConfiguration<Catalog, DataSciencePipeline.Params>(
72+
.RegisterPipeline(
7373
label: "DataScience",
7474
pipeline: DataSciencePipeline.Create,
7575
configurationSection: "Flowthru:Pipelines:DataScience"

‎examples/starter/KedroIrisPython/Program.cs‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -103,19 +103,11 @@ private static void ConfigureServices(IServiceCollection services, string basePa
103103

104104
// Register pipelines with resolved executor
105105
flowthru
106-
.RegisterPipeline<Catalog, Flowthru.Extensions.Python.Execution.IPythonExecutor>(
107-
label: "DataEngineering",
108-
pipeline: DataEngineeringPipeline.Create,
109-
parameters: executor
110-
)
106+
.RegisterPipeline(label: "DataEngineering", pipeline: DataEngineeringPipeline.Create)
111107
.WithDescription("Splits iris data into training and test sets using Python");
112108

113109
flowthru
114-
.RegisterPipeline<Catalog, Flowthru.Extensions.Python.Execution.IPythonExecutor>(
115-
label: "DataScience",
116-
pipeline: DataSciencePipeline.Create,
117-
parameters: executor
118-
)
110+
.RegisterPipeline(label: "DataScience", pipeline: DataSciencePipeline.Create)
119111
.WithDescription(
120112
"Trains multi-class logistic regression model and evaluates predictions using Python"
121113
);

‎examples/starter/KedroSpaceflights/Program.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@ private static void ConfigureServices(IServiceCollection services, string basePa
6161

6262
// Register data processing pipeline
6363
flowthru
64-
.RegisterPipeline<Catalog>(label: "DataProcessing", pipeline: DataProcessingPipeline.Create)
64+
.RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create)
6565
.WithDescription("Preprocesses companies and shuttles data");
6666

6767
// Register data science pipeline with configuration parameters
6868
flowthru
69-
.RegisterPipelineWithConfiguration<Catalog, DataSciencePipeline.Params>(
69+
.RegisterPipeline(
7070
label: "DataScience",
7171
pipeline: DataSciencePipeline.Create,
7272
configurationSection: "Flowthru:Pipelines:DataScience"
@@ -75,7 +75,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa
7575

7676
// Register reporting pipeline with configuration parameters
7777
flowthru
78-
.RegisterPipelineWithConfiguration<Catalog, ReportingPipeline.Params>(
78+
.RegisterPipeline(
7979
label: "Reporting",
8080
pipeline: ReportingPipeline.Create,
8181
configurationSection: "Flowthru:Pipelines:Reporting"

‎examples/starter/KedroSpaceflightsPython/Program.cs‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -106,29 +106,17 @@ private static void ConfigureServices(IServiceCollection services, string basePa
106106

107107
// Register pipelines with resolved executor
108108
flowthru
109-
.RegisterPipeline<Catalog, Flowthru.Extensions.Python.Execution.IPythonExecutor>(
110-
label: "DataProcessing",
111-
pipeline: DataProcessingPipeline.Create,
112-
parameters: executor
113-
)
109+
.RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create)
114110
.WithDescription("Preprocesses companies, shuttles, and reviews data using Python");
115111

116112
flowthru
117-
.RegisterPipeline<Catalog, Flowthru.Extensions.Python.Execution.IPythonExecutor>(
118-
label: "DataScience",
119-
pipeline: DataSciencePipeline.Create,
120-
parameters: executor
121-
)
113+
.RegisterPipeline(label: "DataScience", pipeline: DataSciencePipeline.Create)
122114
.WithDescription(
123115
"Trains linear regression model for price prediction using Python/scikit-learn"
124116
);
125117

126118
flowthru
127-
.RegisterPipeline<Catalog, Flowthru.Extensions.Python.Execution.IPythonExecutor>(
128-
label: "Reporting",
129-
pipeline: ReportingPipeline.Create,
130-
parameters: executor
131-
)
119+
.RegisterPipeline(label: "Reporting", pipeline: ReportingPipeline.Create)
132120
.WithDescription(
133121
"Generates visualization outputs including passenger capacity plots and confusion matrix"
134122
);

‎examples/starter/Minimal/Program.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa
5757

5858
// Register the greetings pipeline
5959
flowthru
60-
.RegisterPipeline<Catalog>(label: "Greetings", pipeline: GreetingsPipeline.Create)
60+
.RegisterPipeline(label: "Greetings", pipeline: GreetingsPipeline.Create)
6161
.WithDescription(
6262
"A minimal pipeline demonstrating name transformation into multiple greeting formats"
6363
);

‎examples/starter/SpaceflightsEFCore/Program.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@ private static void ConfigureServices(IServiceCollection services, string basePa
6161

6262
// Register data processing pipeline
6363
flowthru
64-
.RegisterPipeline<Catalog>(label: "DataProcessing", pipeline: DataProcessingPipeline.Create)
64+
.RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create)
6565
.WithDescription("Preprocesses companies and shuttles data");
6666

6767
// Register data science pipeline with configuration parameters
6868
flowthru
69-
.RegisterPipelineWithConfiguration<Catalog, DataSciencePipeline.Params>(
69+
.RegisterPipeline(
7070
label: "DataScience",
7171
pipeline: DataSciencePipeline.Create,
7272
configurationSection: "Flowthru:Pipelines:DataScience"
@@ -75,7 +75,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa
7575

7676
// Register reporting pipeline with configuration parameters
7777
flowthru
78-
.RegisterPipelineWithConfiguration<Catalog, ReportingPipeline.Params>(
78+
.RegisterPipeline(
7979
label: "Reporting",
8080
pipeline: ReportingPipeline.Create,
8181
configurationSection: "Flowthru:Pipelines:Reporting"

0 commit comments

Comments
 (0)