diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index b908b557..02960c4e 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -16,6 +16,13 @@ "dotnet-script" ], "rollForward": false + }, + "docfx": { + "version": "2.78.5", + "commands": [ + "docfx" + ], + "rollForward": false } } } \ No newline at end of file diff --git a/.vscode/flowthru.code-snippets b/.vscode/flowthru.code-snippets index 1354eadf..59afcb87 100644 --- a/.vscode/flowthru.code-snippets +++ b/.vscode/flowthru.code-snippets @@ -10,16 +10,16 @@ "scope": "csharp", "prefix": "_pipeline", "body": [ - "using Flowthru.Pipelines;", + "using Flowthru.Flows;", "using $1", "", "public static class ${RELATIVE_FILEPATH/.*?(\\w+).cs$/$1/}", "{", " public static Pipeline Create(Catalog catalog)", " {", - " return PipelineBuilder.CreatePipeline(pipeline =>", + " return FlowBuilder.CreateFlow(pipeline =>", " {", - " pipeline.AddNode(", + " pipeline.AddStep(", " label: \"$2\",", " description: \"\"\"", " $3", @@ -34,7 +34,7 @@ ], "description": "Creates a basic Flowthru pipeline structure." }, - "FlowthruNode": { + "FlowthruStep": { "scope": "csharp", "prefix": "_node", "body": [ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4f84f98b..d4293e5b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,8 +74,8 @@ Flowthru handles these through an **effect type** called `FlowIO`. If you're The key runtime guarantees: - **All I/O is lazy and explicit.** Side effects cannot be accidentally dropped or silently ignored. -- **Errors are captured, never swallowed.** Node failures propagate to structured pipeline results. Silent `catch {}` blocks are a bug. -- **Nodes are isolated.** A failing node halts execution and reports which node failed and why — partial silent failures are not possible. +- **Errors are captured, never swallowed.** Step failures propagate to structured pipeline results. Silent `catch {}` blocks are a bug. +- **Steps are isolated.** A failing node halts execution and reports which node failed and why — partial silent failures are not possible. ## Decision Rules for Contributors diff --git a/Directory.Build.props b/Directory.Build.props index ae821490..b57dec72 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -16,6 +16,11 @@ false + + + true + + 0.1.33 diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 9d82936b..5c144be5 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -27,7 +27,7 @@ Consider these examples: - **"How do I start writing my first pipeline?"** → This is someone learning. They need a tutorial. - **"How can I read data from a database instead of local files?"** → This is someone working with a specific need. They need a guide. - **"Why does Flowthru use so many types?"** → This is someone studying the framework's design. They need an explanation. -- **"What parameters does `CatalogEntry` accept?"** → This is someone looking up technical details. They need reference documentation. +- **"What parameters does `Item` accept?"** → This is someone looking up technical details. They need reference documentation. The same topic can yield different documentation depending on the question: @@ -156,7 +156,7 @@ Let's say you want to document "catalog entries." Start by listing questions: - "How do I create my first catalog entry?" → Tutorial - "How do I configure a catalog entry for Parquet files?" → Guide - "Why are catalog entries properties rather than string keys?" → Explanation -- "What methods are available on `ICatalogEntry`?" → Reference +- "What methods are available on `IItem`?" → Reference Each question becomes a separate piece of documentation in its appropriate category. Together, they serve users at every stage of their journey with catalog entries. diff --git a/docs/explanation/advanced/storage-composition.md b/docs/explanation/advanced/storage-composition.md index 0979521a..be92b825 100644 --- a/docs/explanation/advanced/storage-composition.md +++ b/docs/explanation/advanced/storage-composition.md @@ -140,11 +140,11 @@ public class EFCoreStorageAdapter : IStorageAdapter> ## Catalog-Level Constraint Narrowing -After adapter creation, pipeline authors can further constrain catalog entries using `CatalogEntry.Constrain()`: +After adapter creation, pipeline authors can further constrain catalog entries using `Item.Constrain()`: ```csharp -public ICatalogEntry> Companies => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Csv("companies", "data/companies.csv") +public IItem> Companies => GetOrCreateEntry(() => + ItemFactory.Enumerable.Csv("companies", "data/companies.csv") .Constrain(traits => traits with { CanWrite = false }) ); ``` @@ -201,7 +201,7 @@ The trait system solves all three: - **Composable**: Eight independent boolean flags - **Queryable**: `adapter.Traits.CanWrite` checked before operation -- **Fail-fast**: `CatalogEntry.Constrain()` validates at initialization +- **Fail-fast**: `Item.Constrain()` validates at initialization The old interfaces remain as `[Obsolete]` for backward compatibility but have no effect on pipeline behavior. @@ -211,9 +211,9 @@ The storage architecture is implemented in: - `src/core/Flowthru/Data/Capabilities/StorageTraits.cs` — the trait record - `src/core/Flowthru/Data/Storage/ComposedStorageAdapter.cs` — trait merging logic -- `src/core/Flowthru/Data/CatalogEntry.cs` — `Constrain()` with ratchet validation +- `src/core/Flowthru/Data/Item.cs` — `Constrain()` with ratchet validation - `src/core/Flowthru/Data/Storage/Medium/` — medium implementations with traits - `src/core/Flowthru/Data/Storage/Format/` — format serializers with traits -Catalog entry factories (`CatalogEntries.Enumerable.Csv(...)`) construct composed adapters with sensible defaults. Extension authors creating custom adapters should declare traits that accurately reflect their capabilities. +Catalog entry factories (`ItemFactory.Enumerable.Csv(...)`) construct composed adapters with sensible defaults. Extension authors creating custom adapters should declare traits that accurately reflect their capabilities. diff --git a/docs/explanation/anatomy-of-a-pipeline.md b/docs/explanation/anatomy-of-a-pipeline.md index ce07a87b..2f197142 100644 --- a/docs/explanation/anatomy-of-a-pipeline.md +++ b/docs/explanation/anatomy-of-a-pipeline.md @@ -75,9 +75,9 @@ In this example: // Data/_01_Raw/Catalog.Raw.cs public partial class Catalog { - public ICatalogEntry> IrisRaw => + public IItem> IrisRaw => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "IrisRaw", filePath: $"{_basePath}/_01_Raw/Datasets/iris.csv" ) @@ -95,9 +95,9 @@ In this example: // Data/_04_Feature/Catalog.Feature.cs public partial class Catalog { - public ICatalogEntry> IrisFeatures => + public IItem> IrisFeatures => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "IrisFeatures", filePath: $"{_basePath}/_04_Feature/Datasets/iris_features.parquet" ) @@ -169,13 +169,13 @@ Pipelines are made up of **nodes** — transformation functions that are built t 1. When writing a node, you don't need to worry about how it's connecting to other nodes — you just need to make sure it inputs the schema, and outputs the schema -### Nodes +### Steps -Nodes are simply functions. Easy! The **only** purpose of a node is to take in data that has one schema, and convert it to data that has another schema. +Steps are simply functions. Easy! The **only** purpose of a node is to take in data that has one schema, and convert it to data that has another schema. ```csharp -// Pipelines/DataEngineering/Nodes/SplitAndEncodeNode.cs -public static class SplitAndEncodeNode +// Pipelines/DataEngineering/Steps/SplitAndEncodeStep.cs +public static class SplitAndEncodeStep { public static Func< IEnumerable, // Input Schema @@ -194,8 +194,8 @@ public static class SplitAndEncodeNode Key points: -- Nodes are a *contract*: that data for the node will **always** come in as the input schemas, and **always** come out as the output schemas. -- Nodes can have any number of inputs, and any number of outputs — as long as they're defined in the input and output schemas, you're not limited to just one-in, one-out. +- Steps are a *contract*: that data for the node will **always** come in as the input schemas, and **always** come out as the output schemas. +- Steps can have any number of inputs, and any number of outputs — as long as they're defined in the input and output schemas, you're not limited to just one-in, one-out. ### Pipelines @@ -214,11 +214,11 @@ public static class DataEngineeringPipeline { public static Pipeline Create(Catalog catalog, Params parameters) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "SplitAndEncode", // Unique label for this node in the pipeline - transform: SplitAndEncodeNode.Create(), + transform: SplitAndEncodeStep.Create(), input: catalog.IrisRaw, output: catalog.IrisFeatures ); @@ -229,7 +229,7 @@ public static class DataEngineeringPipeline Key points: -- **Nodes are never directly connected to each other**: Nodes always take in Catalog entries, and output Catalog entries — *never* directly to each other. +- **Steps are never directly connected to each other**: Steps always take in Catalog entries, and output Catalog entries — *never* directly to each other. - **Order doesn't matter.** A node in the pipeline is **only** ever concerned about its input data and output data. Flowthru handles the order when you run the pipeline: as long as the data is available, or generated by another node, your pipeline will run. ## Entry Point @@ -241,7 +241,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa { services.AddFlowthru(flowthru => { - flowthru.UseCatalog(_ => new Catalog(basePath: "Data"))); + flowthru.RegisterCatalog(_ => new Catalog(basePath: "Data"))); flowthru.RegisterPipeline( label: "DataEngineering", @@ -256,4 +256,4 @@ private static void ConfigureServices(IServiceCollection services, string basePa } ``` -And that's it! This finishes the pipeline anatomy. At this point, you have Catalog entries, connected with Nodes in your Pipelines — everything you need to find new and creative ways to organize, analyze, and report on your data! +And that's it! This finishes the pipeline anatomy. At this point, you have Catalog entries, connected with Steps in your Pipelines — everything you need to find new and creative ways to organize, analyze, and report on your data! diff --git a/docs/guides/advanced/container-deployment.md b/docs/guides/advanced/container-deployment.md index 730615e5..0dcf9c75 100644 --- a/docs/guides/advanced/container-deployment.md +++ b/docs/guides/advanced/container-deployment.md @@ -13,7 +13,7 @@ Deploy a Flowthru pipeline as a standalone container image for execution in envi A containerized pipeline replaces `FlowthruCli` with a minimal `Program.cs` that owns its own DI container. The key difference: no CLI argument parsing, no filesystem-based configuration. ```csharp -using Flowthru.Pipelines; +using Flowthru.Flows; using Flowthru.Services; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -26,7 +26,7 @@ services.AddLogging(logging => services.AddFlowthru(flowthru => { flowthru - .UseCatalog() + .RegisterCatalog() .RegisterPipeline("Ingest", catalog => IngestPipeline.Create(catalog)) .RegisterPipeline("Transform", catalog => TransformPipeline.Create(catalog)); @@ -50,19 +50,19 @@ return result.Success ? 0 : 1; // --- helpers --- -static PipelineSliceStrategy BuildSliceStrategy() +static FlowSliceStrategy BuildSliceStrategy() { static HashSet? ParseCsv(string? value) => string.IsNullOrWhiteSpace(value) ? null : new(value.Split(',', StringSplitOptions.RemoveEmptyEntries)); - return new PipelineSliceStrategy + return new FlowSliceStrategy { Pipelines = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_PIPELINES")), - FromNodes = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_FROM_NODES")), - ToNodes = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_TO_NODES")), + FromSteps = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_FROM_NODES")), + ToSteps = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_TO_NODES")), FromData = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_FROM_DATA")), ToData = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_TO_DATA")), - OnlyNodes = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_ONLY_NODES")), + OnlySteps = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_ONLY_NODES")), }; } ``` @@ -87,7 +87,7 @@ services.AddFlowthru(flowthru => { // No UseConfiguration() call — the IConfiguration above is already registered flowthru - .UseCatalog() + .RegisterCatalog() .RegisterPipeline("Ingest", catalog => IngestPipeline.Create(catalog)); }); ``` @@ -152,7 +152,7 @@ docker run \ ## Inspecting Results -`ExecutePipelineAsync` returns a `PipelineResult` — a structured value object with `Success`, `ExecutionTime`, per-node `NodeResults` (including individual timing, I/O counts, and exceptions), and an optional top-level `Exception`. Serialize it to structured output for your runtime's observability: +`ExecutePipelineAsync` returns a `FlowResult` — a structured value object with `Success`, `ExecutionTime`, per-node `StepResults` (including individual timing, I/O counts, and exceptions), and an optional top-level `Exception`. Serialize it to structured output for your runtime's observability: ```csharp var result = await flowthru.ExecutePipelineAsync(options, cancellationToken: cts.Token); @@ -163,9 +163,9 @@ if (!result.Success) result.ExecutionTime.TotalSeconds, result.Exception?.Message); - foreach (var (name, node) in result.NodeResults.Where(n => !n.Value.Success)) + foreach (var (name, node) in result.StepResults.Where(n => !n.Value.Success)) { - logger.LogError(" Node {Node} failed: {Error}", name, node.Exception?.Message); + logger.LogError(" Step {Step} failed: {Error}", name, node.Exception?.Message); } } ``` diff --git a/docs/guides/advanced/metadata-providers.md b/docs/guides/advanced/metadata-providers.md index 817a0cd7..67896dbe 100644 --- a/docs/guides/advanced/metadata-providers.md +++ b/docs/guides/advanced/metadata-providers.md @@ -40,8 +40,8 @@ public class DashboardMetadataProvider : IMetadataProvider { var payload = new { - PipelineName = dag.PipelineName, - NodeCount = dag.Nodes.Count, + FlowName = dag.FlowName, + StepCount = dag.Steps.Count, EdgeCount = dag.Edges.Count, Timestamp = DateTime.UtcNow }; @@ -85,14 +85,14 @@ using Flowthru.Meta.Providers; services.AddFlowthru(flowthru => { - flowthru.UseCatalog(_ => new MyCatalog()); - flowthru.UsePipelines(_ => myPipelines); + flowthru.RegisterCatalog(_ => new MyCatalog()); + flowthru.RegisterPipelines(_ => myPipelines); flowthru.ConfigureMetadata(meta => { meta.AddProvider(json => json .WithOutputDirectory("metadata") - .WithFilenameTemplate("dag-{PipelineName}-{Timestamp}") + .WithFilenameTemplate("dag-{FlowName}-{Timestamp}") .WithTimestamp("yyyyMMdd-HHmmss") .UseCompactFormat()); @@ -143,9 +143,9 @@ var singlePipelineDag = service.GetDagMetadata(pipelineName: "DataEngineering"); // Get DAG with slicing applied var slicedDag = service.GetDagMetadata( - sliceStrategy: new PipelineSliceStrategy + sliceStrategy: new FlowSliceStrategy { - ToNodes = new HashSet { "TransformNode" } + ToSteps = new HashSet { "TransformStep" } } ); ``` @@ -159,7 +159,7 @@ Useful for tooling, tests, or debugging pipeline structure before execution. ```csharp meta.AddProvider(json => json .WithOutputDirectory("metadata") - .WithFilenameTemplate("dag-{PipelineName}-{Timestamp}") + .WithFilenameTemplate("dag-{FlowName}-{Timestamp}") .WithTimestamp("yyyyMMdd-HHmmss") .UseCompactFormat()); ``` @@ -170,6 +170,6 @@ meta.AddProvider(json => json meta.AddProvider(mermaid => mermaid .WithOutputDirectory("metadata") .WithDirection(MermaidFlowchartDirection.LeftToRight) - .WithActiveNodeColor("#90EE90") + .WithActiveStepColor("#90EE90") .WithActiveDataColor("#ADD8E6")); ``` diff --git a/docs/guides/advanced/service-integration.md b/docs/guides/advanced/service-integration.md index 9162654d..757e5826 100644 --- a/docs/guides/advanced/service-integration.md +++ b/docs/guides/advanced/service-integration.md @@ -18,7 +18,7 @@ var builder = WebApplication.CreateBuilder(args); builder.Services.AddFlowthru(flowthru => { flowthru - .UseCatalog() + .RegisterCatalog() .RegisterPipeline("Ingest", c => IngestPipeline.Create(c)) .RegisterPipeline("Transform", c => TransformPipeline.Create(c)) .UseStorageStrategy(); @@ -42,7 +42,7 @@ var myParams = builder.Configuration builder.Services.AddFlowthru(flowthru => { flowthru - .UseCatalog() + .RegisterCatalog() .RegisterPipeline( "Transform", (catalog, p) => TransformPipeline.Create(catalog, p), @@ -57,7 +57,7 @@ builder.Services.AddFlowthru(flowthru => { flowthru .UseConfiguration(opts => opts.BasePath = "/app/pipeline-config") - .UseCatalog() + .RegisterCatalog() .RegisterPipelineWithConfiguration( "Transform", (catalog, p) => TransformPipeline.Create(catalog, p), @@ -79,7 +79,7 @@ app.MapPost("/pipelines/run", async ( { var options = new ExecutionOptions { - SliceStrategy = new PipelineSliceStrategy + SliceStrategy = new FlowSliceStrategy { Pipelines = request.Pipelines is { Count: > 0 } ? request.Pipelines.ToHashSet() @@ -115,25 +115,25 @@ public record PipelineRunRequest public List? Pipelines { get; init; } public List? ToData { get; init; } public List? FromData { get; init; } - public List? OnlyNodes { get; init; } + public List? OnlySteps { get; init; } } ``` -`PipelineSliceStrategy` properties are `IReadOnlySet?` — convert from lists at the boundary. +`FlowSliceStrategy` properties are `IReadOnlySet?` — convert from lists at the boundary. ## Handling Results -`PipelineResult` carries structured execution data. Map it to your application's response model rather than returning it directly: +`FlowResult` carries structured execution data. Map it to your application's response model rather than returning it directly: ```csharp -static PipelineRunResponse MapToResponse(PipelineResult result) +static PipelineRunResponse MapToResponse(FlowResult result) { return new PipelineRunResponse { Success = result.Success, DurationMs = result.ExecutionTime.TotalMilliseconds, - NodesExecuted = result.NodeResults.Count, - NodeSummaries = result.NodeResults.Select(kvp => new NodeSummary + StepsExecuted = result.StepResults.Count, + StepSummaries = result.StepResults.Select(kvp => new StepSummary { Name = kvp.Key, Success = kvp.Value.Success, @@ -168,8 +168,8 @@ public class PipelineBackgroundService : BackgroundService if (result.Success) { _logger.LogInformation( - "Pipeline completed: {NodeCount} nodes in {Duration}s", - result.NodeResults.Count, + "Pipeline completed: {StepCount} nodes in {Duration}s", + result.StepResults.Count, result.ExecutionTime.TotalSeconds); } else diff --git a/docs/guides/constraining-catalog-entries.md b/docs/guides/constraining-catalog-entries.md index 4e08eec3..427749a8 100644 --- a/docs/guides/constraining-catalog-entries.md +++ b/docs/guides/constraining-catalog-entries.md @@ -22,8 +22,8 @@ Call `.Constrain()` on any catalog entry to narrow its capabilities: ```csharp public class DataCatalog : DataCatalogBase { - public ICatalogEntry> Companies => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Csv("companies", "data/companies.csv") + public IItem> Companies => GetOrCreateEntry(() => + ItemFactory.Enumerable.Csv("companies", "data/companies.csv") .Constrain(traits => traits with { CanWrite = false }) ); } @@ -39,20 +39,20 @@ Mark external data sources as read-only to prevent accidental modifications: ```csharp // API endpoint that only supports GET requests -public ICatalogEntry WeatherFeed => GetOrCreateEntry(() => - CatalogEntries.Single.Http("weather", "https://api.weather.com/current") +public IItem WeatherFeed => GetOrCreateEntry(() => + ItemFactory.Single.Http("weather", "https://api.weather.com/current") .Constrain(traits => traits with { CanWrite = false }) ); // Production database view used for reporting -public ICatalogEntry> ProductionSales => GetOrCreateEntry(() => - CatalogEntries.Enumerable.EFCore("sales", dbContext) +public IItem> ProductionSales => GetOrCreateEntry(() => + ItemFactory.Enumerable.EFCore("sales", dbContext) .Constrain(traits => traits with { CanWrite = false }) ); // Archived historical data that must not change -public ICatalogEntry> HistoricalTransactions => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Parquet("archive", "data/2024/transactions.parquet") +public IItem> HistoricalTransactions => GetOrCreateEntry(() => + ItemFactory.Enumerable.Parquet("archive", "data/2024/transactions.parquet") .Constrain(traits => traits with { CanWrite = false }) ); ``` @@ -62,8 +62,8 @@ public ICatalogEntry> HistoricalTransactions => GetOrCr Mark catalog entries that require network connectivity to prevent offline execution attempts: ```csharp -public ICatalogEntry> RemoteUsers => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Http("users", "https://internal.api/users") +public IItem> RemoteUsers => GetOrCreateEntry(() => + ItemFactory.Enumerable.Http("users", "https://internal.api/users") .Constrain(traits => traits with { RequiresNetwork = true, @@ -77,8 +77,8 @@ public ICatalogEntry> RemoteUsers => GetOrCreateEntry(() => Mark in-memory or cache-based entries as non-persistent: ```csharp -public ICatalogEntry Cache => GetOrCreateEntry(() => - CatalogEntries.Single.Memory("cache") +public IItem Cache => GetOrCreateEntry(() => + ItemFactory.Single.Memory("cache") // Memory adapter already sets IsPersistent = false ); ``` @@ -89,7 +89,7 @@ If you try to use a constrained entry incorrectly, the pipeline fails during con ```csharp // ❌ This fails at pipeline.Build() — before any data is processed -pipeline.AddNode( +pipeline.AddStep( name: "WriteToReadOnly", transform: node, input: catalog.InputData, @@ -125,8 +125,8 @@ Attempts to relax constraints throw `InvalidOperationException` with a detailed Apply multiple constraints in a single call: ```csharp -public ICatalogEntry> AuditLogs => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Csv("audit", "logs/audit.csv") +public IItem> AuditLogs => GetOrCreateEntry(() => + ItemFactory.Enumerable.Csv("audit", "logs/audit.csv") .Constrain(traits => traits with { CanWrite = false, @@ -139,8 +139,8 @@ public ICatalogEntry> AuditLogs => GetOrCreateEntry(() => Or chain multiple constraints for readability: ```csharp -public ICatalogEntry> Customers => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Csv("customers", "data/customers.csv") +public IItem> Customers => GetOrCreateEntry(() => + ItemFactory.Enumerable.Csv("customers", "data/customers.csv") .Constrain(traits => traits with { CanWrite = false }) .Constrain(traits => traits with { RequiresNetwork = false }) ); @@ -170,13 +170,13 @@ Default values represent **filesystem-file** semantics — the common case for l Prevent pipelines from modifying reference data used across multiple projects: ```csharp -public ICatalogEntry> Countries => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Csv("countries", "reference/countries.csv") +public IItem> Countries => GetOrCreateEntry(() => + ItemFactory.Enumerable.Csv("countries", "reference/countries.csv") .Constrain(traits => traits with { CanWrite = false }) ); -public ICatalogEntry> Currencies => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Json("currencies", "reference/currencies.json") +public IItem> Currencies => GetOrCreateEntry(() => + ItemFactory.Enumerable.Json("currencies", "reference/currencies.json") .Constrain(traits => traits with { CanWrite = false }) ); ``` @@ -186,9 +186,9 @@ public ICatalogEntry> Currencies => GetOrCreateEntry(() => Use configuration to apply different constraints in different environments: ```csharp -public ICatalogEntry> Orders => GetOrCreateEntry(() => +public IItem> Orders => GetOrCreateEntry(() => { - var entry = CatalogEntries.Enumerable.EFCore("orders", dbContext, readOnly: false); + var entry = ItemFactory.Enumerable.EFCore("orders", dbContext, readOnly: false); // In production, make orders read-only for reporting pipelines if (_environment.IsProduction) @@ -206,13 +206,13 @@ In multi-stage pipelines where early stages produce data for later stages, preve ```csharp // Stage 1: Data ingestion (writes allowed) -public ICatalogEntry> RawData => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Csv("raw", "data/raw.csv") +public IItem> RawData => GetOrCreateEntry(() => + ItemFactory.Enumerable.Csv("raw", "data/raw.csv") ); // Stage 2: Used as read-only input to transformations -public ICatalogEntry> RawDataReadOnly => GetOrCreateEntry(() => - CatalogEntries.Enumerable.Csv("raw_readonly", "data/raw.csv") +public IItem> RawDataReadOnly => GetOrCreateEntry(() => + ItemFactory.Enumerable.Csv("raw_readonly", "data/raw.csv") .Constrain(traits => traits with { CanWrite = false }) ); ``` diff --git a/docs/guides/misc/AGENTS.md b/docs/guides/misc/AGENTS.md index 40a61b43..9b205757 100644 --- a/docs/guides/misc/AGENTS.md +++ b/docs/guides/misc/AGENTS.md @@ -26,9 +26,9 @@ Data/ # Everything related to data definitions └── _08_Reporting/ # Descriptive and summary outputs for business consumption Pipelines/ # All pipelines -└── / - ├── Pipeline.cs # Nodes wired to input/output catalog entries - └── Nodes/ +└── / + ├── Pipeline.cs # Steps wired to input/output catalog entries + └── Steps/ └── *.cs # Individual transformation nodes ``` @@ -49,7 +49,7 @@ Follow these steps in order: 2. **Plan intermediate schemas.** Identify what data shapes are needed between each transformation step. 3. **Write schemas.** Create schema classes for input, output, and all intermediary data in the appropriate `Data//Schemas/` directories. 4. **Create catalog entries.** Add catalog entries for each schema in the corresponding `Catalog..cs` file. Each entry declares its schema type and storage strategy. -5. **Write nodes.** Implement the transformation logic in `Pipelines//Nodes/`. Each node declares typed inputs and outputs matching the schemas from step 3. -6. **Wire the pipeline.** Connect nodes to their catalog entries in `Pipelines//Pipeline.cs`. +5. **Write nodes.** Implement the transformation logic in `Pipelines//Steps/`. Each node declares typed inputs and outputs matching the schemas from step 3. +6. **Wire the pipeline.** Connect nodes to their catalog entries in `Pipelines//Pipeline.cs`. 7. **Register the pipeline.** Add the pipeline to `Program.cs` so the CLI runner can discover it. 8. **Run and confirm.** Execute the pipeline with `dotnet run` and verify the output. diff --git a/docs/guides/slicing-pipelines.md b/docs/guides/slicing-pipelines.md index 3eaddb45..6e8648d2 100644 --- a/docs/guides/slicing-pipelines.md +++ b/docs/guides/slicing-pipelines.md @@ -100,11 +100,11 @@ using Flowthru.Services.Models; var options = new ExecutionOptions { - SliceStrategy = new PipelineSliceStrategy + SliceStrategy = new FlowSliceStrategy { Pipelines = new HashSet { "DataScience" }, - FromNodes = new HashSet { "PreprocessCompanies", "PreprocessShuttles" }, - ToNodes = new HashSet { "CreateModelInput" } + FromSteps = new HashSet { "PreprocessCompanies", "PreprocessShuttles" }, + ToSteps = new HashSet { "CreateModelInput" } } }; @@ -114,14 +114,14 @@ var result = await service.ExecutePipelineAsync(options, exportMetadata: true, m ### All Strategy Properties ```csharp -var strategy = new PipelineSliceStrategy +var strategy = new FlowSliceStrategy { Pipelines = new HashSet { "DataScience", "Reporting" }, // filter by pipeline name - FromNodes = new HashSet { "NodeA", "NodeB" }, // + upstream - ToNodes = new HashSet { "NodeC" }, // + downstream + FromSteps = new HashSet { "StepA", "StepB" }, // + upstream + ToSteps = new HashSet { "StepC" }, // + downstream FromData = new HashSet { "model_input" }, // consumers + downstream ToData = new HashSet { "raw_data" }, // producers + upstream - OnlyNodes = new HashSet { "NodeD", "NodeE" } // explicit allowlist + OnlySteps = new HashSet { "StepD", "StepE" } // explicit allowlist }; ``` @@ -135,12 +135,12 @@ var options = new ExecutionOptions(); // No slicing, all pipelines execute Slicing guarantees runnability — the result is always a valid sub-DAG: -1. **Pipelines** filters nodes by pipeline name prefix (e.g., "DataScience.NodeName") -2. **FromNodes** includes all upstream dependencies (transitive closure) -3. **ToNodes** includes all downstream dependents (transitive closure) +1. **Pipelines** filters nodes by pipeline name prefix (e.g., "DataScience.StepName") +2. **FromSteps** includes all upstream dependencies (transitive closure) +3. **ToSteps** includes all downstream dependents (transitive closure) 4. **FromData** resolves to consumer nodes, then includes downstream (transitive closure) 5. **ToData** resolves to producer nodes, then includes upstream (transitive closure) -6. **OnlyNodes** automatically includes required dependencies +6. **OnlySteps** automatically includes required dependencies 7. Multiple strategies intersect — each narrows the result set **Slicing is additive only.** There is no `--except` flag because subtractive operations break the runnability guarantee. @@ -169,7 +169,7 @@ Available pipelines: DataScience, DataProcessing, Reporting ``` ``` -✗ FromNodes references non-existent node: 'InvalidNode' +✗ FromSteps references non-existent node: 'InvalidStep' Available nodes: PreprocessCompanies, PreprocessShuttles, ... ``` diff --git a/docs/guides/using-efcore-catalog-entries.md b/docs/guides/using-efcore-catalog-entries.md index 38e2e31d..34b33790 100644 --- a/docs/guides/using-efcore-catalog-entries.md +++ b/docs/guides/using-efcore-catalog-entries.md @@ -8,7 +8,7 @@ This guide is for teams who already have a working Flowthru pipeline backed by f dotnet add package Flowthru.Extensions.EFCore ``` -This package provides `EFCoreCatalogEntries`, the primary namespace for creating EFCore-backed catalog entries. +This package provides `EFCoreItemFactory`, the primary namespace for creating EFCore-backed catalog entries. ## What your schemas need @@ -87,7 +87,7 @@ Wire the factory from your DI container: ```csharp // Program.cs -flowthru.UseCatalog(sp => new Catalog( +flowthru.RegisterCatalog(sp => new Catalog( basePath: Path.Combine(basePath, "Data"), contextFactory: sp.GetRequiredService>() )); @@ -103,9 +103,9 @@ using Flowthru.Extensions.EFCore.Data; public partial class Catalog { - public ICatalogEntry> PreprocessedCompanies => + public IItem> PreprocessedCompanies => GetOrCreateEntry(() => - EFCoreCatalogEntries.Enumerable.EFCore( + EFCoreItemFactory.Enumerable.EFCore( label: "PreprocessedCompanies", contextFactory: _contextFactory ) @@ -123,9 +123,9 @@ For tables that store exactly one row — a trained model, a configuration recor // Data/_06_Models/Catalog.Models.cs public partial class Catalog { - public ICatalogEntry Regressor => + public IItem Regressor => GetOrCreateEntry(() => - EFCoreCatalogEntries.Single.EFCore( + EFCoreItemFactory.Single.EFCore( label: "Regressor", contextFactory: _contextFactory ) @@ -141,21 +141,21 @@ The `queryCustomizer` parameter receives the raw `IQueryable` before the adap ```csharp // Order for deterministic output -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItemFactory.Enumerable.EFCore( label: "Shuttles", contextFactory: _contextFactory, queryCustomizer: q => q.OrderBy(s => s.Id) ) // Load with a navigation property -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItemFactory.Enumerable.EFCore( label: "Persons", contextFactory: _contextFactory, queryCustomizer: q => q.Include(p => p.Address).AsNoTracking() ) // Filter to a subset -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItemFactory.Enumerable.EFCore( label: "ActiveRecords", contextFactory: _contextFactory, queryCustomizer: q => q.Where(r => r.IsActive) @@ -170,7 +170,7 @@ The default save strategy is `RemoveRange` + `AddRange` (full replace semantics, ```csharp // Upsert instead of replace -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItemFactory.Enumerable.EFCore( label: "Companies", contextFactory: _contextFactory, saveFunc: async (ctx, data, ct) => @@ -206,7 +206,7 @@ internal static class MyProjectSaveFuncs } // In your catalog entry: -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItemFactory.Enumerable.EFCore( label: "Companies", contextFactory: _contextFactory, saveFunc: MyProjectSaveFuncs.ReplaceCompanies @@ -228,7 +228,7 @@ saveFunc: async (ctx, data, ct) => Use `.Constrain()` to mark source or reference tables as read-only. Flowthru will fail at pipeline build time — before any data moves — if a node attempts to write to this entry: ```csharp -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItemFactory.Enumerable.EFCore( label: "SourceRecords", contextFactory: _sourceFactory) .Constrain(traits => traits with { CanWrite = false }) @@ -241,7 +241,7 @@ See [Constraining Catalog Entries](constraining-catalog-entries.md) for the full By default, an empty table fails pre-flight validation. The adapter expects each catalog entry to have data before the pipeline runs. For tables that are legitimately empty on first run — audit logs, optional output tables, or incremental pipelines — set `allowEmptyData: true`: ```csharp -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItemFactory.Enumerable.EFCore( label: "AuditEvents", contextFactory: _contextFactory, allowEmptyData: true diff --git a/docs/project.json b/docs/project.json new file mode 100644 index 00000000..292c1716 --- /dev/null +++ b/docs/project.json @@ -0,0 +1,19 @@ +{ + "name": "docs", + "$schema": "../node_modules/nx/schemas/project-schema.json", + "projectType": "library", + "sourceRoot": "./docs", + "targets": { + "build:reference": { + "//": "Generates markdown API reference from C# docstrings into docs/reference/src/", + "executor": "nx:run-commands", + "dependsOn": [ "flowthru:restore" ], + "options": { + "commands": [ + "./scripts/docfx-metadata.sh --projects src" + ], + "parallel": false + } + } + } +} diff --git a/docs/reference/.gitkeep b/docs/reference/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.FlowthruSchemaAttribute.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.FlowthruSchemaAttribute.md new file mode 100644 index 00000000..cc8cbee1 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.FlowthruSchemaAttribute.md @@ -0,0 +1,71 @@ +# Class FlowthruSchemaAttribute + +Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md) +Assembly: Flowthru.Core.dll + +Marks a schema type for automatic interface generation. The source generator +will analyze the type's properties and emit the appropriate marker interfaces: +
  • or based on property types
  • for flat schemas (CSV/TSV compatible)
  • for flat schemas (Parquet compatible)
  • for all schemas (JSON/XML compatible)
+ +```csharp +[AttributeUsage(AttributeTargets.Class|AttributeTargets.Struct, Inherited = false, AllowMultiple = false)] +public sealed class FlowthruSchemaAttribute : Attribute +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Attribute](https://learn.microsoft.com/dotnet/api/system.attribute) ← +[FlowthruSchemaAttribute](Flowthru.Abstractions.FlowthruSchemaAttribute.md) + +#### Inherited Members + +[Attribute.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.attribute.equals), +[Attribute.GetCustomAttribute\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-assembly\-system\-type\)), +[Attribute.GetCustomAttribute\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.GetCustomAttribute\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-module\-system\-type\)), +[Attribute.GetCustomAttribute\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.GetCustomAttribute\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Assembly\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\)), +[Attribute.GetCustomAttributes\(Assembly, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-type\)), +[Attribute.GetCustomAttributes\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(MemberInfo\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\)), +[Attribute.GetCustomAttributes\(MemberInfo, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-boolean\)), +[Attribute.GetCustomAttributes\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.GetCustomAttributes\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Module\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\)), +[Attribute.GetCustomAttributes\(Module, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-type\)), +[Attribute.GetCustomAttributes\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(ParameterInfo\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\)), +[Attribute.GetCustomAttributes\(ParameterInfo, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-boolean\)), +[Attribute.GetCustomAttributes\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.GetCustomAttributes\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.attribute.gethashcode), +[Attribute.IsDefaultAttribute\(\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefaultattribute), +[Attribute.IsDefined\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-assembly\-system\-type\)), +[Attribute.IsDefined\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.IsDefined\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-module\-system\-type\)), +[Attribute.IsDefined\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.IsDefined\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.Match\(object?\)](https://learn.microsoft.com/dotnet/api/system.attribute.match), +[Attribute.TypeId](https://learn.microsoft.com/dotnet/api/system.attribute.typeid), +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +The annotated type must be declared as partial. The generator inspects +public instance properties to determine structural classification. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IBinarySerializable.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IBinarySerializable.md new file mode 100644 index 00000000..45d87f08 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IBinarySerializable.md @@ -0,0 +1,68 @@ +# Interface IBinarySerializable + +Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md) +Assembly: Flowthru.Core.dll + +Marker interface for schema types that can be serialized to columnar binary formats (Parquet, Avro). + +```csharp +public interface IBinarySerializable +``` + +## Examples + +
// ✅ Flat schema with binary serialization - optimal
+public sealed record FeatureRow(
+    DateTime Timestamp,
+    double Feature1,
+    double Feature2,
+    int Label
+) : IFlatSchema, IBinarySerializable, ITextSerializable;
+
+// ✅ Multiple format support for flexibility
+public sealed record CompanySchema(
+    int Id,
+    string Name,
+    float Rating
+) : IFlatSchema, ITextSerializable, IBinarySerializable, IStructuredSerializable;
+
+// ⚠️ Nested schema - binary serialization less optimal
+public sealed record OrderWithItemFactory(
+    string OrderId,
+    List<LineItem> ItemFactory  // Nested column in Parquet
+) : INestedSchema, IBinarySerializable, IStructuredSerializable;
+ +## Remarks + +

+Purpose: Indicates a schema is compatible with efficient, columnar binary +formats designed for analytical workloads. +

+

+Compatible Formats: +

+
  • Parquet (Apache Parquet - columnar storage)
  • Avro (Apache Avro - row-based but schema-aware)
  • ORC (Optimized Row Columnar - Hive format)
  • Arrow (Apache Arrow - in-memory columnar format)
+

+Requirements: +

+

+Most binary formats work best with flat schemas, though formats like Parquet +support nested columns: +

+
  • Flat schemas (): Optimal - full columnar compression
  • Nested schemas (): Supported but may lose some benefits
+

+Design Rationale: +

+

+Binary serialization offers significant advantages: +

+
  • Performance: Orders of magnitude faster read/write than CSV
  • Compression: Columnar formats achieve excellent compression ratios
  • Type Preservation: Native storage of numeric types (no string conversion)
  • Predicate Pushdown: Read only required columns/rows
  • Schema Evolution: Built-in schema versioning support
+

+When to Use: +

+
  • Large datasets (>10MB)
  • Analytical queries (aggregations, filters)
  • Data lake storage
  • Machine learning feature stores
  • Long-term data archival
+

+Trade-offs: +

+
  • ❌ Not human-readable (use JSON for debugging)
  • ❌ Requires specialized libraries (not plain text editors)
  • ✅ Superior performance for large-scale data
  • ✅ Industry standard for big data ecosystems
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IFlatSchema.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IFlatSchema.md new file mode 100644 index 00000000..e57eacc0 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IFlatSchema.md @@ -0,0 +1,62 @@ +# Interface IFlatSchema + +Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md) +Assembly: Flowthru.Core.dll + +Marker interface for schema types with flat (non-nested) structure. + +```csharp +public interface IFlatSchema +``` + +## Examples + +
// ✅ Flat schema with multiple serialization capabilities
+public sealed record CompanySchema(
+    int Id,
+    string Name,
+    float Rating,
+    bool IsActive
+) : IFlatSchema, ITextSerializable, IBinarySerializable;
+
+// ❌ Not flat - contains collection
+public sealed record OrderSchema(
+    string OrderId,
+    List<LineItem> ItemFactory  // Collection = nested!
+) : INestedSchema, IStructuredSerializable;
+ +## Remarks + +

+Purpose: Indicates a schema contains only primitive properties, +with no collections or nested objects. +

+

+Flat Structure Requirements: +

+
  • All properties are primitives (int, long, double, decimal, bool, string, DateTime, etc.)
  • Nullable primitives (int?, double?, bool?, etc.) are allowed
  • Enums and nullable enums are allowed
  • Value types that serialize to single values (Guid, TimeSpan, etc.) are allowed
+

+Not Flat (Use INestedSchema instead): +

+
  • Collections: List<T>, Array, IEnumerable<T>, etc.
  • Nested objects: Custom class/record properties
  • Dictionaries or complex structures
+

+Design Rationale: +

+

+Separating structural properties (flat vs nested) from serialization capabilities +(text/structured/binary) enables: +- Explicit schema documentation +- Compile-time format validation +- Format serializers can enforce appropriate constraints +- Clear intent about data structure complexity +

+

+Relationship with Serialization Markers: +

+

+A flat schema should also implement one or more serialization capability markers: +- - Can serialize to CSV/TSV +- - Can serialize to JSON/XML +- - Can serialize to Parquet/Avro +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IFlatSerializable.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IFlatSerializable.md new file mode 100644 index 00000000..0ea58d3c --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IFlatSerializable.md @@ -0,0 +1,70 @@ +# Interface IFlatSerializable + +Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md) +Assembly: Flowthru.Core.dll + +Marker interface for schema types that contain only flat, primitive data. + +```csharp +public interface IFlatSerializable +``` + +## Examples + +
// ✅ Flat schema - CSV compatible
+public record CompanySchema : IFlatSerializable
+{
+    public string Id { get; init; } = null!;
+    public string Name { get; init; } = null!;
+    public decimal Rating { get; init; }
+    public int FoundedYear { get; init; }
+    public bool IataApproved { get; init; }
+}
+
+// ✅ Compiles successfully - flat schema with flat storage
+catalog.Companies = CreateCsvDataset<CompanySchema>("companies", "companies.csv");
+
+// ❌ Nested schema - requires JSON or Parquet
+public record CrossValidationResults : INestedSerializable
+{
+    public List<FoldMetric> FoldMetrics { get; init; } = new();
+    public double MeanR2Score { get; init; }
+}
+
+// ❌ Compile error: CrossValidationResults does not implement IFlatSerializable
+catalog.Results = CreateCsvDataset<CrossValidationResults>("results", "results.csv");
+
+// ✅ Correct: Use JSON for nested data
+catalog.Results = CreateJsonObject<CrossValidationResults>("results", "results.json");
+ +## Remarks + +

+Purpose: Enables compile-time validation that flat file formats +(CSV, TSV, etc.) receive compatible schema types. +

+

+What qualifies as "flat"? +

+

+A schema is considered flat if all properties are: +

+
  • Primitive types (int, long, double, decimal, bool, string, DateTime, DateTimeOffset, etc.)
  • Nullable primitives (int?, double?, bool?, etc.)
  • Enums (and nullable enums)
  • Value types that serialize to single values (Guid, TimeSpan, etc.)
+

+A schema is NOT flat if it contains: +

+
  • Collections (List<T>, Array, IEnumerable<T>, etc.)
  • Nested objects (custom class/record properties)
  • Dictionaries or other complex structures
+

+Compatible Storage Formats: +

+
  • CSV files (CsvCatalogDataset<T>)
  • Excel files (ExcelCatalogDataset<T>)
  • JSON files (JsonCatalogDataset<T> - also supports nested)
  • Parquet files (ParquetCatalogDataset<T> - also supports nested)
  • Database tables (flat row structure)
+

+Design Rationale: +

+

+This marker interface follows Flowthru's philosophy of "fail at compile-time, not runtime." +By requiring flat-only storage formats to constrain their generic type parameter with +where T : IFlatSerializable, we catch schema-format mismatches during compilation +rather than discovering silent data loss or runtime serialization errors during execution. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.INestedSchema.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.INestedSchema.md new file mode 100644 index 00000000..7375820c --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.INestedSchema.md @@ -0,0 +1,66 @@ +# Interface INestedSchema + +Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md) +Assembly: Flowthru.Core.dll + +Marker interface for schema types with nested structure (collections or nested objects). + +```csharp +public interface INestedSchema +``` + +## Examples + +
// ✅ Nested schema with collection
+public sealed record CrossValidationResults(
+    List<FoldMetric> FoldMetrics,  // Nested collection
+    double MeanR2Score,
+    int NumFolds
+) : INestedSchema, IStructuredSerializable;
+
+// ✅ Nested schema with nested object
+public sealed record CustomerOrder(
+    string OrderId,
+    CustomerInfo Customer,  // Nested object
+    DateTime OrderDate
+) : INestedSchema, IStructuredSerializable;
+
+// Individual nested type can be flat
+public sealed record FoldMetric(
+    int FoldNumber,
+    double R2Score
+) : IFlatSchema, ITextSerializable;
+ +## Remarks + +

+Purpose: Indicates a schema contains hierarchical data requiring +storage formats that support nested structures. +

+

+Nested Structure Characteristics: +

+
  • Collections: List<T>, Array, IEnumerable<T>, ICollection<T>, etc.
  • Nested objects: Properties that are custom classes or records
  • Dictionaries: Dictionary<TKey, TValue>, IDictionary, etc.
  • Complex hierarchical structures
+

+Compatible Storage Formats: +

+
  • JSON files (preserves hierarchy)
  • Parquet files (supports nested columns)
  • XML files (hierarchical by nature)
  • Document databases (MongoDB, etc.)
+

+Incompatible Storage Formats: +

+
  • ❌ CSV files - cannot represent nested structures
  • ❌ Excel files - limited nesting support, loses structure
  • ❌ Flat relational tables - requires denormalization
+

+Design Rationale: +

+

+This marker interface serves multiple purposes: +

+
  1. Self-Documentation: Clearly signals schema complexity
  2. Intent Declaration: Distinguishes "happens to be flat" from "designed to be flat"
  3. Migration Safety: Identifies schemas that cannot use flat formats
  4. Future Validation: Enables build-time checks via analyzers
+

+Relationship with IFlatSchema: +

+

+These interfaces are mutually exclusive. A schema should implement exactly one: +

+
  • - All primitive properties
  • - Contains collections or nested objects
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.INestedSerializable.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.INestedSerializable.md new file mode 100644 index 00000000..779d3d4c --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.INestedSerializable.md @@ -0,0 +1,78 @@ +# Interface INestedSerializable + +Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md) +Assembly: Flowthru.Core.dll + +Marker interface for schema types that contain nested structures or collections. + +```csharp +public interface INestedSerializable +``` + +## Examples + +
// ✅ Nested schema - requires hierarchical storage
+public record CrossValidationResults : INestedSerializable
+{
+    // Collection of nested objects
+    public List<FoldMetric> FoldMetrics { get; init; } = new();
+
+    // Flat properties are fine in nested schemas
+    public double MeanR2Score { get; init; }
+    public double StdDevR2Score { get; init; }
+    public int NumFolds { get; init; }
+}
+
+public record FoldMetric : IFlatSerializable  // Individual fold is flat
+{
+    public int FoldNumber { get; init; }
+    public double R2Score { get; init; }
+    public double MeanAbsoluteError { get; init; }
+}
+
+// ✅ Correct: Use JSON for nested data
+catalog.CrossValidationResults = CreateJsonObject<CrossValidationResults>(
+    "cross_validation_results",
+    "model_output/cross_validation_results.json");
+
+// ❌ Compile error if attempted with CSV (prevented by IFlatSerializable constraint)
+// catalog.CrossValidationResults = CreateCsvDataset<CrossValidationResults>(...);
+ +## Remarks + +

+Purpose: Documents that a schema contains hierarchical data and +requires storage formats that support nested structures (JSON, Parquet, XML, etc.). +

+

+What qualifies as "nested"? +

+

+A schema is considered nested if it contains one or more of: +

+
  • Collections (List<T>, Array, IEnumerable<T>, ICollection<T>, etc.)
  • Nested objects (properties that are custom classes or records)
  • Dictionaries (Dictionary<TKey, TValue>, IDictionary, etc.)
  • Complex hierarchical structures
+

+Compatible Storage Formats: +

+
  • JSON files (JsonCatalogDataset<T>, JsonCatalogObject<T>)
  • Parquet files (ParquetCatalogDataset<T> - supports nested columns)
  • XML files (if implemented)
  • Document databases (MongoDB, etc.)
+

+Incompatible Storage Formats: +

+
  • ❌ CSV files - cannot represent nested structures
  • ❌ Excel files - limited nesting support, loses structure
  • ❌ Relational database tables - requires denormalization or separate tables
+

+Design Rationale: +

+

+While this interface is not enforced by generic constraints (since nested-compatible +formats like JSON can also handle flat data), it serves important documentation and +validation purposes: +

+
  1. Self-Documentation: Clearly signals schema complexity to developers
  2. Future Analyzer Support: Enables build-time validation via Roslyn analyzers
  3. Intent Declaration: Distinguishes "happens to be flat" from "designed to be flat"
  4. Migration Safety: Helps identify schemas that cannot be migrated to flat formats
+

+Relationship with IFlatSerializable: +

+

+These interfaces are mutually exclusive. A schema should implement exactly one: +

+
  • - All primitive properties, CSV-compatible
  • - Contains collections or nested objects
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IScalar.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IScalar.md new file mode 100644 index 00000000..cd7e7808 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IScalar.md @@ -0,0 +1,67 @@ +# Interface IScalar + +Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md) +Assembly: Flowthru.Core.dll + +Marker interface for types that serialize to a single primitive value — +i.e., they produce "key": value in JSON, not "key": {...} or "key": [...]. + +```csharp +public interface IScalar +``` + +## Examples + +
// ✅ Single-value wrapper — safe to implement IScalar
+public readonly record struct CustomerId(string Value) : IScalar;
+
+// ✅ Schema using the NewType is classified as flat
+[FlowthruSchema]
+public partial record OrderSchema
+{
+    public required CustomerId Id { get; init; }
+    public required string Name { get; init; }
+}
+
+// ❌ Multi-property struct — NOT a scalar, do not implement IScalar
+public readonly record struct Address(string Street, string City) /* : IScalar — wrong */;
+ +## Remarks + +

+Purpose: Enables user-defined NewTypes and value-object wrappers to participate +in flat schema classification. Without this interface, the source generator cannot distinguish +a CustomerId wrapping a string from a nested object requiring {...} +serialization. +

+

+The JSON Test: +

+

+The definitive question when implementing this interface is: does your type serialize to a +single JSON value? A type is a flat scalar if and only if it produces one of: +

+
  • "key": 42 — numeric
  • "key": "abc" — string
  • "key": true — boolean
  • "key": null — null
+

+If your type requires "key": { "inner": ... } or "key": [...], it is NOT a +scalar and implementing this interface would misrepresent its structure, causing silent data +loss or serialization failures in flat formats like CSV. +

+

+Typical Use Cases: +

+
  • NewType / value-object wrappers: record CustomerId(string Value) : IScalar
  • Strong-typed identifiers backed by a primitive: record OrderRef(Guid Id) : IScalar
  • Domain primitives that round-trip through a single string or numeric field
+

+What this is NOT for: +

+
  • Types with multiple public properties — those are nested objects
  • Collections or dictionaries
  • BCL types like Guid, DateTime, TimeSpan — those are recognized +automatically by the source generator as known flat scalars
+

+Relationship with : +

+

+ marks a row type — a schema whose properties are all +scalars. marks a property type — a single value that can +appear as a column in a flat row. Nesting a flat row inside another row is still nesting. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IStructuredSerializable.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IStructuredSerializable.md new file mode 100644 index 00000000..5024d787 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IStructuredSerializable.md @@ -0,0 +1,66 @@ +# Interface IStructuredSerializable + +Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md) +Assembly: Flowthru.Core.dll + +Marker interface for schema types that can be serialized to structured formats (JSON, XML). + +```csharp +public interface IStructuredSerializable +``` + +## Examples + +
// ✅ Nested schema - structured serialization required
+public sealed record CrossValidationResults(
+    List<FoldMetric> FoldMetrics,
+    double MeanScore
+) : INestedSchema, IStructuredSerializable;
+
+// ✅ Flat schema - structured serialization optional
+public sealed record ModelMetrics(
+    double Accuracy,
+    double Precision,
+    double Recall
+) : IFlatSchema, IStructuredSerializable, ITextSerializable;
+
+// ✅ Configuration object
+public sealed record TrainingConfig(
+    int Epochs,
+    double LearningRate,
+    Dictionary<string, object> Hyperparameters
+) : INestedSchema, IStructuredSerializable;
+ +## Remarks + +

+Purpose: Indicates a schema is compatible with structured, hierarchical +formats that can represent nested objects and collections. +

+

+Compatible Formats: +

+
  • JSON (JavaScript Object Notation)
  • XML (Extensible Markup Language)
  • YAML (YAML Ain't Markup Language)
  • TOML (Tom's Obvious Minimal Language)
+

+Flexibility: +

+

+Unlike which requires flat schemas, structured formats +can handle both: +

+
  • - Simple, flat data structures
  • - Complex, hierarchical data structures
+

+Design Rationale: +

+

+Structured serialization is the most flexible format capability: +- Can represent any schema structure (flat or nested) +- Human-readable and widely supported +- Preserves type information and hierarchy +- Suitable for configuration, results, and complex data +

+

+Typical Usage Patterns: +

+
  • Nested schemas: Must use structured serialization (JSON/XML only option)
  • Flat schemas: Can use structured serialization when human-readability matters
  • Configuration objects: Often use JSON for flexibility
  • Model metadata: JSON captures complex metrics and parameters
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.ITextSerializable.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.ITextSerializable.md new file mode 100644 index 00000000..add434c6 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.ITextSerializable.md @@ -0,0 +1,67 @@ +# Interface ITextSerializable + +Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md) +Assembly: Flowthru.Core.dll + +Marker interface for schema types that can be serialized to text-based formats (CSV, TSV). + +```csharp +public interface ITextSerializable +``` + +## Examples + +
// ✅ Flat schema with text serialization
+public sealed record CompanySchema(
+    int Id,
+    string Name,
+    float Rating
+) : IFlatSchema, ITextSerializable;
+
+// ✅ Flat schema with multiple serialization capabilities
+public sealed record DataRow(
+    DateTime Timestamp,
+    double Value,
+    string Category
+) : IFlatSchema, ITextSerializable, IBinarySerializable;
+
+// ❌ Cannot be text serialized - nested structure
+public sealed record NestedData(
+    string Id,
+    List<string> Tags  // Collection!
+) : INestedSchema, IStructuredSerializable;  // Not ITextSerializable
+ +## Remarks + +

+Purpose: Indicates a schema is compatible with text-based, flat file +formats that represent data in rows and columns. +

+

+Compatible Formats: +

+
  • CSV (Comma-Separated Values)
  • TSV (Tab-Separated Values)
  • Other delimited text formats
+

+Requirements: +

+

+Schemas implementing this interface must: +

+
  • Also implement (no nested structures)
  • Have properties that can be converted to/from string representation
  • Support single-value serialization per field
+

+Design Rationale: +

+

+Separating serialization capability from structure allows: +- Compile-time enforcement: CsvFormatSerializer<T> where T : IFlatSchema, ITextSerializable +- Multiple serialization targets: A flat schema can be both text and binary serializable +- Clear documentation of supported formats +

+

+Typical Usage: +

+

+Most flat schemas should implement both and + to support maximum format flexibility. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.SerializedEnumAttribute.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.SerializedEnumAttribute.md new file mode 100644 index 00000000..5cdb99dc --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.SerializedEnumAttribute.md @@ -0,0 +1,171 @@ +# Class SerializedEnumAttribute + +Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md) +Assembly: Flowthru.Core.dll + +Specifies the serialized string value for an enum member when written to or read from storage. + +```csharp +[AttributeUsage(AttributeTargets.Field, AllowMultiple = false, Inherited = false)] +public sealed class SerializedEnumAttribute : Attribute +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Attribute](https://learn.microsoft.com/dotnet/api/system.attribute) ← +[SerializedEnumAttribute](Flowthru.Abstractions.SerializedEnumAttribute.md) + +#### Inherited Members + +[Attribute.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.attribute.equals), +[Attribute.GetCustomAttribute\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-assembly\-system\-type\)), +[Attribute.GetCustomAttribute\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.GetCustomAttribute\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-module\-system\-type\)), +[Attribute.GetCustomAttribute\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.GetCustomAttribute\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Assembly\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\)), +[Attribute.GetCustomAttributes\(Assembly, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-type\)), +[Attribute.GetCustomAttributes\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(MemberInfo\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\)), +[Attribute.GetCustomAttributes\(MemberInfo, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-boolean\)), +[Attribute.GetCustomAttributes\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.GetCustomAttributes\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Module\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\)), +[Attribute.GetCustomAttributes\(Module, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-type\)), +[Attribute.GetCustomAttributes\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(ParameterInfo\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\)), +[Attribute.GetCustomAttributes\(ParameterInfo, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-boolean\)), +[Attribute.GetCustomAttributes\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.GetCustomAttributes\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.attribute.gethashcode), +[Attribute.IsDefaultAttribute\(\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefaultattribute), +[Attribute.IsDefined\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-assembly\-system\-type\)), +[Attribute.IsDefined\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.IsDefined\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-module\-system\-type\)), +[Attribute.IsDefined\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.IsDefined\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.Match\(object?\)](https://learn.microsoft.com/dotnet/api/system.attribute.match), +[Attribute.TypeId](https://learn.microsoft.com/dotnet/api/system.attribute.typeid), +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
// Example: Magic: The Gathering color enum
+public enum MtgColor
+{
+    [SerializedEnum("W")]
+    White,
+
+    [SerializedEnum("U")]
+    Blue,
+
+    [SerializedEnum("B")]
+    Black,
+
+    [SerializedEnum("R")]
+    Red,
+
+    [SerializedEnum("G")]
+    Green
+}
+
+// Example: Rarity enum with lowercase convention
+public enum Rarity
+{
+    [SerializedEnum("common")]
+    Common,
+
+    [SerializedEnum("uncommon")]
+    Uncommon,
+
+    [SerializedEnum("rare")]
+    Rare,
+
+    [SerializedEnum("mythic")]
+    Mythic
+}
+
+// Usage in schema - no additional configuration needed
+public record Card(
+    string Name,
+    MtgColor Color,  // Automatically serializes using [SerializedEnum] mappings
+    Rarity Rarity    // Works across all formats (JSON, CSV, Excel, Parquet)
+) : IFlatSchema, ITextSerializable;
+ +## Remarks + +

+Purpose: Provides explicit string mappings for enum values across all storage formats. +

+

+Format Agnostic: Works uniformly across CSV, Excel, JSON, Parquet, and future formats. +The underlying serialization mechanism is abstracted away. +

+

+Required for All Enum Members: Every enum member must have this attribute when used +in schemas that will be serialized. This ensures explicit, documented mappings and prevents +accidental mismatches between C# names and external data formats. +

+

+Validation: The Flowthru pipeline validates at build time that all enum members +used in schemas have this attribute applied. +

+

+Common Use Cases: +

+
  • Abbreviated values: [SerializedEnum("W")] White
  • Lowercase conventions: [SerializedEnum("common")] Common
  • Snake case: [SerializedEnum("double_faced_token")] DoubleFacedToken
  • Legacy formats: [SerializedEnum("STATUS_ACTIVE")] Active
+ +## Constructors + +### SerializedEnumAttribute\(string\) + +Initializes a new instance of the class. + +```csharp +public SerializedEnumAttribute(string value) +``` + +#### Parameters + +`value` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The string value to use when serializing this enum member. + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +Thrown when value is null. + + [ArgumentException](https://learn.microsoft.com/dotnet/api/system.argumentexception) + +Thrown when value is empty or whitespace. + +## Properties + +### Value + +Gets the serialized string value for this enum member. + +```csharp +public string Value { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.SerializedLabelAttribute.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.SerializedLabelAttribute.md new file mode 100644 index 00000000..383b534e --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.SerializedLabelAttribute.md @@ -0,0 +1,143 @@ +# Class SerializedLabelAttribute + +Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md) +Assembly: Flowthru.Core.dll + +Specifies the external field name for a property when serialized to/from storage. + +```csharp +[AttributeUsage(AttributeTargets.Property, AllowMultiple = false, Inherited = true)] +public sealed class SerializedLabelAttribute : Attribute +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Attribute](https://learn.microsoft.com/dotnet/api/system.attribute) ← +[SerializedLabelAttribute](Flowthru.Abstractions.SerializedLabelAttribute.md) + +#### Inherited Members + +[Attribute.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.attribute.equals), +[Attribute.GetCustomAttribute\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-assembly\-system\-type\)), +[Attribute.GetCustomAttribute\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.GetCustomAttribute\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-module\-system\-type\)), +[Attribute.GetCustomAttribute\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.GetCustomAttribute\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Assembly\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\)), +[Attribute.GetCustomAttributes\(Assembly, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-type\)), +[Attribute.GetCustomAttributes\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(MemberInfo\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\)), +[Attribute.GetCustomAttributes\(MemberInfo, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-boolean\)), +[Attribute.GetCustomAttributes\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.GetCustomAttributes\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Module\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\)), +[Attribute.GetCustomAttributes\(Module, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-type\)), +[Attribute.GetCustomAttributes\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(ParameterInfo\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\)), +[Attribute.GetCustomAttributes\(ParameterInfo, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-boolean\)), +[Attribute.GetCustomAttributes\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.GetCustomAttributes\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.attribute.gethashcode), +[Attribute.IsDefaultAttribute\(\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefaultattribute), +[Attribute.IsDefined\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-assembly\-system\-type\)), +[Attribute.IsDefined\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.IsDefined\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-module\-system\-type\)), +[Attribute.IsDefined\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.IsDefined\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.Match\(object?\)](https://learn.microsoft.com/dotnet/api/system.attribute.match), +[Attribute.TypeId](https://learn.microsoft.com/dotnet/api/system.attribute.typeid), +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
// Tier 1: No annotation - property name matches external field name
+public record SimpleSchema(
+    int Id,           // Looks for "Id" in external data
+    string Name       // Looks for "Name" in external data
+) : IFlatSchema, ITextSerializable;
+
+// Tier 2: Explicit annotation - handle naming mismatches
+public record ShuttleSchema(
+    string Id,                                    // Looks for "Id"
+    [SerializedLabel("shuttle_location")]         // Looks for "shuttle_location"
+    string ShuttleLocation,
+
+    [SerializedLabel("d_check_complete")]         // Looks for "d_check_complete"
+    bool DCheckComplete,
+
+    [SerializedLabel("company id")]               // Handles space-separated names
+    int CompanyId
+) : IFlatSchema, ITextSerializable;
+ +## Remarks + +

+Purpose: Bridges C# property names with external data field names across all storage formats. +

+

+Format Agnostic: Works uniformly across CSV, Excel, JSON, Parquet, and future formats. +The serialization mechanism (CsvHelper, ExcelDataReader, System.Text.Json, etc.) is abstracted away. +

+

+Opt-in Override: If not present, the property name is used as-is. +Use this attribute when the external field name differs from the C# property name. +

+

+Common Use Cases: +

+
  • snake_case data sources: [SerializedLabel("company_id")] int CompanyId
  • kebab-case data sources: [SerializedLabel("company-id")] int CompanyId
  • Space-separated names: [SerializedLabel("Company ID")] int CompanyId
  • Legacy column names: [SerializedLabel("CMPNY_ID")] int CompanyId
+ +## Constructors + +### SerializedLabelAttribute\(string\) + +Initializes a new instance of the class. + +```csharp +public SerializedLabelAttribute(string label) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The external field name in the serialized data + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +Thrown when label is null + + [ArgumentException](https://learn.microsoft.com/dotnet/api/system.argumentexception) + +Thrown when label is empty or whitespace + +## Properties + +### Label + +Gets the external field name for serialization. + +```csharp +public string Label { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.md new file mode 100644 index 00000000..92f38f24 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.md @@ -0,0 +1,53 @@ +# Namespace Flowthru.Abstractions + +### Classes + + [FlowthruSchemaAttribute](Flowthru.Abstractions.FlowthruSchemaAttribute.md) + +Marks a schema type for automatic interface generation. The source generator +will analyze the type's properties and emit the appropriate marker interfaces: +
  • or based on property types
  • for flat schemas (CSV/TSV compatible)
  • for flat schemas (Parquet compatible)
  • for all schemas (JSON/XML compatible)
+ + [SerializedEnumAttribute](Flowthru.Abstractions.SerializedEnumAttribute.md) + +Specifies the serialized string value for an enum member when written to or read from storage. + + [SerializedLabelAttribute](Flowthru.Abstractions.SerializedLabelAttribute.md) + +Specifies the external field name for a property when serialized to/from storage. + +### Interfaces + + [IBinarySerializable](Flowthru.Abstractions.IBinarySerializable.md) + +Marker interface for schema types that can be serialized to columnar binary formats (Parquet, Avro). + + [IFlatSchema](Flowthru.Abstractions.IFlatSchema.md) + +Marker interface for schema types with flat (non-nested) structure. + + [IFlatSerializable](Flowthru.Abstractions.IFlatSerializable.md) + +Marker interface for schema types that contain only flat, primitive data. + + [INestedSchema](Flowthru.Abstractions.INestedSchema.md) + +Marker interface for schema types with nested structure (collections or nested objects). + + [INestedSerializable](Flowthru.Abstractions.INestedSerializable.md) + +Marker interface for schema types that contain nested structures or collections. + + [IScalar](Flowthru.Abstractions.IScalar.md) + +Marker interface for types that serialize to a single primitive value — +i.e., they produce "key": value in JSON, not "key": {...} or "key": [...]. + + [IStructuredSerializable](Flowthru.Abstractions.IStructuredSerializable.md) + +Marker interface for schema types that can be serialized to structured formats (JSON, XML). + + [ITextSerializable](Flowthru.Abstractions.ITextSerializable.md) + +Marker interface for schema types that can be serialized to text-based formats (CSV, TSV). + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.FlowthruCli.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.FlowthruCli.md new file mode 100644 index 00000000..2b200f12 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.FlowthruCli.md @@ -0,0 +1,130 @@ +# Class FlowthruCli + +Namespace: [Flowthru.Cli](Flowthru.Cli.md) +Assembly: Flowthru.Core.dll + +Command-line interface wrapper for IFlowthruService. + +```csharp +public sealed class FlowthruCli +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowthruCli](Flowthru.Cli.FlowthruCli.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+FlowthruCli provides a thin CLI layer over the core IFlowthruService. +It handles: +- Command-line argument parsing +- Help/version display +- Result formatting +- Exit code generation +

+

+The CLI delegates all business logic to IFlowthruService, making the +service layer testable and reusable in non-CLI scenarios. +

+ +## Constructors + +### FlowthruCli\(IFlowthruService, ILogger, TextWriter?\) + +Initializes a new CLI instance. + +```csharp +public FlowthruCli(IFlowthruService service, ILogger logger, TextWriter? output = null) +``` + +#### Parameters + +`service` [IFlowthruService](Flowthru.Services.IFlowthruService.md) + +Flowthru service + +`logger` [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger\-1)<[FlowthruCli](Flowthru.Cli.FlowthruCli.md)\> + +Logger instance + +`output` [TextWriter](https://learn.microsoft.com/dotnet/api/system.io.textwriter)? + +Output writer (defaults to Console.Out) + +## Methods + +### RunAsync\(string\[\], CancellationToken\) + +Runs the CLI with the specified arguments. + +```csharp +public Task RunAsync(string[] args, CancellationToken cancellationToken = default) +``` + +#### Parameters + +`args` [string](https://learn.microsoft.com/dotnet/api/system.string)\[\] + +Command-line arguments + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[int](https://learn.microsoft.com/dotnet/api/system.int32)\> + +Exit code (0 for success, non-zero for errors) + +### RunStandaloneAsync\(string\[\], Action, CancellationToken\) + +Creates and runs a standalone Flowthru CLI application with automatic service provider lifecycle management. + +```csharp +public static Task RunStandaloneAsync(string[] args, Action configure, CancellationToken cancellationToken = default) +``` + +#### Parameters + +`args` [string](https://learn.microsoft.com/dotnet/api/system.string)\[\] + +Command-line arguments + +`configure` [Action](https://learn.microsoft.com/dotnet/api/system.action\-1)<[IServiceCollection](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dependencyinjection.iservicecollection)\> + +Configuration callback to register pipelines and services + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[int](https://learn.microsoft.com/dotnet/api/system.int32)\> + +Exit code (0 for success, non-zero for errors) + +#### Remarks + +

+This is the recommended entry point for standalone console applications using Flowthru. +It manages the ServiceProvider lifecycle automatically, ensuring proper disposal of +logging providers and other resources so the process exits cleanly after pipeline completion. +

+

+For applications that integrate Flowthru into an existing DI container (e.g., ASP.NET Core), +use the standard constructor and let the host application manage the ServiceProvider lifecycle. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.md new file mode 100644 index 00000000..cda960d0 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Cli + +### Classes + + [FlowthruCli](Flowthru.Cli.FlowthruCli.md) + +Command-line interface wrapper for IFlowthruService. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.CatalogOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.CatalogOptions.md new file mode 100644 index 00000000..2070ff44 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.CatalogOptions.md @@ -0,0 +1,88 @@ +# Class CatalogOptions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Configuration options for data catalog construction. + +```csharp +public class CatalogOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[CatalogOptions](Flowthru.Configuration.CatalogOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### BasePath + +Base path for dataset files (common constructor parameter). + +```csharp +public string? BasePath { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### ConnectionString + +Connection string for database catalogs (common constructor parameter). + +```csharp +public string? ConnectionString { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### ConstructorArgs + +Constructor arguments for the catalog (mapped to constructor parameters by name). + +```csharp +public Dictionary ConstructorArgs { get; set; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [object](https://learn.microsoft.com/dotnet/api/system.object)\> + +### Environment + +Environment-specific catalog configuration (e.g., local vs. remote). + +```csharp +public string? Environment { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### Type + +The fully-qualified type name of the catalog class (e.g., "MyApp.Data.MyCatalog"). + +```csharp +public string? Type { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ConfigurationExtensions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ConfigurationExtensions.md new file mode 100644 index 00000000..ed6be21f --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ConfigurationExtensions.md @@ -0,0 +1,100 @@ +# Class ConfigurationExtensions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Extension methods for configuration-related operations. + +```csharp +public static class ConfigurationExtensions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ConfigurationExtensions](Flowthru.Configuration.ConfigurationExtensions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### GetValidated\(IConfiguration, string\) + +Binds a configuration section to a strongly-typed object and validates it. + +```csharp +public static T GetValidated(this IConfiguration configuration, string sectionPath) where T : new() +``` + +#### Parameters + +`configuration` [IConfiguration](https://learn.microsoft.com/dotnet/api/microsoft.extensions.configuration.iconfiguration) + +The configuration instance + +`sectionPath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The configuration section path (e.g., "DataScience:ModelParams") + +#### Returns + + T + +The bound and validated object + +#### Type Parameters + +`T` + +The type to bind to + +#### Exceptions + + [ValidationException](https://learn.microsoft.com/dotnet/api/system.componentmodel.dataannotations.validationexception) + +Thrown if DataAnnotations validation fails + +### GetValidatedOrDefault\(IConfiguration, string\) + +Attempts to bind and validate a configuration section, returning null if not found. + +```csharp +public static T? GetValidatedOrDefault(this IConfiguration configuration, string sectionPath) where T : class, new() +``` + +#### Parameters + +`configuration` [IConfiguration](https://learn.microsoft.com/dotnet/api/microsoft.extensions.configuration.iconfiguration) + +The configuration instance + +`sectionPath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The configuration section path + +#### Returns + + T? + +The bound and validated object, or null if section doesn't exist + +#### Type Parameters + +`T` + +The type to bind to + +#### Exceptions + + [ValidationException](https://learn.microsoft.com/dotnet/api/system.componentmodel.dataannotations.validationexception) + +Thrown if DataAnnotations validation fails + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowOptions.md new file mode 100644 index 00000000..20739fa7 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowOptions.md @@ -0,0 +1,90 @@ +# Class FlowOptions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Configuration options for a single flow. + +```csharp +public class FlowOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowOptions](Flowthru.Configuration.FlowOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### Description + +Human-readable description of the Flow. + +```csharp +public string? Description { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### FactoryMethod + +The name of the static factory method (default: "Create"). + +```csharp +public string FactoryMethod { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Parameters + +Flow-specific parameters (nested configuration section). +The structure must match the Flow's parameter type. + +```csharp +public Dictionary? Parameters { get; set; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [object](https://learn.microsoft.com/dotnet/api/system.object)\>? + +### Type + +The fully-qualified type name of the flow factory class. +Must have a static Create method that accepts (catalog, parameters?). + +```csharp +public string? Type { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### Validation + +Validation configuration for this flow. + +```csharp +public FlowValidationOptions? Validation { get; set; } +``` + +#### Property Value + + [FlowValidationOptions](Flowthru.Configuration.FlowValidationOptions.md)? + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowValidationOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowValidationOptions.md new file mode 100644 index 00000000..d69c5704 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowValidationOptions.md @@ -0,0 +1,53 @@ +# Class FlowValidationOptions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Configuration options for flow validation behavior. + +```csharp +public class FlowValidationOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowValidationOptions](Flowthru.Configuration.FlowValidationOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### DefaultInspectionLevel + +Default inspection level for all Layer 0 inputs. + +```csharp +public string? DefaultInspectionLevel { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### InspectionLevels + +Per-catalog-entry inspection level overrides. +Key: catalog entry key, Value: inspection level (None, Shallow, Deep). + +```csharp +public Dictionary InspectionLevels { get; set; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [string](https://learn.microsoft.com/dotnet/api/system.string)\> + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruConfigurationOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruConfigurationOptions.md new file mode 100644 index 00000000..a42213ef --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruConfigurationOptions.md @@ -0,0 +1,231 @@ +# Class FlowthruConfigurationOptions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Options for configuring how Flowthru loads configuration files. + +```csharp +public class FlowthruConfigurationOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Flowthru uses Microsoft.Extensions.Configuration with layered configuration files. +By default, configuration is loaded in the following order (later files override earlier): +

+
  1. appsettings.json - Base configuration (required)
  2. appsettings.{Environment}.json - Environment-specific overrides (optional)
  3. appsettings.Local.json - Local/user-specific overrides (optional, gitignored)
+

+Both JSON and YAML formats are supported. YAML files follow the same pattern: +appsettings.yml, appsettings.{Environment}.yml, appsettings.Local.yml +

+ +## Properties + +### ConfigurationFileName + +The base filename for configuration files (without extension). + +```csharp +public string ConfigurationFileName { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Defaults to "appsettings". Change this to use a different naming convention +(e.g., "parameters" to match Kedro's convention). + +### ConfigurationPath + +The base path where configuration files are located. + +```csharp +public string ConfigurationPath { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Defaults to the current directory. Can be set to "conf" for Kedro-style projects +or any other directory containing configuration files. + +### EnableYamlSupport + +Whether to support YAML configuration files in addition to JSON. + +```csharp +public bool EnableYamlSupport { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +When enabled, Flowthru will load both .json and .yml/.yaml files. +Requires NetEscapades.Configuration.Yaml package. +Defaults to true for Kedro compatibility. + +### Environment + +The environment name used to load environment-specific configuration files. + +```csharp +public string? Environment { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +

+If not explicitly set, Flowthru will attempt to resolve the environment in this order: +

+
  1. Value passed to WithEnvironment()
  2. Environment variable specified by
  3. DOTNET_ENVIRONMENT environment variable
  4. ASPNETCORE_ENVIRONMENT environment variable
  5. "Production" (default)
+ +### EnvironmentVariable + +The name of the environment variable to check for environment name. + +```csharp +public string? EnvironmentVariable { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +Defaults to "FLOWTHRU_ENV". Set to null to disable environment variable resolution. +Standard .NET environment variables (DOTNET_ENVIRONMENT, ASPNETCORE_ENVIRONMENT) +are always checked as fallbacks. + +## Methods + +### WithConfigurationFileName\(string\) + +Sets the base filename for configuration files (without extension). + +```csharp +public FlowthruConfigurationOptions WithConfigurationFileName(string fileName) +``` + +#### Parameters + +`fileName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The base filename (e.g., "parameters", "config") + +#### Returns + + [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md) + +This options instance for fluent chaining + +### WithConfigurationPath\(string\) + +Sets the base path where configuration files are located. + +```csharp +public FlowthruConfigurationOptions WithConfigurationPath(string path) +``` + +#### Parameters + +`path` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The configuration directory path + +#### Returns + + [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md) + +This options instance for fluent chaining + +### WithEnvironment\(string\) + +Sets the environment name explicitly. + +```csharp +public FlowthruConfigurationOptions WithEnvironment(string environment) +``` + +#### Parameters + +`environment` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The environment name (e.g., "Development", "Production") + +#### Returns + + [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md) + +This options instance for fluent chaining + +### WithEnvironmentVariable\(string?\) + +Sets the environment variable name to check for environment resolution. + +```csharp +public FlowthruConfigurationOptions WithEnvironmentVariable(string? variableName) +``` + +#### Parameters + +`variableName` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +The environment variable name + +#### Returns + + [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md) + +This options instance for fluent chaining + +### WithYamlSupport\(bool\) + +Enables or disables YAML configuration file support. + +```csharp +public FlowthruConfigurationOptions WithYamlSupport(bool enabled = true) +``` + +#### Parameters + +`enabled` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +Whether to enable YAML support + +#### Returns + + [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md) + +This options instance for fluent chaining + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruOptions.md new file mode 100644 index 00000000..94e0a292 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruOptions.md @@ -0,0 +1,95 @@ +# Class FlowthruOptions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Root configuration options for Flowthru applications. + +```csharp +public class FlowthruOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowthruOptions](Flowthru.Configuration.FlowthruOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +This class represents the top-level "Flowthru" section in configuration files. +All Flowthru-specific configuration should be nested under this section. + +## Fields + +### SectionName + +Configuration section name in appsettings.json. + +```csharp +public const string SectionName = "Flowthru" +``` + +#### Field Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +## Properties + +### Catalog + +Data catalog configuration. + +```csharp +public CatalogOptions Catalog { get; set; } +``` + +#### Property Value + + [CatalogOptions](Flowthru.Configuration.CatalogOptions.md) + +### Flows + +Flow registration and configuration. + +```csharp +public Dictionary Flows { get; set; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [FlowOptions](Flowthru.Configuration.FlowOptions.md)\> + +### Logging + +Logging configuration (extends standard .NET logging configuration). + +```csharp +public LoggingOptions? Logging { get; set; } +``` + +#### Property Value + + [LoggingOptions](Flowthru.Configuration.LoggingOptions.md)? + +### Metadata + +Metadata collection and export configuration. + +```csharp +public MetadataOptions Metadata { get; set; } +``` + +#### Property Value + + [MetadataOptions](Flowthru.Configuration.MetadataOptions.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ICatalogFactory.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ICatalogFactory.md new file mode 100644 index 00000000..f8d7085b --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ICatalogFactory.md @@ -0,0 +1,43 @@ +# Interface ICatalogFactory + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Factory interface for creating data catalog instances from configuration. + +```csharp +public interface ICatalogFactory +``` + +## Remarks + +Implement this interface to enable configuration-based catalog construction. +The factory receives the full configuration and can use it to construct +environment-specific catalogs (e.g., local files in dev, remote DB in prod). + +## Methods + +### CreateCatalog\(CatalogOptions, IServiceProvider\) + +Creates a catalog instance based on configuration. + +```csharp +CatalogAbstract CreateCatalog(CatalogOptions options, IServiceProvider serviceProvider) +``` + +#### Parameters + +`options` [CatalogOptions](Flowthru.Configuration.CatalogOptions.md) + +Catalog configuration options + +`serviceProvider` [IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider) + +Service provider for dependency injection + +#### Returns + + [CatalogAbstract](Flowthru.Data.CatalogAbstract.md) + +The configured catalog instance + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.JsonMetadataOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.JsonMetadataOptions.md new file mode 100644 index 00000000..410b24c8 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.JsonMetadataOptions.md @@ -0,0 +1,52 @@ +# Class JsonMetadataOptions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Configuration options for JSON metadata export. + +```csharp +public class JsonMetadataOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[JsonMetadataOptions](Flowthru.Configuration.JsonMetadataOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### IncludeTypeInfo + +Whether to include full type information in the export. + +```csharp +public bool IncludeTypeInfo { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### UseCompactFormat + +Whether to use compact (minified) JSON format. + +```csharp +public bool UseCompactFormat { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.LoggingOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.LoggingOptions.md new file mode 100644 index 00000000..8cc85225 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.LoggingOptions.md @@ -0,0 +1,64 @@ +# Class LoggingOptions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Logging configuration options (extends standard .NET logging). + +```csharp +public class LoggingOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[LoggingOptions](Flowthru.Configuration.LoggingOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### EnableConsole + +Whether console logging is enabled. + +```csharp +public bool EnableConsole { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### LogLevel + +Per-category log level overrides. + +```csharp +public Dictionary LogLevel { get; set; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [string](https://learn.microsoft.com/dotnet/api/system.string)\> + +### MinimumLevel + +Minimum log level (Trace, Debug, Information, Warning, Error, Critical). + +```csharp +public string MinimumLevel { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MermaidMetadataOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MermaidMetadataOptions.md new file mode 100644 index 00000000..9cd9b6e8 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MermaidMetadataOptions.md @@ -0,0 +1,98 @@ +# Class MermaidMetadataOptions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Configuration options for Mermaid diagram export. + +```csharp +public class MermaidMetadataOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[MermaidMetadataOptions](Flowthru.Configuration.MermaidMetadataOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### ActiveDataColor + +Hex color code for active (sliced) catalog entries. + +```csharp +public string ActiveDataColor { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Color applied to data catalog entries produced by sliced nodes. +Default: #2E7D32 (Material Design green-800). + +### ActiveStepColor + +Hex color code for active (sliced) nodes. + +```csharp +public string ActiveStepColor { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Color applied to nodes that are in the execution slice. +Default: #2E7D32 (Material Design green-800). + +### Direction + +Flowchart direction (TopToBottom, LeftToRight, etc.). + +```csharp +public string Direction { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### ShowDatasetDetails + +Whether to include dataset details in nodes. + +```csharp +public bool ShowDatasetDetails { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### ShowParameters + +Whether to include parameter information in nodes. + +```csharp +public bool ShowParameters { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MetadataOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MetadataOptions.md new file mode 100644 index 00000000..1769439b --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MetadataOptions.md @@ -0,0 +1,130 @@ +# Class MetadataOptions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Configuration options for metadata collection and export. + +```csharp +public class MetadataOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[MetadataOptions](Flowthru.Configuration.MetadataOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### Enabled + +Whether metadata collection is enabled. + +```csharp +public bool Enabled { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### FilenameTemplate + +Filename template for metadata exports. + +```csharp +public string FilenameTemplate { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +

+Supports dynamic tokens that are replaced during export: +

+
  • {FlowName} - Sanitized flow name
  • {Timestamp} - Formatted timestamp (empty if disabled in Timestamp.IncludeTimestamp)
  • {SliceType} - "FromNodes", "Tags", "Mixed", or empty if unsliced
  • {FromNodes} - Comma-separated list of from-nodes
  • {ToNodes} - Comma-separated list of to-nodes
  • {FromInputs} - Comma-separated list of from-inputs
  • {OnlyNodes} - Comma-separated list of only-nodes
  • {Tags} - Comma-separated list of tags
+

+Empty tokens are automatically collapsed to prevent double-separators. +File extensions are added by individual providers (.json, .md, etc.). +

+

+Default: "dag-{FlowName}-{Timestamp}-{SliceType}" +

+

+Examples: +

+
  • Unsliced: dag-DataProcessing-20260304-153045.json
  • Sliced: dag-DataProcessing-20260304-153045-FromNodes.json
+ +### Json + +Configuration specific to the JSON metadata provider. + +```csharp +public JsonMetadataOptions? Json { get; set; } +``` + +#### Property Value + + [JsonMetadataOptions](Flowthru.Configuration.JsonMetadataOptions.md)? + +### Mermaid + +Configuration specific to the Mermaid metadata provider. + +```csharp +public MermaidMetadataOptions? Mermaid { get; set; } +``` + +#### Property Value + + [MermaidMetadataOptions](Flowthru.Configuration.MermaidMetadataOptions.md)? + +### OutputDirectory + +Directory where metadata files will be written. + +```csharp +public string OutputDirectory { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Providers + +List of metadata providers to enable (e.g., "Json", "Mermaid", "Csv"). + +```csharp +public List Providers { get; set; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +### Timestamp + +Configuration for timestamp generation in metadata filenames. + +```csharp +public TimestampConfiguration Timestamp { get; set; } +``` + +#### Property Value + + [TimestampConfiguration](Flowthru.Configuration.TimestampConfiguration.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineOptions.md new file mode 100644 index 00000000..b04d7da1 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineOptions.md @@ -0,0 +1,90 @@ +# Class PipelineOptions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Configuration options for a single pipeline. + +```csharp +public class PipelineOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PipelineOptions](Flowthru.Configuration.PipelineOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### Description + +Human-readable description of the pipeline. + +```csharp +public string? Description { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### FactoryMethod + +The name of the static factory method (default: "Create"). + +```csharp +public string FactoryMethod { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Parameters + +Pipeline-specific parameters (nested configuration section). +The structure must match the pipeline's parameter type. + +```csharp +public Dictionary? Parameters { get; set; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [object](https://learn.microsoft.com/dotnet/api/system.object)\>? + +### Type + +The fully-qualified type name of the pipeline factory class. +Must have a static Create method that accepts (catalog, parameters?). + +```csharp +public string? Type { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### Validation + +Validation configuration for this pipeline. + +```csharp +public PipelineValidationOptions? Validation { get; set; } +``` + +#### Property Value + + [PipelineValidationOptions](Flowthru.Configuration.PipelineValidationOptions.md)? + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineValidationOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineValidationOptions.md new file mode 100644 index 00000000..cd4c4980 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineValidationOptions.md @@ -0,0 +1,53 @@ +# Class PipelineValidationOptions + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Configuration options for pipeline validation behavior. + +```csharp +public class PipelineValidationOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PipelineValidationOptions](Flowthru.Configuration.PipelineValidationOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### DefaultInspectionLevel + +Default inspection level for all Layer 0 inputs. + +```csharp +public string? DefaultInspectionLevel { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### InspectionLevels + +Per-catalog-entry inspection level overrides. +Key: catalog entry key, Value: inspection level (None, Shallow, Deep). + +```csharp +public Dictionary InspectionLevels { get; set; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [string](https://learn.microsoft.com/dotnet/api/system.string)\> + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.TimestampConfiguration.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.TimestampConfiguration.md new file mode 100644 index 00000000..27d406e4 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.TimestampConfiguration.md @@ -0,0 +1,64 @@ +# Class TimestampConfiguration + +Namespace: [Flowthru.Configuration](Flowthru.Configuration.md) +Assembly: Flowthru.Core.dll + +Configuration for timestamp generation in metadata filenames. + +```csharp +public class TimestampConfiguration +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[TimestampConfiguration](Flowthru.Configuration.TimestampConfiguration.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### Format + +Timestamp format string (see .NET DateTime formatting). + +```csharp +public string Format { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### IncludeTimestamp + +Whether to include a timestamp in the filename. + +```csharp +public bool IncludeTimestamp { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### TimeZone + +Time zone for the timestamp (e.g., "UTC", "Local"). + +```csharp +public string TimeZone { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.md new file mode 100644 index 00000000..b8cd4260 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.md @@ -0,0 +1,54 @@ +# Namespace Flowthru.Configuration + +### Classes + + [CatalogOptions](Flowthru.Configuration.CatalogOptions.md) + +Configuration options for data catalog construction. + + [ConfigurationExtensions](Flowthru.Configuration.ConfigurationExtensions.md) + +Extension methods for configuration-related operations. + + [FlowOptions](Flowthru.Configuration.FlowOptions.md) + +Configuration options for a single flow. + + [FlowValidationOptions](Flowthru.Configuration.FlowValidationOptions.md) + +Configuration options for flow validation behavior. + + [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md) + +Options for configuring how Flowthru loads configuration files. + + [FlowthruOptions](Flowthru.Configuration.FlowthruOptions.md) + +Root configuration options for Flowthru applications. + + [JsonMetadataOptions](Flowthru.Configuration.JsonMetadataOptions.md) + +Configuration options for JSON metadata export. + + [LoggingOptions](Flowthru.Configuration.LoggingOptions.md) + +Logging configuration options (extends standard .NET logging). + + [MermaidMetadataOptions](Flowthru.Configuration.MermaidMetadataOptions.md) + +Configuration options for Mermaid diagram export. + + [MetadataOptions](Flowthru.Configuration.MetadataOptions.md) + +Configuration options for metadata collection and export. + + [TimestampConfiguration](Flowthru.Configuration.TimestampConfiguration.md) + +Configuration for timestamp generation in metadata filenames. + +### Interfaces + + [ICatalogFactory](Flowthru.Configuration.ICatalogFactory.md) + +Factory interface for creating data catalog instances from configuration. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.StorageTraits.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.StorageTraits.md new file mode 100644 index 00000000..39753a89 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.StorageTraits.md @@ -0,0 +1,224 @@ +# Class StorageTraits + +Namespace: [Flowthru.Data.Capabilities](Flowthru.Data.Capabilities.md) +Assembly: Flowthru.Core.dll + +Describes the structural constraints and capabilities of a storage implementation. +Defaults represent filesystem-file baseline behavior. + +```csharp +public record StorageTraits : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Design Philosophy: +

+

+A filesystem file is the median storage mechanism — the "zero" from which we measure deviations. +A constraint narrows from this baseline (e.g., read-only, non-persistent, non-inspectable). +A capability widens beyond it (e.g., streamable, appendable, transactional). +

+

+Constraint Examples: +

+
  • Read-only sources: HTTP GET endpoints, Excel files, database views — set CanWrite = false
  • Write-only sinks: Logging endpoints, append-only audit tables — set CanRead = false
  • Non-inspectable: Remote sources that can't be sampled cheaply — set CanInspect = false
  • Non-persistent: In-memory caches, temporary buffers — set IsPersistent = false
  • Network-dependent: Remote databases, S3, HTTP — set RequiresNetwork = true
+

+Capability Examples: +

+
  • Streamable: CSV files, database queries, Parquet — set CanStream = true
  • Appendable: Log files, Spark SaveMode.Append, append-only tables — set CanAppend = true
  • Transactional: Database writes, ACID-compliant stores — set IsTransactional = true
+

+Two-Level Constraint Model: +

+

+Traits are declared at two levels: +

+
  • Adapter level: The adapter author declares what the storage medium intrinsically supports. +These are structural truths (e.g., an HTTP GET endpoint cannot write).
  • Catalog level: The pipeline author can further constrain an adapter using Item.Constrain(). +Constraints can only tighten, never loosen (one-way ratchet).
+

+Usage in Adapters: +

+
public sealed class EFCoreStorageAdapter<T> : IStorageAdapter<IEnumerable<T>>
+{
+    public StorageTraits Traits { get; }
+
+    public EFCoreStorageAdapter(DbContext context, bool readOnly = false)
+    {
+        Traits = new StorageTraits
+        {
+            CanWrite = !readOnly,
+            RequiresNetwork = true,
+            IsTransactional = true,
+            CanStream = true,
+        };
+    }
+}
+

+Usage in Catalogs: +

+
public IItem<IEnumerable<Company>> ReferenceData =>
+    GetOrCreateEntry(() => Items.Enumerable.Csv<Company>(
+        "ref_data", $"{_basePath}/reference.csv")
+        .Constrain(t => t with { CanWrite = false }));
+ +## Properties + +### CanAppend + +Can data be appended without replacing existing data? + +```csharp +public bool CanAppend { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Default: false (filesystem file writes typically overwrite). +Set to true for append-only logs, Spark SaveMode.Append, incremental tables. + +### CanInspect + +Can the source be inspected for pre-flight validation? + +```csharp +public bool CanInspect { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Default: true (filesystem files can be sampled). +Set to false for sources that are expensive to validate (remote HTTP, distributed Spark). + +### CanRead + +Can data be read from this source? + +```csharp +public bool CanRead { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Default: true (filesystem files are readable). +Set to false for write-only sinks (logging endpoints, audit tables). + +### CanStream + +Can data be lazily streamed without full materialization? + +```csharp +public bool CanStream { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Default: false (filesystem file I/O typically buffers). +Set to true for CSV streaming, database cursors, Parquet row groups. +Enables memory-efficient processing of large datasets. + +### CanWrite + +Can data be written to this source? + +```csharp +public bool CanWrite { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Default: true (filesystem files are writable). +Set to false for read-only sources (HTTP GET, Excel files, database views). + +### IsPersistent + +Does data survive across pipeline runs? + +```csharp +public bool IsPersistent { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Default: true (filesystem files persist). +Set to false for in-memory caches, temporary buffers, or transient state. + +### IsTransactional + +Are writes atomic (all-or-nothing)? + +```csharp +public bool IsTransactional { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Default: false (filesystem file writes are not ACID). +Set to true for database transactions, ACID-compliant stores. + +### RequiresNetwork + +Does this storage require network access? + +```csharp +public bool RequiresNetwork { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Default: false (filesystem files are local). +Set to true for remote databases, S3, HTTP endpoints. +Used for pre-flight validation in offline/CI environments. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.md new file mode 100644 index 00000000..be13d989 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.md @@ -0,0 +1,9 @@ +# Namespace Flowthru.Data.Capabilities + +### Classes + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +Describes the structural constraints and capabilities of a storage implementation. +Defaults represent filesystem-file baseline behavior. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogAbstract.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogAbstract.md new file mode 100644 index 00000000..94b36f3d --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogAbstract.md @@ -0,0 +1,262 @@ +# Class CatalogAbstract + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Base class for strongly-typed catalog implementations with automatic property caching. + +```csharp +public abstract class CatalogAbstract +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[CatalogAbstract](Flowthru.Data.CatalogAbstract.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Problem Solved: +Expression-bodied properties (Property => new Item()) create new instances on each access, +breaking DAG dependency resolution which relies on object identity. +

+

+Solution: +Uses reflection to: +1. Discover all IItem properties on derived classes +2. Create backing fields to cache instances +3. Intercept property getters to return cached instances +

+

+Usage Pattern: +

public class MyCatalog : DataCatalogBase
+{
+    public MyCatalog(string basePath = "Data") : base()
+    {
+        BasePath = basePath;
+        InitializeCatalogProperties();
+    }
+
+    protected string BasePath { get; }
+
+    // Declare once - automatically cached!
+    public IItem<IEnumerable<MyData>> MyData =>
+        CreateItem(() => new CsvCatalogItem<MyData>("my_data", $"{BasePath}/data.csv"));
+}
+

+

+Key Benefits: +- Declare catalog items ONCE (no redundant constructor code) +- Automatic instance caching (object identity preserved) +- Type-safe (compile-time checks) +- Zero runtime overhead after first access (cached delegates) +

+ +## Constructors + +### CatalogAbstract\(string?\) + +```csharp +protected CatalogAbstract(string? catalogLabel = null) +``` + +#### Parameters + +`catalogLabel` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Optional display label for this catalog instance. When omitted, defaults to the +concrete class name via GetType().Name. + +## Properties + +### CatalogLabel + +The display label used to identify this catalog instance in flow metadata. +Defaults to the concrete class name when not specified. + +```csharp +public string CatalogLabel { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Pass an explicit label when constructing multiple instances of the same catalog type +in a single flow (e.g., per-partition or per-shard catalogs) so their items +receive distinct qualified identifiers in the DAG: CatalogLabel.ItemLabel. + +### Services + +Optional service provider for dependency injection into catalog items. + +```csharp +public IServiceProvider? Services { get; set; } +``` + +#### Property Value + + [IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider)? + +#### Remarks + +Set by the service layer before flow execution to enable catalog +items to resolve services (e.g., database connections, HTTP clients). + +## Methods + +### ClearCache\(\) + +Clears the property cache. Use with caution! + +```csharp +protected void ClearCache() +``` + +#### Remarks + +

+Warning: Clearing the cache after flow construction will break +DAG dependencies since new instances will be created on next access. +

+

+Use Case: Primarily for testing scenarios where you need to reset +catalog state between test runs. +

+ +### CreateItem\(Func\>, string\) + +Gets or creates a unified catalog item, caching it for subsequent accesses. + +```csharp +protected IItem CreateItem(Func> factory, string propertyName = "") +``` + +#### Parameters + +`factory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-1)<[IItem](Flowthru.Data.IItem\-1.md)\> + +Factory function to create the item on first access + +`propertyName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Auto-populated by compiler with calling property name + +#### Returns + + [IItem](Flowthru.Data.IItem\-1.md) + +Cached catalog item instance + +#### Type Parameters + +`T` + +The data type stored in this catalog item. +For singletons: Use T directly (e.g., LinearRegressionModel) +For collections: Use IEnumerable<T> (e.g., IEnumerable<FeatureRow>) + +#### Remarks + +

+Unified API (v0.5.0): This single method replaces GetOrCreateObject +and GetOrCreateDataset. Cardinality is determined by the type parameter T. +

+

+Usage Examples: +

// Singleton object
+public IItem<LinearRegressionModel> Model =>
+    GetOrCreateItem(() => Items.Single.Memory<LinearRegressionModel>("model"));
+
+// Collection
+public IItem<IEnumerable<FeatureRow>> Features =>
+    GetOrCreateItem(() => Items.Enumerable.Csv<FeatureRow>("features", "data.csv"));
+

+ +### CreateItem\(Func\>, string\) + +Gets or creates a unified catalog item with service provider access. + +```csharp +protected IItem CreateItem(Func> factory, string propertyName = "") +``` + +#### Parameters + +`factory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider)?, [IItem](Flowthru.Data.IItem\-1.md)\> + +Factory function that receives service provider + +`propertyName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Auto-populated by compiler with calling property name + +#### Returns + + [IItem](Flowthru.Data.IItem\-1.md) + +Cached catalog item instance + +#### Type Parameters + +`T` + +The data type (singleton or collection) + +### GetAllItems\(\) + +Gets all cached catalog items. + +```csharp +protected IEnumerable GetAllItems() +``` + +#### Returns + + [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)<[IItem](Flowthru.Data.IItem.md)\> + +Enumerable of all initialized catalog items + +#### Remarks + +Useful for diagnostic purposes or when you need to iterate over all items +(e.g., for validation, cleanup, or reporting). + +### InitializeCatalogProperties\(\) + +Initializes all catalog item properties by invoking their getters once. + +```csharp +protected void InitializeCatalogProperties() +``` + +#### Remarks + +

+Purpose: Eager initialization ensures all items are cached +before flow construction begins, preventing any potential race conditions +or unexpected lazy initialization behavior. +

+

+When to Call: At the end of the derived catalog's constructor, +after all configuration properties (like BasePath) are set. +

+

+How It Works: +Uses reflection to find all public instance properties that return IItem, +then invokes each getter once to populate the cache. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntries.Single.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntries.Single.md new file mode 100644 index 00000000..427eb252 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntries.Single.md @@ -0,0 +1,228 @@ +# Class Items.Single + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Factory methods for single (non-collection) values. + +```csharp +public static class Items.Single +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Items.Single](Flowthru.Data.Items.Single.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+These methods create catalog entries for single objects rather than collections. +

+

+Use Cases: +

+
  • Model files (ML models, configuration objects)
  • Metrics and evaluation results (single JSON objects)
  • Text reports (Markdown, plain text)
  • Binary files (images, PDFs)
  • Side-effect-only nodes (null/void semantics)
+ +## Methods + +### Binary\(string, string\) + +Creates a binary file catalog entry. + +```csharp +public static Item Binary(string label, string filePath) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to binary file (.png, .jpg, .pdf, etc.) + +#### Returns + + [Item](Flowthru.Data.Item\-1.md)<[byte](https://learn.microsoft.com/dotnet/api/system.byte)\[\]\> + +Catalog entry for binary file with byte array content + +#### Remarks + +

+Use Case: Images (PNG, JPG), PDFs, any binary data +

+

+Implementation: Reads entire file as byte array. +

+

+Storage Traits: All traits use filesystem baseline defaults +

+ +### Json\(string, string\) + +Creates a JSON file catalog entry for a single object (non-collection). + +```csharp +public static Item Json(string label, string filePath) where T : IStructuredSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to JSON file + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +Catalog entry for singleton JSON object + +#### Type Parameters + +`T` + +Object type (must be structured-serializable) + +#### Remarks + +

+Use Case: Model files, configuration objects, metrics, single records +

+

+Serialization: Single JSON object (not wrapped in array) +

+

+Implementation: Uses SingletonJsonStorageAdapter which bypasses +format/container composition for direct object serialization. +

+ +### Memory\(string\) + +Creates a memory catalog entry for a single object (non-collection). + +```csharp +public static Item Memory(string label) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +Catalog entry for in-memory singleton + +#### Type Parameters + +`T` + +Object type + +#### Remarks + +

+Use Case: Models, charts, computed metrics that stay in memory +

+

+Examples: +

+
  • ML models (LinearRegressionModel)
  • Charts (GenericChart from Plotly.NET)
  • Metrics objects (ModelMetrics, CrossValidationResults)
  • Any singleton data that doesn't need persistence
+ +### Null\(string\) + +Creates a null catalog entry for side-effect-only nodes. + +```csharp +public static Item Null(string label) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +Catalog entry for void/no-data semantics + +#### Type Parameters + +`T` + +The data type (typically NoData) + +#### Remarks + +

+Use Case: Steps that perform side effects (logging, visualization) without producing meaningful data +

+

+Implementation: Uses NullStorageAdapter which performs no I/O operations. +

+

+Storage Traits: +

+
  • CanWrite: false (Save is a no-op)
  • CanRead: false (Load throws NotSupportedException)
+ +### Text\(string, string\) + +Creates a plain text file catalog entry. + +```csharp +public static Item Text(string label, string filePath) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to text file (.txt, .md, etc.) + +#### Returns + + [Item](Flowthru.Data.Item\-1.md)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +Catalog entry for text file with string content + +#### Remarks + +

+Use Case: Markdown reports, plain text logs, configuration files +

+

+Implementation: Reads entire file as single string. +

+

+Storage Traits: All traits use filesystem baseline defaults +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntries.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntries.md new file mode 100644 index 00000000..a8cd5d7c --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntries.md @@ -0,0 +1,136 @@ +# Class Items + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Static factory methods for creating catalog entries with common configurations. + +```csharp +public static class Items +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Items](Flowthru.Data.Items.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
// Tier 1: No annotations - property names match external field names
+public record SimpleSchema(
+    int Id,           // Looks for "Id" in CSV/Excel/JSON
+    string Name       // Looks for "Name" in CSV/Excel/JSON
+) : IFlatSchema, ITextSerializable;
+
+var simple = Items.Enumerable.Csv<SimpleSchema>("data", "data.csv");
+
+// Tier 2: Explicit annotations - handle naming mismatches
+public record ShuttleSchema(
+    [SerializedLabel("id")]
+    string Id,
+
+    [SerializedLabel("shuttle_location")]        // snake_case in file
+    string ShuttleLocation,
+
+    [SerializedLabel("d_check_complete")]        // snake_case in file
+    bool DCheckComplete,
+
+    [SerializedLabel("Company ID")]              // space-separated in file
+    int CompanyId
+) : IFlatSchema, ITextSerializable;
+
+var shuttles = Items.Enumerable.Excel<ShuttleSchema>(
+    "shuttles",
+    "data/shuttles.xlsx",
+    "Sheet1"
+);
+
+// Same schema works across all formats
+var csv = Items.Enumerable.Csv<ShuttleSchema>("shuttles", "data/shuttles.csv");
+var json = Items.Enumerable.Json<ShuttleSchema>("shuttles", "data/shuttles.json");
+ +## Remarks + +

+Design Pattern: Static factory methods that compose storage adapters +from medium + format + container layers. +

+

+Discoverability: All factory methods are in one place with IntelliSense support. +

+

+Type Safety: Generic constraints enforce schema compatibility at compile-time. +

+

+Field Name Mapping with SerializedLabel: +

+

+Use the SerializedLabelAttribute to map C# property names to external field names +when they differ. This works uniformly across CSV, Excel, JSON, and most formats. +

+ +## Properties + +### Enumerable + +Factory methods for catalog entries. + +```csharp +public static EnumerableItems Enumerable { get; } +``` + +#### Property Value + + [EnumerableItems](Flowthru.Data.EnumerableItems.md) + +## Methods + +### Null\(string\) + +Creates a null catalog entry for side-effect-only nodes. + +```csharp +public static Item Null(string label) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +Catalog entry for void/no-data semantics + +#### Type Parameters + +`T` + +The data type (typically NoData) + +#### Remarks + +

+Use Case: Steps that perform side effects (logging, visualization) without producing meaningful data +

+

+Implementation: Uses NullStorageAdapter which performs no I/O operations. +

+

+Storage Traits: +

+
  • CanWrite: false (Save is a no-op)
  • CanRead: false (Load throws NotSupportedException)
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntry-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntry-1.md new file mode 100644 index 00000000..4c3309d5 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntry-1.md @@ -0,0 +1,347 @@ +# Class Item + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Standard catalog entry implementation that delegates to a storage adapter. + +```csharp +public sealed class Item : IItem, IItem +``` + +#### Type Parameters + +`T` + +The data type (container with rows) + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Item](Flowthru.Data.Item\-1.md) + +#### Implements + +[IItem](Flowthru.Data.IItem\-1.md), +[IItem](Flowthru.Data.IItem.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Delegation Pattern: +

+

+This class is a thin wrapper that delegates all operations to an . +The storage adapter handles the actual I/O logic, while this class provides: +- IItem interface implementation +- Identity for DAG dependency resolution (via Key) +- Type erasure for pipeline heterogeneous collections +

+

+Construction: +

+

+Typically created via static factory methods in : +

+
var entry = Items.Csv<CompanySchema>("companies", "data.csv");
+// Returns: IItem<IEnumerable<CompanySchema>>
+

+Composition vs Inheritance: +

+

+Previous design: Inheritance hierarchy (CsvItem, JsonItem, etc.) +New design: Single class + composed storage adapter +

+

+Benefits: +- No class explosion for format × container combinations +- Custom storage via IStorageAdapter implementation +- Clear separation of concerns +

+

+Capability Forwarding: +

+

+The underlying storage adapter provides inspection methods, which this catalog +entry automatically forwards. All storage adapters are required to implement inspection. +

+ +## Constructors + +### Item\(string, IStorageAdapter\) + +Creates a new catalog entry with the specified key and storage adapter. + +```csharp +public Item(string label, IStorageAdapter storage) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this catalog entry + +`storage` [IStorageAdapter](Flowthru.Data.Storage.IStorageAdapter\-1.md) + +Storage adapter that handles I/O operations + +## Properties + +### DataType + +The runtime type of data stored in this catalog entry. +For singletons: Returns typeof(T). +For collections: Returns typeof(IEnumerable<T>). + +```csharp +public Type DataType { get; } +``` + +#### Property Value + + [Type](https://learn.microsoft.com/dotnet/api/system.type) + +### Label + +Unique label identifying this catalog entry within the data catalog. + +```csharp +public string Label { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### OwningCatalogLabel + +The label of the -derived class that created +this entry. Set automatically by GetOrCreateEntry; null for entries created outside +a catalog or by custom implementations. + +```csharp +public string? OwningCatalogLabel { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +Used by the metadata layer to produce fully-qualified entry identifiers in the form +CatalogLabel.EntryLabel. First-write-wins: cross-catalog shared entries retain +the label of the catalog that originally created them. + +### PreferredInspectionLevel + +Gets the preferred inspection level for this catalog entry. + +```csharp +public InspectionLevel? PreferredInspectionLevel { get; } +``` + +#### Property Value + + [InspectionLevel](Flowthru.Data.Validation.InspectionLevel.md)? + +### Traits + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +## Methods + +### Constrain\(Func\) + +```csharp +public Item Constrain(Func constraintFn) +``` + +#### Parameters + +`constraintFn` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md), [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md)\> + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +### Exists\(\) + +Checks if data exists at this catalog entry location. +Returns an effect that can fail. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +### GetCountAsync\(\) + +Gets the count of items in this catalog entry. +For collections (IEnumerable<T>), returns the enumerable count. +For singletons, returns 1 if exists, 0 otherwise. + +```csharp +public FlowIO GetCountAsync() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[int](https://learn.microsoft.com/dotnet/api/system.int32)\> + +### InspectDeep\(\) + +Performs deep validation of this catalog entry. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +Forwards the call directly to the underlying storage adapter. +All storage adapters must implement inspection. + +### InspectShallow\(int\) + +Performs shallow validation of this catalog entry. + +```csharp +public FlowIO InspectShallow(int sampleSize = 100) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +Forwards the call directly to the underlying storage adapter. +All storage adapters must implement inspection. + +### Load\(\) + +Load data as an effect (can fail, is async, can be cancelled). +Returns T directly, which may itself be an IEnumerable or Seq. + +```csharp +public FlowIO Load() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +### LoadUntyped\(\) + +Loads data from the catalog entry as an untyped object. +Returns an effect that can fail. +The returned type matches the DataType property. + +```csharp +public FlowIO LoadUntyped() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[object](https://learn.microsoft.com/dotnet/api/system.object)\> + +### Save\(T\) + +Save data as an effect. +Accepts T directly, which may itself be an IEnumerable or Seq. + +```csharp +public FlowIO Save(T data) +``` + +#### Parameters + +`data` T + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +### SaveUntyped\(object\) + +Saves untyped data to the catalog entry. +Returns an effect that can fail. +The data type must be compatible with the DataType property. + +```csharp +public FlowIO SaveUntyped(object data) +``` + +#### Parameters + +`data` [object](https://learn.microsoft.com/dotnet/api/system.object) + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +### WithInspectionLevel\(InspectionLevel\) + +Sets the preferred inspection level for this catalog entry. + +```csharp +public Item WithInspectionLevel(InspectionLevel level) +``` + +#### Parameters + +`level` [InspectionLevel](Flowthru.Data.Validation.InspectionLevel.md) + +The inspection level to use + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +This catalog entry for method chaining + +#### Remarks + +

+Used to configure how this entry should be validated before pipeline execution. +

+

+Example: +

+
var entry = Items.Csv<Company>("companies", "data.csv")
+    .WithInspectionLevel(InspectionLevel.Deep);
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.DataCatalogBase.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.DataCatalogBase.md new file mode 100644 index 00000000..5b2f24d9 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.DataCatalogBase.md @@ -0,0 +1,262 @@ +# Class DataCatalogBase + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Base class for strongly-typed catalog implementations with automatic property caching. + +```csharp +public abstract class DataCatalogBase +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[DataCatalogBase](Flowthru.Data.DataCatalogBase.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Problem Solved: +Expression-bodied properties (Property => new Entry()) create new instances on each access, +breaking DAG dependency resolution which relies on object identity. +

+

+Solution: +Uses reflection to: +1. Discover all IItem properties on derived classes +2. Create backing fields to cache instances +3. Intercept property getters to return cached instances +

+

+Usage Pattern: +

public class MyCatalog : DataCatalogBase
+{
+    public MyCatalog(string basePath = "Data") : base()
+    {
+        BasePath = basePath;
+        InitializeCatalogProperties();
+    }
+
+    protected string BasePath { get; }
+
+    // Declare once - automatically cached!
+    public IItem<IEnumerable<MyData>> MyData =>
+        GetOrCreateEntry(() => new CsvItem<MyData>("my_data", $"{BasePath}/data.csv"));
+}
+

+

+Key Benefits: +- Declare catalog entries ONCE (no redundant constructor code) +- Automatic instance caching (object identity preserved) +- Type-safe (compile-time checks) +- Zero runtime overhead after first access (cached delegates) +

+ +## Constructors + +### DataCatalogBase\(string?\) + +```csharp +protected DataCatalogBase(string? catalogLabel = null) +``` + +#### Parameters + +`catalogLabel` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Optional display label for this catalog instance. When omitted, defaults to the +concrete class name via GetType().Name. + +## Properties + +### CatalogLabel + +The display label used to identify this catalog instance in pipeline metadata. +Defaults to the concrete class name when not specified. + +```csharp +public string CatalogLabel { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Pass an explicit label when constructing multiple instances of the same catalog type +in a single pipeline (e.g., per-partition or per-shard catalogs) so their entries +receive distinct qualified identifiers in the DAG: CatalogLabel.EntryLabel. + +### Services + +Optional service provider for dependency injection into catalog entries. + +```csharp +public IServiceProvider? Services { get; set; } +``` + +#### Property Value + + [IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider)? + +#### Remarks + +Set by the service layer before pipeline execution to enable catalog +entries to resolve services (e.g., database connections, HTTP clients). + +## Methods + +### ClearCache\(\) + +Clears the property cache. Use with caution! + +```csharp +protected void ClearCache() +``` + +#### Remarks + +

+Warning: Clearing the cache after pipeline construction will break +DAG dependencies since new instances will be created on next access. +

+

+Use Case: Primarily for testing scenarios where you need to reset +catalog state between test runs. +

+ +### GetAllEntries\(\) + +Gets all cached catalog entries. + +```csharp +protected IEnumerable GetAllEntries() +``` + +#### Returns + + [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)<[IItem](Flowthru.Data.IItem.md)\> + +Enumerable of all initialized catalog entries + +#### Remarks + +Useful for diagnostic purposes or when you need to iterate over all entries +(e.g., for validation, cleanup, or reporting). + +### GetOrCreateEntry\(Func\>, string\) + +Gets or creates a unified catalog entry, caching it for subsequent accesses. + +```csharp +protected IItem GetOrCreateEntry(Func> factory, string propertyName = "") +``` + +#### Parameters + +`factory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-1)<[IItem](Flowthru.Data.IItem\-1.md)\> + +Factory function to create the entry on first access + +`propertyName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Auto-populated by compiler with calling property name + +#### Returns + + [IItem](Flowthru.Data.IItem\-1.md) + +Cached catalog entry instance + +#### Type Parameters + +`T` + +The data type stored in this catalog entry. +For singletons: Use T directly (e.g., LinearRegressionModel) +For collections: Use IEnumerable<T> (e.g., IEnumerable<FeatureRow>) + +#### Remarks + +

+Unified API (v0.5.0): This single method replaces GetOrCreateObject +and GetOrCreateDataset. Cardinality is determined by the type parameter T. +

+

+Usage Examples: +

// Singleton object
+public IItem<LinearRegressionModel> Model =>
+    GetOrCreateEntry(() => Items.Single.Memory<LinearRegressionModel>("model"));
+
+// Collection
+public IItem<IEnumerable<FeatureRow>> Features =>
+    GetOrCreateEntry(() => Items.Enumerable.Csv<FeatureRow>("features", "data.csv"));
+

+ +### GetOrCreateEntry\(Func\>, string\) + +Gets or creates a unified catalog entry with service provider access. + +```csharp +protected IItem GetOrCreateEntry(Func> factory, string propertyName = "") +``` + +#### Parameters + +`factory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider)?, [IItem](Flowthru.Data.IItem\-1.md)\> + +Factory function that receives service provider + +`propertyName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Auto-populated by compiler with calling property name + +#### Returns + + [IItem](Flowthru.Data.IItem\-1.md) + +Cached catalog entry instance + +#### Type Parameters + +`T` + +The data type (singleton or collection) + +### InitializeCatalogProperties\(\) + +Initializes all catalog entry properties by invoking their getters once. + +```csharp +protected void InitializeCatalogProperties() +``` + +#### Remarks + +

+Purpose: Eager initialization ensures all entries are cached +before pipeline construction begins, preventing any potential race conditions +or unexpected lazy initialization behavior. +

+

+When to Call: At the end of the derived catalog's constructor, +after all configuration properties (like BasePath) are set. +

+

+How It Works: +Uses reflection to find all public instance properties that return IItem, +then invokes each getter once to populate the cache. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.EnumerableCatalogEntries.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.EnumerableCatalogEntries.md new file mode 100644 index 00000000..8515785f --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.EnumerableCatalogEntries.md @@ -0,0 +1,126 @@ +# Class EnumerableItems + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Extension point for factory methods. + +```csharp +public sealed class EnumerableItems +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EnumerableItems](Flowthru.Data.EnumerableItems.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+IEnumerable<T> is the standard .NET collection interface. +

+

+Characteristics: +

+
  • Lazy evaluation: LINQ queries deferred until enumeration
  • Re-enumerable: Can cause side effects (multiple DB hits, file reads)
  • Mutable: Underlying collection can be modified
  • Standard .NET: Works with all .NET libraries
+

+Use Cases: +

+
  • Standard data processing pipelines (90% of cases)
  • Interop with .NET libraries expecting IEnumerable
  • LINQ query composition
  • Large datasets where you'll enumerate only once
+

+Format-specific factory methods (CSV, Parquet, Excel) are provided as extension +methods by their respective packages. Add extension methods to this type to +register new formats. +

+ +## Methods + +### Json\(string, string\) + +Creates a JSON file catalog entry with IEnumerable container for collections. + +```csharp +public Item> Json(string label, string filePath) where TRow : notnull, IStructuredSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to JSON file + +#### Returns + + [Item](Flowthru.Data.Item\-1.md)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Catalog entry with file + JSON + IEnumerable composition + +#### Type Parameters + +`TRow` + +Row schema type (must be structured-serializable) + +#### Remarks + +

+Requirements: +

+
  • TRow must implement IStructuredSerializable
  • TRow supports both flat and nested schemas
+

+Supports: +

+
  • Traditional schemas with parameterless constructors
  • Modern schemas with required properties (C# 11+)
  • Positional records with primary constructors
+

+Serialization: JSON array format for collections +

+ +### Memory\(string\) + +Creates an in-memory transient catalog entry with IEnumerable container. + +```csharp +public Item> Memory(string label) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +#### Returns + + [Item](Flowthru.Data.Item\-1.md)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Catalog entry with memory storage (no serialization) + +#### Type Parameters + +`TRow` + +Row schema type + +#### Remarks + +

+Use Case: Intermediate pipeline data that doesn't need persistence +

+

+Storage Traits: +

+
  • IsPersistent: false (data lost when process ends)
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.EnumerableItems.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.EnumerableItems.md new file mode 100644 index 00000000..89c2bb04 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.EnumerableItems.md @@ -0,0 +1,126 @@ +# Class EnumerableItems + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Extension point for factory methods. + +```csharp +public sealed class EnumerableItems +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EnumerableItems](Flowthru.Data.EnumerableItems.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+IEnumerable<T> is the standard .NET collection interface. +

+

+Characteristics: +

+
  • Lazy evaluation: LINQ queries deferred until enumeration
  • Re-enumerable: Can cause side effects (multiple DB hits, file reads)
  • Mutable: Underlying collection can be modified
  • Standard .NET: Works with all .NET libraries
+

+Use Cases: +

+
  • Standard data processing flows (90% of cases)
  • Interop with .NET libraries expecting IEnumerable
  • LINQ query composition
  • Large datasets where you'll enumerate only once
+

+Format-specific factory methods (CSV, Parquet, Excel) are provided as extension +methods by their respective packages. Add extension methods to this type to +register new formats. +

+ +## Methods + +### Json\(string, string\) + +Creates a JSON file catalog item with IEnumerable container for collections. + +```csharp +public Item> Json(string label, string filePath) where TRow : notnull, IStructuredSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to JSON file + +#### Returns + + [Item](Flowthru.Data.Item\-1.md)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Catalog item with file + JSON + IEnumerable composition + +#### Type Parameters + +`TRow` + +Row schema type (must be structured-serializable) + +#### Remarks + +

+Requirements: +

+
  • TRow must implement IStructuredSerializable
  • TRow supports both flat and nested schemas
+

+Supports: +

+
  • Traditional schemas with parameterless constructors
  • Modern schemas with required properties (C# 11+)
  • Positional records with primary constructors
+

+Serialization: JSON array format for collections +

+ +### Memory\(string\) + +Creates an in-memory transient catalog item with IEnumerable container. + +```csharp +public Item> Memory(string label) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +#### Returns + + [Item](Flowthru.Data.Item\-1.md)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Catalog item with memory storage (no serialization) + +#### Type Parameters + +`TRow` + +Row schema type + +#### Remarks + +

+Use Case: Intermediate flow data that doesn't need persistence +

+

+Storage Traits: +

+
  • IsPersistent: false (data lost when process ends)
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.ICatalogEntry-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.ICatalogEntry-1.md new file mode 100644 index 00000000..f7d70362 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.ICatalogEntry-1.md @@ -0,0 +1,74 @@ +# Interface IItem + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Unified catalog entry with cardinality encoded in the type parameter. + +```csharp +public interface IItem : IItem +``` + +#### Type Parameters + +`T` + +The data type stored in this catalog entry. +Cardinality is determined by T itself: +- For singletons: Use T directly (e.g., LinearRegressionModel, ModelMetrics) +- For collections: Use IEnumerable<T> (e.g., IEnumerable<FeatureRow>) + +#### Implements + +[IItem](Flowthru.Data.IItem.md) + +## Remarks + +

+Unified Design: This single interface replaces the previous dual-interface +system (ICatalogObject/ICatalogDataset). Cardinality is now purely a type-level concern. +

+

+Type Alignment: Step TInput/TOutput types should directly match catalog entry +T types, eliminating the need for wrapping/unwrapping ceremony. +

+

+Effect Types: All operations return FlowIO<T> - an effect that represents +an async computation that can fail. This provides: +- Explicit error handling +- Cancellation support +- Functional composition +

+ +## Methods + +### Load\(\) + +Load data as an effect (can fail, is async, can be cancelled). +Returns T directly, which may itself be an IEnumerable or Seq. + +```csharp +FlowIO Load() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +### Save\(T\) + +Save data as an effect. +Accepts T directly, which may itself be an IEnumerable or Seq. + +```csharp +FlowIO Save(T data) +``` + +#### Parameters + +`data` T + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.ICatalogEntry.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.ICatalogEntry.md new file mode 100644 index 00000000..14d0a9fa --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.ICatalogEntry.md @@ -0,0 +1,172 @@ +# Interface IItem + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Non-generic base interface for catalog entries. +Provides untyped operations for internal use by the pipeline executor and mapping layer. + +```csharp +public interface IItem +``` + +## Remarks + +This interface enables the pipeline to work with catalog entries +without knowing their specific type parameter at compile-time. + +## Properties + +### DataType + +The runtime type of data stored in this catalog entry. +For singletons: Returns typeof(T). +For collections: Returns typeof(IEnumerable<T>). + +```csharp +Type DataType { get; } +``` + +#### Property Value + + [Type](https://learn.microsoft.com/dotnet/api/system.type) + +### Label + +Unique label identifying this catalog entry within the data catalog. + +```csharp +string Label { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### OwningCatalogLabel + +The label of the -derived class that created +this entry. Set automatically by GetOrCreateEntry; null for entries created outside +a catalog or by custom implementations. + +```csharp +string? OwningCatalogLabel { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +Used by the metadata layer to produce fully-qualified entry identifiers in the form +CatalogLabel.EntryLabel. First-write-wins: cross-catalog shared entries retain +the label of the catalog that originally created them. + +### PreferredInspectionLevel + +Gets the preferred inspection level for this catalog entry. + +```csharp +InspectionLevel? PreferredInspectionLevel { get; } +``` + +#### Property Value + + [InspectionLevel](Flowthru.Data.Validation.InspectionLevel.md)? + +## Methods + +### Exists\(\) + +Checks if data exists at this catalog entry location. +Returns an effect that can fail. + +```csharp +FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +### GetCountAsync\(\) + +Gets the count of items in this catalog entry. +For collections (IEnumerable<T>), returns the enumerable count. +For singletons, returns 1 if exists, 0 otherwise. + +```csharp +FlowIO GetCountAsync() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[int](https://learn.microsoft.com/dotnet/api/system.int32)\> + +### InspectDeep\(\) + +Performs deep validation of this catalog entry. + +```csharp +FlowIO InspectDeep() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +### InspectShallow\(int\) + +Performs shallow validation of this catalog entry. + +```csharp +FlowIO InspectShallow(int sampleSize = 100) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +### LoadUntyped\(\) + +Loads data from the catalog entry as an untyped object. +Returns an effect that can fail. +The returned type matches the DataType property. + +```csharp +FlowIO LoadUntyped() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[object](https://learn.microsoft.com/dotnet/api/system.object)\> + +### SaveUntyped\(object\) + +Saves untyped data to the catalog entry. +Returns an effect that can fail. +The data type must be compatible with the DataType property. + +```csharp +FlowIO SaveUntyped(object data) +``` + +#### Parameters + +`data` [object](https://learn.microsoft.com/dotnet/api/system.object) + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.IItem-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.IItem-1.md new file mode 100644 index 00000000..7a760396 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.IItem-1.md @@ -0,0 +1,74 @@ +# Interface IItem + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Unified catalog item with cardinality encoded in the type parameter. + +```csharp +public interface IItem : IItem +``` + +#### Type Parameters + +`T` + +The data type stored in this catalog item. +Cardinality is determined by T itself: +- For singletons: Use T directly (e.g., LinearRegressionModel, ModelMetrics) +- For collections: Use IEnumerable<T> (e.g., IEnumerable<FeatureRow>) + +#### Implements + +[IItem](Flowthru.Data.IItem.md) + +## Remarks + +

+Unified Design: This single interface replaces the previous dual-interface +system (ICatalogObject/ICatalogDataset). Cardinality is now purely a type-level concern. +

+

+Type Alignment: Step TInput/TOutput types should directly match catalog item +T types, eliminating the need for wrapping/unwrapping ceremony. +

+

+Effect Types: All operations return FlowIO<T> - an effect that represents +an async computation that can fail. This provides: +- Explicit error handling +- Cancellation support +- Functional composition +

+ +## Methods + +### Load\(\) + +Load data as an effect (can fail, is async, can be cancelled). +Returns T directly, which may itself be an IEnumerable or Seq. + +```csharp +FlowIO Load() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +### Save\(T\) + +Save data as an effect. +Accepts T directly, which may itself be an IEnumerable or Seq. + +```csharp +FlowIO Save(T data) +``` + +#### Parameters + +`data` T + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.IItem.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.IItem.md new file mode 100644 index 00000000..7fda960d --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.IItem.md @@ -0,0 +1,172 @@ +# Interface IItem + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Non-generic base interface for catalog items. +Provides untyped operations for internal use by the flow executor and mapping layer. + +```csharp +public interface IItem +``` + +## Remarks + +This interface enables the flow to work with catalog items +without knowing their specific type parameter at compile-time. + +## Properties + +### DataType + +The runtime type of data stored in this catalog item. +For singletons: Returns typeof(T). +For collections: Returns typeof(IEnumerable<T>). + +```csharp +Type DataType { get; } +``` + +#### Property Value + + [Type](https://learn.microsoft.com/dotnet/api/system.type) + +### Label + +Unique label identifying this catalog item within the data catalog. + +```csharp +string Label { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### OwningCatalogLabel + +The label of the -derived class that created +this item. Set automatically by GetOrCreateItem; null for items created outside +a catalog or by custom implementations. + +```csharp +string? OwningCatalogLabel { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +Used by the metadata layer to produce fully-qualified item identifiers in the form +CatalogLabel.ItemLabel. First-write-wins: cross-catalog shared items retain +the label of the catalog that originally created them. + +### PreferredInspectionLevel + +Gets the preferred inspection level for this catalog item. + +```csharp +InspectionLevel? PreferredInspectionLevel { get; } +``` + +#### Property Value + + [InspectionLevel](Flowthru.Data.Validation.InspectionLevel.md)? + +## Methods + +### Exists\(\) + +Checks if data exists at this catalog item location. +Returns an effect that can fail. + +```csharp +FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +### GetCountAsync\(\) + +Gets the count of items in this catalog item. +For collections (IEnumerable<T>), returns the enumerable count. +For singletons, returns 1 if exists, 0 otherwise. + +```csharp +FlowIO GetCountAsync() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[int](https://learn.microsoft.com/dotnet/api/system.int32)\> + +### InspectDeep\(\) + +Performs deep validation of this catalog item. + +```csharp +FlowIO InspectDeep() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +### InspectShallow\(int\) + +Performs shallow validation of this catalog item. + +```csharp +FlowIO InspectShallow(int sampleSize = 100) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +### LoadUntyped\(\) + +Loads data from the catalog item as an untyped object. +Returns an effect that can fail. +The returned type matches the DataType property. + +```csharp +FlowIO LoadUntyped() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[object](https://learn.microsoft.com/dotnet/api/system.object)\> + +### SaveUntyped\(object\) + +Saves untyped data to the catalog item. +Returns an effect that can fail. +The data type must be compatible with the DataType property. + +```csharp +FlowIO SaveUntyped(object data) +``` + +#### Parameters + +`data` [object](https://learn.microsoft.com/dotnet/api/system.object) + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Item-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Item-1.md new file mode 100644 index 00000000..9d419781 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Item-1.md @@ -0,0 +1,347 @@ +# Class Item + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Standard catalog item implementation that delegates to a storage adapter. + +```csharp +public sealed class Item : IItem, IItem +``` + +#### Type Parameters + +`T` + +The data type (container with rows) + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Item](Flowthru.Data.Item\-1.md) + +#### Implements + +[IItem](Flowthru.Data.IItem\-1.md), +[IItem](Flowthru.Data.IItem.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Delegation Pattern: +

+

+This class is a thin wrapper that delegates all operations to an . +The storage adapter handles the actual I/O logic, while this class provides: +- ICatalogItem interface implementation +- Identity for DAG dependency resolution (via Key) +- Type erasure for Flow heterogeneous collections +

+

+Construction: +

+

+Typically created via static factory methods in : +

+
var item = CatalogItems.Csv<CompanySchema>("companies", "data.csv");
+// Returns: ICatalogItem<IEnumerable<CompanySchema>>
+

+Composition vs Inheritance: +

+

+Previous design: Inheritance hierarchy (CsvCatalogItem, JsonCatalogItem, etc.) +New design: Single class + composed storage adapter +

+

+Benefits: +- No class explosion for format × container combinations +- Custom storage via IStorageAdapter implementation +- Clear separation of concerns +

+

+Capability Forwarding: +

+

+The underlying storage adapter provides inspection methods, which this catalog +item automatically forwards. All storage adapters are required to implement inspection. +

+ +## Constructors + +### Item\(string, IStorageAdapter\) + +Creates a new catalog item with the specified key and storage adapter. + +```csharp +public Item(string label, IStorageAdapter storage) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this catalog item + +`storage` [IStorageAdapter](Flowthru.Data.Storage.IStorageAdapter\-1.md) + +Storage adapter that handles I/O operations + +## Properties + +### DataType + +The runtime type of data stored in this catalog item. +For singletons: Returns typeof(T). +For collections: Returns typeof(IEnumerable<T>). + +```csharp +public Type DataType { get; } +``` + +#### Property Value + + [Type](https://learn.microsoft.com/dotnet/api/system.type) + +### Label + +Unique label identifying this catalog item within the data catalog. + +```csharp +public string Label { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### OwningCatalogLabel + +The label of the -derived class that created +this item. Set automatically by GetOrCreateItem; null for items created outside +a catalog or by custom implementations. + +```csharp +public string? OwningCatalogLabel { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +Used by the metadata layer to produce fully-qualified item identifiers in the form +CatalogLabel.ItemLabel. First-write-wins: cross-catalog shared items retain +the label of the catalog that originally created them. + +### PreferredInspectionLevel + +Gets the preferred inspection level for this catalog item. + +```csharp +public InspectionLevel? PreferredInspectionLevel { get; } +``` + +#### Property Value + + [InspectionLevel](Flowthru.Data.Validation.InspectionLevel.md)? + +### Traits + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +## Methods + +### Constrain\(Func\) + +```csharp +public Item Constrain(Func constraintFn) +``` + +#### Parameters + +`constraintFn` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md), [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md)\> + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +### Exists\(\) + +Checks if data exists at this catalog item location. +Returns an effect that can fail. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +### GetCountAsync\(\) + +Gets the count of items in this catalog item. +For collections (IEnumerable<T>), returns the enumerable count. +For singletons, returns 1 if exists, 0 otherwise. + +```csharp +public FlowIO GetCountAsync() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[int](https://learn.microsoft.com/dotnet/api/system.int32)\> + +### InspectDeep\(\) + +Performs deep validation of this catalog item. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +Forwards the call directly to the underlying storage adapter. +All storage adapters must implement inspection. + +### InspectShallow\(int\) + +Performs shallow validation of this catalog item. + +```csharp +public FlowIO InspectShallow(int sampleSize = 100) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +Forwards the call directly to the underlying storage adapter. +All storage adapters must implement inspection. + +### Load\(\) + +Load data as an effect (can fail, is async, can be cancelled). +Returns T directly, which may itself be an IEnumerable or Seq. + +```csharp +public FlowIO Load() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +### LoadUntyped\(\) + +Loads data from the catalog item as an untyped object. +Returns an effect that can fail. +The returned type matches the DataType property. + +```csharp +public FlowIO LoadUntyped() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[object](https://learn.microsoft.com/dotnet/api/system.object)\> + +### Save\(T\) + +Save data as an effect. +Accepts T directly, which may itself be an IEnumerable or Seq. + +```csharp +public FlowIO Save(T data) +``` + +#### Parameters + +`data` T + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +### SaveUntyped\(object\) + +Saves untyped data to the catalog item. +Returns an effect that can fail. +The data type must be compatible with the DataType property. + +```csharp +public FlowIO SaveUntyped(object data) +``` + +#### Parameters + +`data` [object](https://learn.microsoft.com/dotnet/api/system.object) + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +### WithInspectionLevel\(InspectionLevel\) + +Sets the preferred inspection level for this catalog item. + +```csharp +public Item WithInspectionLevel(InspectionLevel level) +``` + +#### Parameters + +`level` [InspectionLevel](Flowthru.Data.Validation.InspectionLevel.md) + +The inspection level to use + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +This catalog item for method chaining + +#### Remarks + +

+Used to configure how this item should be validated before flow execution. +

+

+Example: +

+
var item = CatalogItems.Csv<Company>("companies", "data.csv")
+    .WithInspectionLevel(InspectionLevel.Deep);
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Items.Single.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Items.Single.md new file mode 100644 index 00000000..62522fb5 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Items.Single.md @@ -0,0 +1,228 @@ +# Class Items.Single + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Factory methods for single (non-collection) values. + +```csharp +public static class Items.Single +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Items.Single](Flowthru.Data.Items.Single.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+These methods create catalog items for single objects rather than collections. +

+

+Use Cases: +

+
  • Model files (ML models, configuration objects)
  • Metrics and evaluation results (single JSON objects)
  • Text reports (Markdown, plain text)
  • Binary files (images, PDFs)
  • Side-effect-only steps (null/void semantics)
+ +## Methods + +### Binary\(string, string\) + +Creates a binary file catalog item. + +```csharp +public static Item Binary(string label, string filePath) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to binary file (.png, .jpg, .pdf, etc.) + +#### Returns + + [Item](Flowthru.Data.Item\-1.md)<[byte](https://learn.microsoft.com/dotnet/api/system.byte)\[\]\> + +Catalog item for binary file with byte array content + +#### Remarks + +

+Use Case: Images (PNG, JPG), PDFs, any binary data +

+

+Implementation: Reads entire file as byte array. +

+

+Storage Traits: All traits use filesystem baseline defaults +

+ +### Json\(string, string\) + +Creates a JSON file catalog item for a single object (non-collection). + +```csharp +public static Item Json(string label, string filePath) where T : IStructuredSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to JSON file + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +Catalog item for singleton JSON object + +#### Type Parameters + +`T` + +Object type (must be structured-serializable) + +#### Remarks + +

+Use Case: Model files, configuration objects, metrics, single records +

+

+Serialization: Single JSON object (not wrapped in array) +

+

+Implementation: Uses SingletonJsonStorageAdapter which bypasses +format/container composition for direct object serialization. +

+ +### Memory\(string\) + +Creates a memory catalog item for a single object (non-collection). + +```csharp +public static Item Memory(string label) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +Catalog item for in-memory singleton + +#### Type Parameters + +`T` + +Object type + +#### Remarks + +

+Use Case: Models, charts, computed metrics that stay in memory +

+

+Examples: +

+
  • ML models (LinearRegressionModel)
  • Charts (GenericChart from Plotly.NET)
  • Metrics objects (ModelMetrics, CrossValidationResults)
  • Any singleton data that doesn't need persistence
+ +### Null\(string\) + +Creates a null catalog item for side-effect-only steps. + +```csharp +public static Item Null(string label) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +Catalog item for void/no-data semantics + +#### Type Parameters + +`T` + +The data type (typically NoData) + +#### Remarks + +

+Use Case: Steps that perform side effects (logging, visualization) without producing meaningful data +

+

+Implementation: Uses NullStorageAdapter which performs no I/O operations. +

+

+Storage Traits: +

+
  • CanWrite: false (Save is a no-op)
  • CanRead: false (Load throws NotSupportedException)
+ +### Text\(string, string\) + +Creates a plain text file catalog item . + +```csharp +public static Item Text(string label, string filePath) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to text file (.txt, .md, etc.) + +#### Returns + + [Item](Flowthru.Data.Item\-1.md)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +Catalog item for text file with string content + +#### Remarks + +

+Use Case: Markdown reports, plain text logs, configuration files +

+

+Implementation: Reads entire file as single string. +

+

+Storage Traits: All traits use filesystem baseline defaults +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Items.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Items.md new file mode 100644 index 00000000..97a82415 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Items.md @@ -0,0 +1,136 @@ +# Class Items + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Core.dll + +Static factory methods for creating catalog entries with common configurations. + +```csharp +public static class Items +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Items](Flowthru.Data.Items.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
// Tier 1: No annotations - property names match external field names
+public record SimpleSchema(
+    int Id,           // Looks for "Id" in CSV/Excel/JSON
+    string Name       // Looks for "Name" in CSV/Excel/JSON
+) : IFlatSchema, ITextSerializable;
+
+var simple = Items.Enumerable.Csv<SimpleSchema>("data", "data.csv");
+
+// Tier 2: Explicit annotations - handle naming mismatches
+public record ShuttleSchema(
+    [SerializedLabel("id")]
+    string Id,
+
+    [SerializedLabel("shuttle_location")]        // snake_case in file
+    string ShuttleLocation,
+
+    [SerializedLabel("d_check_complete")]        // snake_case in file
+    bool DCheckComplete,
+
+    [SerializedLabel("Company ID")]              // space-separated in file
+    int CompanyId
+) : IFlatSchema, ITextSerializable;
+
+var shuttles = Items.Enumerable.Excel<ShuttleSchema>(
+    "shuttles",
+    "data/shuttles.xlsx",
+    "Sheet1"
+);
+
+// Same schema works across all formats
+var csv = Items.Enumerable.Csv<ShuttleSchema>("shuttles", "data/shuttles.csv");
+var json = Items.Enumerable.Json<ShuttleSchema>("shuttles", "data/shuttles.json");
+ +## Remarks + +

+Design Pattern: Static factory methods that compose storage adapters +from medium + format + container layers. +

+

+Discoverability: All factory methods are in one place with IntelliSense support. +

+

+Type Safety: Generic constraints enforce schema compatibility at compile-time. +

+

+Field Name Mapping with SerializedLabel: +

+

+Use the SerializedLabelAttribute to map C# property names to external field names +when they differ. This works uniformly across CSV, Excel, JSON, and most formats. +

+ +## Properties + +### Enumerable + +Factory methods for catalog entries. + +```csharp +public static EnumerableItems Enumerable { get; } +``` + +#### Property Value + + [EnumerableItems](Flowthru.Data.EnumerableItems.md) + +## Methods + +### Null\(string\) + +Creates a null catalog entry for side-effect-only nodes. + +```csharp +public static Item Null(string label) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +#### Returns + + [Item](Flowthru.Data.Item\-1.md) + +Catalog entry for void/no-data semantics + +#### Type Parameters + +`T` + +The data type (typically NoData) + +#### Remarks + +

+Use Case: Nodes that perform side effects (logging, visualization) without producing meaningful data +

+

+Implementation: Uses NullStorageAdapter which performs no I/O operations. +

+

+Storage Traits: +

+
  • CanWrite: false (Save is a no-op)
  • CanRead: false (Load throws NotSupportedException)
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.BinaryFileStorageAdapter.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.BinaryFileStorageAdapter.md new file mode 100644 index 00000000..5ea4982a --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.BinaryFileStorageAdapter.md @@ -0,0 +1,239 @@ +# Class BinaryFileStorageAdapter + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Storage adapter for binary files with byte array content. + +```csharp +public sealed class BinaryFileStorageAdapter : IStorageAdapter +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[BinaryFileStorageAdapter](Flowthru.Data.Storage.BinaryFileStorageAdapter.md) + +#### Implements + +[IStorageAdapter](Flowthru.Data.Storage.IStorageAdapter\-1.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Use Cases: Images (PNG, JPG), PDFs, serialized binary data +

+

+Storage Traits: All traits use filesystem baseline defaults +

+ +## Constructors + +### BinaryFileStorageAdapter\(string\) + +```csharp +public BinaryFileStorageAdapter(string filePath) +``` + +#### Parameters + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +## Properties + +### Traits + +Structural constraints and capabilities of this storage implementation. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +

+Adapter authors must declare what their storage can and cannot do. +These are intrinsic properties of the storage medium, not runtime state. +

+

+Pipeline validation uses these traits to fail fast when a pipeline attempts +invalid operations (e.g., writing to a read-only source). +

+ +## Methods + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+Delegates to the underlying medium's Exists check. +Used to determine if a catalog entry is a seed (Layer 0 input). +

+ +### InspectDeep\(\) + +Performs deep validation by examining the entire dataset. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that all data is available, accessible, and valid. +

+

+Additional Checks Beyond Shallow: +

+
  • Validate ALL rows can be deserialized (not just sample)
  • Check data quality constraints across entire dataset
  • Detect corruption or inconsistencies throughout data
+

+Implementation Guidelines: +

+
  • File adapters: Read and validate entire file
  • Memory adapters: Validate all stored data
  • Database adapters: Full table scan with validation
  • Null adapters: Always return success (no data required)
+

+Performance: Potentially expensive - only use when data integrity is critical. +

+ +### InspectShallow\(int\) + +Performs shallow validation by checking data availability and sampling a subset of data. + +```csharp +public FlowIO InspectShallow(int sampleSize) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that data is available and accessible. +

+

+Typical Checks: +

+
  • Data source exists (file, table, etc.)
  • Data source is accessible (permissions, connectivity)
  • Sample rows can be read and deserialized successfully
  • Schema matches expected structure
+

+Implementation Guidelines: +

+
  • File adapters: Check file exists, read and validate sample rows
  • Memory adapters: Check if data has been initialized
  • Database adapters: Check table exists, query sample rows
  • Null adapters: Always return success (no data required)
+

+Performance: Should be fast (~10-100ms) - suitable for pre-flight validation. +

+ +### Load\(\) + +Loads data from storage. + +```csharp +public FlowIO Load() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[byte](https://learn.microsoft.com/dotnet/api/system.byte)\[\]\> + +Effect that produces data on success + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. medium.ReadStream()           → Stream
+2. format.DeserializeRows()      → IAsyncEnumerable<TRow>
+3. container.FromRows()          → TContainer
+

+Error Handling: +

+

+Errors from any layer are propagated: +- Medium errors (file not found, access denied) +- Format errors (parse failures, schema mismatches) +- Container errors (memory allocation, type conversion) +

+ +### Save\(byte\[\]\) + +Saves data to storage. + +```csharp +public FlowIO Save(byte[] data) +``` + +#### Parameters + +`data` [byte](https://learn.microsoft.com/dotnet/api/system.byte)\[\] + +The data to save + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +Effect that completes on successful save + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. container.ToRows()            → IAsyncEnumerable<TRow>
+2. format.SerializeRows()        → Stream
+3. medium.WriteStream()          → FlowUnit
+

+Atomicity: +

+

+Implementations should strive for atomic saves to avoid partial writes on failure. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.ComposedStorageAdapter-2.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.ComposedStorageAdapter-2.md new file mode 100644 index 00000000..74c45c72 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.ComposedStorageAdapter-2.md @@ -0,0 +1,304 @@ +# Class ComposedStorageAdapter + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Composed storage adapter that delegates to medium, format, and container layers. + +```csharp +public sealed class ComposedStorageAdapter : IStorageAdapter where TRow : notnull +``` + +#### Type Parameters + +`TContainer` + +The in-memory container type (IEnumerable, IDataView, Seq) + +`TRow` + +The row schema type + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ComposedStorageAdapter](Flowthru.Data.Storage.ComposedStorageAdapter\-2.md) + +#### Implements + +[IStorageAdapter](Flowthru.Data.Storage.IStorageAdapter\-1.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
// CSV file with IEnumerable container
+var csvEnum = new ComposedStorageAdapter<IEnumerable<Company>, Company>(
+    medium: new FileStorageMedium("data.csv"),
+    format: new CsvFormatSerializer<Company>(),
+    container: new EnumerableContainerAdapter<Company>()
+);
+
+// Same CSV file with IDataView container
+var csvDataView = new ComposedStorageAdapter<IDataView, Company>(
+    medium: new FileStorageMedium("data.csv"),
+    format: new CsvFormatSerializer<Company>(),
+    container: new DataViewContainerAdapter<Company>(mlContext)
+);
+
+// JSON in memory with IEnumerable container
+var jsonEnumerable = new ComposedStorageAdapter<IEnumerable<Order>, Order>(
+    medium: new MemoryStorageMedium(),
+    format: new JsonFormatSerializer<Order>(),
+    container: new EnumerableContainerAdapter<Order>()
+);
+ +## Remarks + +

+Composition Pattern: +

+

+This class composes three independent concerns: +

+
Medium (WHERE)    → Format (HOW)         → Container (WHAT)
+File/Memory/Net   → CSV/JSON/Parquet     → IEnumerable/IDataView/Seq
+

+Multiplicative Flexibility: +

+

+With M mediums, F formats, and C containers, you get M × F × C combinations +with only M + F + C implementations. +

+

+Example: 3 mediums × 4 formats × 3 containers = 36 combinations with 10 implementations. +

+

+Capability Implementation: +

+

+This adapter can optionally implement capability interfaces based on the +underlying medium and format capabilities. Capabilities are implemented +as explicit interface implementations to avoid polluting the base interface. +

+ +## Constructors + +### ComposedStorageAdapter\(IStorageMedium, IFormatSerializer, IContainerAdapter\) + +Creates a new composed storage adapter. + +```csharp +public ComposedStorageAdapter(IStorageMedium medium, IFormatSerializer format, IContainerAdapter container) +``` + +#### Parameters + +`medium` [IStorageMedium](Flowthru.Data.Storage.IStorageMedium.md) + +The storage medium (file, memory, etc.) + +`format` [IFormatSerializer](Flowthru.Data.Storage.IFormatSerializer\-1.md) + +The format serializer (CSV, JSON, etc.) + +`container` [IContainerAdapter](Flowthru.Data.Storage.IContainerAdapter\-2.md) + +The container adapter (IEnumerable, IDataView, etc.) + +## Properties + +### Traits + +Structural constraints and capabilities of this storage implementation. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +

+Adapter authors must declare what their storage can and cannot do. +These are intrinsic properties of the storage medium, not runtime state. +

+

+Pipeline validation uses these traits to fail fast when a pipeline attempts +invalid operations (e.g., writing to a read-only source). +

+ +## Methods + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+Delegates to the underlying medium's Exists check. +Used to determine if a catalog entry is a seed (Layer 0 input). +

+ +### InspectDeep\(\) + +Performs deep validation by examining the entire dataset. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that all data is available, accessible, and valid. +

+

+Additional Checks Beyond Shallow: +

+
  • Validate ALL rows can be deserialized (not just sample)
  • Check data quality constraints across entire dataset
  • Detect corruption or inconsistencies throughout data
+

+Implementation Guidelines: +

+
  • File adapters: Read and validate entire file
  • Memory adapters: Validate all stored data
  • Database adapters: Full table scan with validation
  • Null adapters: Always return success (no data required)
+

+Performance: Potentially expensive - only use when data integrity is critical. +

+ +### InspectShallow\(int\) + +Performs shallow validation by checking data availability and sampling a subset of data. + +```csharp +public FlowIO InspectShallow(int sampleSize) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that data is available and accessible. +

+

+Typical Checks: +

+
  • Data source exists (file, table, etc.)
  • Data source is accessible (permissions, connectivity)
  • Sample rows can be read and deserialized successfully
  • Schema matches expected structure
+

+Implementation Guidelines: +

+
  • File adapters: Check file exists, read and validate sample rows
  • Memory adapters: Check if data has been initialized
  • Database adapters: Check table exists, query sample rows
  • Null adapters: Always return success (no data required)
+

+Performance: Should be fast (~10-100ms) - suitable for pre-flight validation. +

+ +### Load\(\) + +Loads data from storage. + +```csharp +public FlowIO Load() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +Effect that produces data on success + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. medium.ReadStream()           → Stream
+2. format.DeserializeRows()      → IAsyncEnumerable<TRow>
+3. container.FromRows()          → TContainer
+

+Error Handling: +

+

+Errors from any layer are propagated: +- Medium errors (file not found, access denied) +- Format errors (parse failures, schema mismatches) +- Container errors (memory allocation, type conversion) +

+ +### Save\(TContainer\) + +Saves data to storage. + +```csharp +public FlowIO Save(TContainer data) +``` + +#### Parameters + +`data` TContainer + +The data to save + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +Effect that completes on successful save + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. container.ToRows()            → IAsyncEnumerable<TRow>
+2. format.SerializeRows()        → Stream
+3. medium.WriteStream()          → FlowUnit
+

+Atomicity: +

+

+Implementations should strive for atomic saves to avoid partial writes on failure. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Container.EnumerableContainerAdapter-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Container.EnumerableContainerAdapter-1.md new file mode 100644 index 00000000..f8cb82b9 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Container.EnumerableContainerAdapter-1.md @@ -0,0 +1,143 @@ +# Class EnumerableContainerAdapter + +Namespace: [Flowthru.Data.Storage.Container](Flowthru.Data.Storage.Container.md) +Assembly: Flowthru.Core.dll + +Container adapter for IEnumerable<T> - standard .NET collection type. + +```csharp +public sealed class EnumerableContainerAdapter : IContainerAdapter, T> +``` + +#### Type Parameters + +`T` + +The element type + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EnumerableContainerAdapter](Flowthru.Data.Storage.Container.EnumerableContainerAdapter\-1.md) + +#### Implements + +[IContainerAdapter, T\>](Flowthru.Data.Storage.IContainerAdapter\-2.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
var adapter = new EnumerableContainerAdapter<CompanySchema>();
+
+// From rows (materializes to List)
+var companies = await adapter.FromRows(rowStream);
+
+// Multiple enumeration - safe
+var count = companies.Count();
+var firstFive = companies.Take(5).ToList();
+
+// Back to rows
+var rowsAgain = adapter.ToRows(companies);
+ +## Remarks + +

+Characteristics: +

+
  • Eager materialization: Loads all rows into memory (List)
  • Standard .NET: Works with all .NET LINQ operations
  • Multiple enumeration: Safe - data is cached in memory
  • Memory bound: Not suitable for very large datasets
+

+Use Cases: +

+
  • Small to medium datasets (<100K rows)
  • When multiple enumerations are needed
  • Standard .NET pipelines without functional dependencies
  • Testing and prototyping
+

+Alternatives: +

+
  • Large datasets: Use DataViewContainerAdapter<T> for ML.NET IDataView
  • Lazy evaluation: Use SeqContainerAdapter<T> for LanguageExt Seq
+ +## Constructors + +### EnumerableContainerAdapter\(\) + +Creates a new enumerable container adapter. + +```csharp +public EnumerableContainerAdapter() +``` + +## Methods + +### FromRows\(IAsyncEnumerable\) + +Materializes an async stream of rows into an in-memory container. + +```csharp +public Task> FromRows(IAsyncEnumerable rows) +``` + +#### Parameters + +`rows` [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +Async stream of rows from format deserialization + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Task producing the populated container + +#### Remarks + +

+Materialization Strategy: +

+

+Different containers have different materialization approaches: +

+
  • IEnumerable: Eager - load all rows into List
  • Seq: Lazy - wrap async enumerable
  • IDataView: Columnar - convert to columnar format
+

+Memory Considerations: +

+

+Be aware that eager containers (IEnumerable) will load all data into memory. +For large datasets, prefer streaming containers (Seq, IDataView with lazy loading). +

+ +### ToRows\(IEnumerable\) + +Converts an in-memory container back to an async stream of rows. + +```csharp +public IAsyncEnumerable ToRows(IEnumerable container) +``` + +#### Parameters + +`container` [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1) + +The container to stream from + +#### Returns + + [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +Async enumerable of rows for format serialization + +#### Remarks + +

+Streaming Strategy: +

+

+Rows should be yielded lazily if the container supports it: +

+
  • IEnumerable: Enumerate and yield
  • Seq: Already lazy - expose as async enumerable
  • IDataView: Enumerate rows from columnar format
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Container.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Container.md new file mode 100644 index 00000000..2501a501 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Container.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Data.Storage.Container + +### Classes + + [EnumerableContainerAdapter](Flowthru.Data.Storage.Container.EnumerableContainerAdapter\-1.md) + +Container adapter for IEnumerable<T> - standard .NET collection type. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Format.JsonFormatSerializer-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Format.JsonFormatSerializer-1.md new file mode 100644 index 00000000..b4ff515f --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Format.JsonFormatSerializer-1.md @@ -0,0 +1,331 @@ +# Class JsonFormatSerializer + +Namespace: [Flowthru.Data.Storage.Format](Flowthru.Data.Storage.Format.md) +Assembly: Flowthru.Core.dll + +Format serializer for JSON (JavaScript Object Notation) files. + +```csharp +public sealed class JsonFormatSerializer : IFormatSerializer where TRow : notnull, IStructuredSerializable +``` + +#### Type Parameters + +`TRow` + +The row schema type + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[JsonFormatSerializer](Flowthru.Data.Storage.Format.JsonFormatSerializer\-1.md) + +#### Implements + +[IFormatSerializer](Flowthru.Data.Storage.IFormatSerializer\-1.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
// Flat schema
+public record MetricsSchema(
+    double Accuracy,
+    double Precision,
+    double Recall
+) : IFlatSchema, IStructuredSerializable;
+
+// Nested schema
+public record ResultsSchema(
+    List<MetricsSchema> FoldMetrics,
+    double MeanAccuracy
+) : INestedSchema, IStructuredSerializable;
+
+var serializer = new JsonFormatSerializer<ResultsSchema>();
+
+// Serialize
+var results = new[] {
+    new ResultsSchema(new List<MetricsSchema> { /* ... */ }, 0.95)
+};
+
+using var writeStream = File.Create("results.json");
+await serializer.SerializeRows(writeStream, results.ToAsyncEnumerable());
+ +## Remarks + +

+Type Constraints: +

+

+TRow must implement , which supports both: +

+
  • - Simple flat structures
  • - Complex nested structures
+

+JSON is flexible and can handle any schema structure, making it suitable for: +- Configuration objects +- Model metadata and metrics +- Nested result structures +- Human-readable data files +

+

+Configuration: +

+

+Uses System.Text.Json with default configuration: +- WriteIndented = true (pretty printing) +- PropertyNamingPolicy = CamelCase +- DefaultIgnoreCondition = WhenWritingNull +

+

+Custom JsonSerializerOptions can be provided via constructor. +

+

+Streaming Behavior: +

+

+JSON serialization streams rows as a JSON array: +

+
[
+  { "id": 1, "name": "Item 1" },
+  { "id": 2, "name": "Item 2" }
+]
+

+Both deserialization and serialization are streaming, yielding/consuming +rows lazily for memory efficiency. +

+ +## Constructors + +### JsonFormatSerializer\(\) + +Creates a new JSON format serializer with default configuration. + +```csharp +public JsonFormatSerializer() +``` + +#### Remarks + +

+Property Naming: No default naming policy is applied. +Use to specify property names explicitly. +If no SerializedLabel is present, the C# property name is used as-is. +

+ +### JsonFormatSerializer\(JsonSerializerOptions\) + +Creates a new JSON format serializer with custom options. + +```csharp +public JsonFormatSerializer(JsonSerializerOptions options) +``` + +#### Parameters + +`options` [JsonSerializerOptions](https://learn.microsoft.com/dotnet/api/system.text.json.jsonserializeroptions) + +JSON serialization options + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +Thrown if options is null + +## Properties + +### Options + +Gets the JSON serialization options for this serializer. + +```csharp +public JsonSerializerOptions Options { get; } +``` + +#### Property Value + + [JsonSerializerOptions](https://learn.microsoft.com/dotnet/api/system.text.json.jsonserializeroptions) + +### Traits + +Structural capabilities of this format serializer. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +

+Format traits focus on HOW data is serialized and whether it supports streaming. +For composed adapters, these traits are merged with medium and container traits. +

+

+Most formats should declare CanStream = true if they can deserialize row-by-row +without buffering the entire stream (e.g., CSV, JSONL). Formats that require full +parsing before yielding rows (e.g., JSON arrays) should set CanStream = false. +

+ +## Methods + +### DeserializeRows\(Stream\) + +Deserializes a stream of bytes into a stream of rows. + +```csharp +public IAsyncEnumerable DeserializeRows(Stream stream) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +The stream containing serialized data + +#### Returns + + [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +Async enumerable of deserialized rows + +#### Remarks + +

+Streaming Behavior: +

+

+Rows should be yielded as they are deserialized (lazy evaluation). +This allows processing large datasets without loading everything into memory. +

+

+Error Handling: +

+

+Deserialization errors should throw exceptions: +- Format exceptions (malformed CSV, invalid JSON) +- Schema mismatches (missing columns, type conversion failures) +- I/O errors during stream reading +

+

+The caller should handle these exceptions appropriately. +

+ +### GetPropertyMappingConfiguration\(\) + +Configures how this serializer handles property-to-field name mapping for the schema. + +```csharp +public PropertyMappingConfiguration GetPropertyMappingConfiguration() +``` + +#### Returns + + [PropertyMappingConfiguration](Flowthru.Data.Storage.PropertyMappingConfiguration.md) + +Property mapping configuration describing the mapping strategy + +#### Examples + +
// CSV serializer using SerializedLabel
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.FromSerializedLabel<TRow>();
+
+// ML.NET serializer using native attributes
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.FromNativeAttributes("LoadColumnAttribute");
+
+// Parquet with library-controlled mapping
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.LibraryControlled();
+ +#### Remarks + +

+Contractual Obligation: Every format serializer MUST implement this method +to explicitly declare how it handles property name mapping. +

+

+Implementation Strategies: +

+
  • +SerializedLabel Support: Use +to respect [SerializedLabel] attributes. Return . +
  • +Native Attribute Mapping: Use format-specific attributes (e.g., ML.NET's [LoadColumn]). +Return with the attribute type name. +
  • +Library-Controlled: When the underlying library handles mapping internally +(e.g., Parquet.NET with no programmatic API). Return PropertyMappingConfiguration.LibraryControlled(). +
  • +Adapter Pattern: Bridge between SerializedLabel and native attributes. +Return . +
+

+Design Intent: This contract makes property mapping an explicit, discoverable +capability rather than an implicit behavior, enabling: +

+
  • Runtime introspection of mapping capabilities
  • Better error messages when schemas don't match storage
  • Documentation generation for serializer capabilities
  • Testing framework validation of mapping correctness
+ +### SerializeRows\(Stream, IAsyncEnumerable\) + +Serializes a stream of rows into a stream of bytes. + +```csharp +public Task SerializeRows(Stream stream, IAsyncEnumerable rows) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +The stream to write serialized data to + +`rows` [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +The rows to serialize + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task) + +Task that completes when serialization finishes + +#### Remarks + +

+Streaming Behavior: +

+

+Rows should be written as they are enumerated (lazy evaluation). +This allows handling large datasets efficiently. +

+

+Format-Specific Headers: +

+

+Implementations should handle format-specific initialization: +- CSV: Write header row with column names +- JSON: Write opening bracket for array +- Parquet: Write schema metadata +

+

+Error Handling: +

+

+Serialization errors should throw exceptions: +- Type conversion failures +- I/O errors during stream writing +- Invalid data values for format constraints +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Format.PropertyMappingHelper.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Format.PropertyMappingHelper.md new file mode 100644 index 00000000..4e17c4b8 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Format.PropertyMappingHelper.md @@ -0,0 +1,122 @@ +# Class PropertyMappingHelper + +Namespace: [Flowthru.Data.Storage.Format](Flowthru.Data.Storage.Format.md) +Assembly: Flowthru.Core.dll + +Helper for mapping external field names to C# property names using SerializedLabel attribute. + +```csharp +public static class PropertyMappingHelper +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PropertyMappingHelper](Flowthru.Data.Storage.Format.PropertyMappingHelper.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Purpose: Centralized property mapping logic used by all format serializers. +

+

+Mapping Strategy: +

+
  1. Check for [SerializedLabel] attribute - if present, use that label
  2. Fall back to property name if no attribute
  3. Use case-insensitive comparison for lookups
+

+Extensibility: New format serializers should use this helper to ensure +consistent behavior across all storage mechanisms. +

+ +## Methods + +### BuildPropertyMap\(\) + +Builds a mapping from external field names to PropertyInfo objects. + +```csharp +public static Dictionary BuildPropertyMap() +``` + +#### Returns + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [PropertyInfo](https://learn.microsoft.com/dotnet/api/system.reflection.propertyinfo)\> + +Dictionary mapping external field names (case-insensitive) to PropertyInfo + +#### Type Parameters + +`T` + +The schema type + +#### Examples + +
// Build property map for a schema
+var propertyMap = PropertyMappingHelper.BuildPropertyMap<ShuttleSchema>();
+
+// Look up property by external field name
+if (propertyMap.TryGetValue("shuttle_location", out var property))
+{
+    // Maps to ShuttleLocation property
+    var value = reader.GetValue("shuttle_location");
+    property.SetValue(instance, value);
+}
+ +#### Remarks + +

+The returned dictionary uses case-insensitive string comparison, allowing flexible +matching of external field names regardless of casing. +

+

+Lookup Priority: +

+
  1. [SerializedLabel("field_name")] - explicit attribute takes precedence
  2. Property.Name - fallback if no attribute present
+ +### GetFieldName\(PropertyInfo\) + +Gets the external field name for a property. + +```csharp +public static string GetFieldName(PropertyInfo property) +``` + +#### Parameters + +`property` [PropertyInfo](https://learn.microsoft.com/dotnet/api/system.reflection.propertyinfo) + +The property to get the field name for + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +The external field name (from SerializedLabel or property name) + +#### Examples + +
var properties = typeof(ShuttleSchema).GetProperties();
+foreach (var property in properties)
+{
+    var fieldName = PropertyMappingHelper.GetFieldName(property);
+    writer.WriteField(fieldName, property.GetValue(instance));
+}
+ +#### Remarks + +

+This method is useful for serialization scenarios where you need to write +property values to external storage with the correct field names. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Format.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Format.md new file mode 100644 index 00000000..8fa95b1a --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Format.md @@ -0,0 +1,12 @@ +# Namespace Flowthru.Data.Storage.Format + +### Classes + + [JsonFormatSerializer](Flowthru.Data.Storage.Format.JsonFormatSerializer\-1.md) + +Format serializer for JSON (JavaScript Object Notation) files. + + [PropertyMappingHelper](Flowthru.Data.Storage.Format.PropertyMappingHelper.md) + +Helper for mapping external field names to C# property names using SerializedLabel attribute. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IContainerAdapter-2.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IContainerAdapter-2.md new file mode 100644 index 00000000..3d8be172 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IContainerAdapter-2.md @@ -0,0 +1,136 @@ +# Interface IContainerAdapter + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Interface for container adaptation - converts between streaming rows and in-memory containers. + +```csharp +public interface IContainerAdapter +``` + +#### Type Parameters + +`TContainer` + +The in-memory container type (IEnumerable, IDataView, Seq, etc.) + +`TRow` + +The row type (schema) + +## Examples + +
// IEnumerable container adapter
+var enumerableAdapter = new EnumerableContainerAdapter<CompanySchema>();
+var companies = await enumerableAdapter.FromRows(rowStream);
+// Type: IEnumerable<CompanySchema>
+
+// IDataView container adapter
+var dataViewAdapter = new DataViewContainerAdapter<CompanySchema>(mlContext);
+var dataView = await dataViewAdapter.FromRows(rowStream);
+// Type: IDataView (ML.NET)
+ +## Remarks + +

+Responsibility: Abstract WHAT in-memory representation to use for data. +

+

+Separation of Concerns: +

+

+The container adapter is isolated from: +- Storage location (file vs memory) - handled by +- Serialization format (CSV vs JSON) - handled by +

+

+Bridge Pattern: +

+

+This adapter bridges between: +- Streaming rows () - format layer +- In-memory container (IEnumerable, IDataView) - application layer +

+

+Container Type Examples: +

+
  • IEnumerable<TRow> - Standard .NET collections
  • IDataView - ML.NET's columnar data representation
  • DataFrame - Pandas-style dataframe (future)
  • IObservable<TRow> - Reactive streams (future)
+

+Design Pattern: +

+

+This is the top layer in the composition pattern: +

+
Medium (bytes) → Format (rows) → Container (in-memory)
+Stream         → IAsyncEnumerable<TRow> → IEnumerable<TRow> / IDataView
+ +## Methods + +### FromRows\(IAsyncEnumerable\) + +Materializes an async stream of rows into an in-memory container. + +```csharp +Task FromRows(IAsyncEnumerable rows) +``` + +#### Parameters + +`rows` [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +Async stream of rows from format deserialization + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1) + +Task producing the populated container + +#### Remarks + +

+Materialization Strategy: +

+

+Different containers have different materialization approaches: +

+
  • IEnumerable: Eager - load all rows into List
  • Seq: Lazy - wrap async enumerable
  • IDataView: Columnar - convert to columnar format
+

+Memory Considerations: +

+

+Be aware that eager containers (IEnumerable) will load all data into memory. +For large datasets, prefer streaming containers (Seq, IDataView with lazy loading). +

+ +### ToRows\(TContainer\) + +Converts an in-memory container back to an async stream of rows. + +```csharp +IAsyncEnumerable ToRows(TContainer container) +``` + +#### Parameters + +`container` TContainer + +The container to stream from + +#### Returns + + [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +Async enumerable of rows for format serialization + +#### Remarks + +

+Streaming Strategy: +

+

+Rows should be yielded lazily if the container supports it: +

+
  • IEnumerable: Enumerate and yield
  • Seq: Already lazy - expose as async enumerable
  • IDataView: Enumerate rows from columnar format
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IFormatSerializer-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IFormatSerializer-1.md new file mode 100644 index 00000000..2ebbffca --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IFormatSerializer-1.md @@ -0,0 +1,260 @@ +# Interface IFormatSerializer + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Interface for format serialization - handles row-based serialization/deserialization. + +```csharp +public interface IFormatSerializer where TRow : notnull +``` + +#### Type Parameters + +`TRow` + +The row type (schema) to serialize + +## Examples + +
// CSV serializer with flat schema constraint
+var csvSerializer = new CsvFormatSerializer<CompanySchema>();
+
+// Deserialize from stream to rows
+await foreach (var row in csvSerializer.DeserializeRows(stream))
+{
+    Console.WriteLine($"Company: {row.Name}");
+}
+
+// Serialize rows to stream
+await csvSerializer.SerializeRows(stream, rows);
+ +## Remarks + +

+Responsibility: Abstract HOW data is serialized (CSV, JSON, Parquet, etc.). +

+

+Separation of Concerns: +

+

+The format serializer is isolated from: +- Storage location (file vs memory) - handled by +- Container type (IEnumerable vs IDataView) - handled by +

+

+Streaming Design: +

+

+Uses for row streaming to: +- Support large datasets without loading everything into memory +- Enable backpressure and cancellation +- Allow format-agnostic streaming pipelines +

+

+Type Constraints: +

+

+The notnull constraint ensures all format serializers support modern C# patterns: +

+
  • Traditional schemas: Classes/records with parameterless constructors
  • Required members: C# 11+ schemas with required properties
  • Positional records: Records with primary constructors
+

+This constraint explicitly prohibits the new() constraint, which is incompatible +with required members and positional records. Format serializers must use + or equivalent techniques to instantiate schemas. +

+

+Format serializers may add additional constraints for format-specific requirements: +

+
public class CsvFormatSerializer<T> : IFormatSerializer<T>
+    where T : notnull, IFlatSchema, ITextSerializable
+{
+    // Compile-time enforcement: notnull + flat + text serializable
+}
+

+Design Pattern: +

+

+This is the middle layer in the composition pattern: +

+
Medium (bytes) → Format (rows) → Container (in-memory)
+Stream         → IAsyncEnumerable<TRow> → IEnumerable<TRow>
+ +## Properties + +### Traits + +Structural capabilities of this format serializer. + +```csharp +StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +

+Format traits focus on HOW data is serialized and whether it supports streaming. +For composed adapters, these traits are merged with medium and container traits. +

+

+Most formats should declare CanStream = true if they can deserialize row-by-row +without buffering the entire stream (e.g., CSV, JSONL). Formats that require full +parsing before yielding rows (e.g., JSON arrays) should set CanStream = false. +

+ +## Methods + +### DeserializeRows\(Stream\) + +Deserializes a stream of bytes into a stream of rows. + +```csharp +IAsyncEnumerable DeserializeRows(Stream stream) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +The stream containing serialized data + +#### Returns + + [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +Async enumerable of deserialized rows + +#### Remarks + +

+Streaming Behavior: +

+

+Rows should be yielded as they are deserialized (lazy evaluation). +This allows processing large datasets without loading everything into memory. +

+

+Error Handling: +

+

+Deserialization errors should throw exceptions: +- Format exceptions (malformed CSV, invalid JSON) +- Schema mismatches (missing columns, type conversion failures) +- I/O errors during stream reading +

+

+The caller should handle these exceptions appropriately. +

+ +### GetPropertyMappingConfiguration\(\) + +Configures how this serializer handles property-to-field name mapping for the schema. + +```csharp +PropertyMappingConfiguration GetPropertyMappingConfiguration() +``` + +#### Returns + + [PropertyMappingConfiguration](Flowthru.Data.Storage.PropertyMappingConfiguration.md) + +Property mapping configuration describing the mapping strategy + +#### Examples + +
// CSV serializer using SerializedLabel
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.FromSerializedLabel<TRow>();
+
+// ML.NET serializer using native attributes
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.FromNativeAttributes("LoadColumnAttribute");
+
+// Parquet with library-controlled mapping
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.LibraryControlled();
+ +#### Remarks + +

+Contractual Obligation: Every format serializer MUST implement this method +to explicitly declare how it handles property name mapping. +

+

+Implementation Strategies: +

+
  • +SerializedLabel Support: Use +to respect [SerializedLabel] attributes. Return . +
  • +Native Attribute Mapping: Use format-specific attributes (e.g., ML.NET's [LoadColumn]). +Return with the attribute type name. +
  • +Library-Controlled: When the underlying library handles mapping internally +(e.g., Parquet.NET with no programmatic API). Return PropertyMappingConfiguration.LibraryControlled(). +
  • +Adapter Pattern: Bridge between SerializedLabel and native attributes. +Return . +
+

+Design Intent: This contract makes property mapping an explicit, discoverable +capability rather than an implicit behavior, enabling: +

+
  • Runtime introspection of mapping capabilities
  • Better error messages when schemas don't match storage
  • Documentation generation for serializer capabilities
  • Testing framework validation of mapping correctness
+ +### SerializeRows\(Stream, IAsyncEnumerable\) + +Serializes a stream of rows into a stream of bytes. + +```csharp +Task SerializeRows(Stream stream, IAsyncEnumerable rows) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +The stream to write serialized data to + +`rows` [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +The rows to serialize + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task) + +Task that completes when serialization finishes + +#### Remarks + +

+Streaming Behavior: +

+

+Rows should be written as they are enumerated (lazy evaluation). +This allows handling large datasets efficiently. +

+

+Format-Specific Headers: +

+

+Implementations should handle format-specific initialization: +- CSV: Write header row with column names +- JSON: Write opening bracket for array +- Parquet: Write schema metadata +

+

+Error Handling: +

+

+Serialization errors should throw exceptions: +- Type conversion failures +- I/O errors during stream writing +- Invalid data values for format constraints +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IStorageAdapter-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IStorageAdapter-1.md new file mode 100644 index 00000000..8053b57e --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IStorageAdapter-1.md @@ -0,0 +1,265 @@ +# Interface IStorageAdapter + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Interface for high-level storage operations - abstracts Load/Save with any storage implementation. + +```csharp +public interface IStorageAdapter +``` + +#### Type Parameters + +`T` + +The data type (container with rows) + +## Examples + +
// Composed storage adapter
+var storage = new ComposedStorageAdapter<IEnumerable<CompanySchema>, CompanySchema>(
+    medium: new FileStorageMedium("data.csv"),
+    format: new CsvFormatSerializer<CompanySchema>(),
+    container: new EnumerableContainerAdapter<CompanySchema>()
+);
+
+var loadResult = await storage.Load().Run();
+loadResult.Match(
+    Succ: data => Console.WriteLine($"Loaded {data.Count()} rows"),
+    Fail: err => Console.WriteLine($"Load failed: {err}")
+);
+
+var saveResult = await storage.Save(companies).Run();
+ +## Remarks + +

+Responsibility: Provide simple Load/Save API regardless of underlying storage strategy. +

+

+Abstraction Layer: +

+

+This interface hides the complexity of: +- Medium selection (file vs memory) +- Format serialization (CSV vs JSON vs Parquet) +- Container adaptation (IEnumerable vs IDataView) +

+

+Implementation Strategies: +

+
  • Composed: - composition of medium + format + container
  • Custom: User-defined implementations (database, API, etc.)
+

+Effect Types: +

+

+All operations return effects to represent: +- I/O operations that can fail +- Async execution +- Cancellation support +- Functional composition +

+

+Usage in Catalog Entries: +

+

+ delegates to this interface: +

+
public class Item<T> : IItem<T>
+{
+    private readonly IStorageAdapter<T> _storage;
+
+    public FlowIO<T> Load() => _storage.Load();
+    public FlowIO<FlowUnit> Save(T data) => _storage.Save(data);
+}
+ +## Properties + +### Traits + +Structural constraints and capabilities of this storage implementation. + +```csharp +StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +

+Adapter authors must declare what their storage can and cannot do. +These are intrinsic properties of the storage medium, not runtime state. +

+

+Pipeline validation uses these traits to fail fast when a pipeline attempts +invalid operations (e.g., writing to a read-only source). +

+ +## Methods + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+Delegates to the underlying medium's Exists check. +Used to determine if a catalog entry is a seed (Layer 0 input). +

+ +### InspectDeep\(\) + +Performs deep validation by examining the entire dataset. + +```csharp +FlowIO InspectDeep() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that all data is available, accessible, and valid. +

+

+Additional Checks Beyond Shallow: +

+
  • Validate ALL rows can be deserialized (not just sample)
  • Check data quality constraints across entire dataset
  • Detect corruption or inconsistencies throughout data
+

+Implementation Guidelines: +

+
  • File adapters: Read and validate entire file
  • Memory adapters: Validate all stored data
  • Database adapters: Full table scan with validation
  • Null adapters: Always return success (no data required)
+

+Performance: Potentially expensive - only use when data integrity is critical. +

+ +### InspectShallow\(int\) + +Performs shallow validation by checking data availability and sampling a subset of data. + +```csharp +FlowIO InspectShallow(int sampleSize) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that data is available and accessible. +

+

+Typical Checks: +

+
  • Data source exists (file, table, etc.)
  • Data source is accessible (permissions, connectivity)
  • Sample rows can be read and deserialized successfully
  • Schema matches expected structure
+

+Implementation Guidelines: +

+
  • File adapters: Check file exists, read and validate sample rows
  • Memory adapters: Check if data has been initialized
  • Database adapters: Check table exists, query sample rows
  • Null adapters: Always return success (no data required)
+

+Performance: Should be fast (~10-100ms) - suitable for pre-flight validation. +

+ +### Load\(\) + +Loads data from storage. + +```csharp +FlowIO Load() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +Effect that produces data on success + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. medium.ReadStream()           → Stream
+2. format.DeserializeRows()      → IAsyncEnumerable<TRow>
+3. container.FromRows()          → TContainer
+

+Error Handling: +

+

+Errors from any layer are propagated: +- Medium errors (file not found, access denied) +- Format errors (parse failures, schema mismatches) +- Container errors (memory allocation, type conversion) +

+ +### Save\(T\) + +Saves data to storage. + +```csharp +FlowIO Save(T data) +``` + +#### Parameters + +`data` T + +The data to save + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +Effect that completes on successful save + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. container.ToRows()            → IAsyncEnumerable<TRow>
+2. format.SerializeRows()        → Stream
+3. medium.WriteStream()          → FlowUnit
+

+Atomicity: +

+

+Implementations should strive for atomic saves to avoid partial writes on failure. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IStorageMedium.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IStorageMedium.md new file mode 100644 index 00000000..ea5724fc --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.IStorageMedium.md @@ -0,0 +1,158 @@ +# Interface IStorageMedium + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Interface for storage medium - handles raw byte stream I/O. + +```csharp +public interface IStorageMedium +``` + +## Examples + +
// File storage medium
+var fileMedium = new FileStorageMedium("data/file.csv");
+var readResult = await fileMedium.ReadStream().Run();
+
+// Memory storage medium
+var memoryMedium = new MemoryStorageMedium();
+var writeResult = await memoryMedium.WriteStream(stream).Run();
+ +## Remarks + +

+Responsibility: Abstract WHERE data is stored (file system, memory, network, database). +

+

+Separation of Concerns: +

+

+The storage medium layer is isolated from: +- Format serialization (CSV, JSON, Parquet) - handled by +- Container representation (IEnumerable, IDataView) - handled by +

+

+Design Pattern: +

+

+This is the lowest layer in the composition pattern: +

+
Medium (bytes) → Format (rows) → Container (in-memory)
+File/Memory    → CSV/JSON      → IEnumerable/IDataView
+

+Effect Types: +

+

+All operations return IO<A> effects to represent: +- I/O operations that can fail +- Async execution +- Cancellation support +- Functional composition +

+ +## Properties + +### Traits + +Structural constraints and capabilities of this storage medium. + +```csharp +StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +Medium traits focus on WHERE data is stored and the access patterns it supports. +For composed adapters, these traits are merged with format and container traits. + +## Methods + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+This is used to determine if a catalog entry is a "seed" (Layer 0 input) +or if it's produced by a node in the pipeline. +

+ +### ReadStream\(\) + +Reads raw bytes from storage as a stream. + +```csharp +FlowIO ReadStream() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[Stream](https://learn.microsoft.com/dotnet/api/system.io.stream)\> + +Effect that produces a readable stream on success + +#### Remarks + +

+The returned stream should be positioned at the beginning and ready to read. +The caller is responsible for disposing the stream. +

+

+Error Conditions: +

+
  • Storage location does not exist
  • Access denied (permissions)
  • Network failure (for remote storage)
  • I/O error
+ +### WriteStream\(Stream\) + +Writes raw bytes to storage from a stream. + +```csharp +FlowIO WriteStream(Stream stream) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +Stream containing data to write + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +Effect that completes on successful write + +#### Remarks + +

+The stream will be read from its current position to the end. +The implementation should handle creating parent directories if needed. +

+

+Atomicity: +

+

+Implementations should strive for atomic writes (write to temp, then rename) +to avoid partial writes on failure. +

+

+Error Conditions: +

+
  • Insufficient disk space
  • Access denied (permissions)
  • Network failure (for remote storage)
  • I/O error
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Medium.FileStorageMedium.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Medium.FileStorageMedium.md new file mode 100644 index 00000000..11baf204 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Medium.FileStorageMedium.md @@ -0,0 +1,207 @@ +# Class FileStorageMedium + +Namespace: [Flowthru.Data.Storage.Medium](Flowthru.Data.Storage.Medium.md) +Assembly: Flowthru.Core.dll + +Storage medium for file-based I/O operations. + +```csharp +public sealed class FileStorageMedium : IStorageMedium +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FileStorageMedium](Flowthru.Data.Storage.Medium.FileStorageMedium.md) + +#### Implements + +[IStorageMedium](Flowthru.Data.Storage.IStorageMedium.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
var medium = new FileStorageMedium("data/companies.csv");
+
+// Check if file exists
+var exists = await medium.Exists().Run();
+
+// Read from file
+var readResult = await medium.ReadStream().Run();
+readResult.Match(
+    Succ: stream => { /* process stream */ },
+    Fail: error => Console.WriteLine($"Read failed: {error}")
+);
+
+// Write to file
+using var writeStream = new MemoryStream(data);
+var writeResult = await medium.WriteStream(writeStream).Run();
+ +## Remarks + +

+Responsibility: Handle reading and writing raw byte streams to/from files. +

+

+Features: +

+
  • Automatic directory creation for parent paths
  • Atomic writes via temp file + rename
  • Support for both absolute and relative paths
  • All storage traits use filesystem baseline defaults
+

+Thread Safety: +

+

+This class is thread-safe for reads but writes should be coordinated externally +if multiple threads write to the same file. +

+ +## Constructors + +### FileStorageMedium\(string\) + +Creates a new file storage medium. + +```csharp +public FileStorageMedium(string filePath) +``` + +#### Parameters + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to the file (absolute or relative) + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +Thrown if filePath is null + + [ArgumentException](https://learn.microsoft.com/dotnet/api/system.argumentexception) + +Thrown if filePath is empty or whitespace + +## Properties + +### FilePath + +Gets the file path for this storage medium. + +```csharp +public string FilePath { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Traits + +Structural constraints and capabilities of this storage medium. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +Medium traits focus on WHERE data is stored and the access patterns it supports. +For composed adapters, these traits are merged with format and container traits. + +## Methods + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+This is used to determine if a catalog entry is a "seed" (Layer 0 input) +or if it's produced by a node in the pipeline. +

+ +### ReadStream\(\) + +Reads raw bytes from storage as a stream. + +```csharp +public FlowIO ReadStream() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[Stream](https://learn.microsoft.com/dotnet/api/system.io.stream)\> + +Effect that produces a readable stream on success + +#### Remarks + +

+The returned stream should be positioned at the beginning and ready to read. +The caller is responsible for disposing the stream. +

+

+Error Conditions: +

+
  • Storage location does not exist
  • Access denied (permissions)
  • Network failure (for remote storage)
  • I/O error
+ +### WriteStream\(Stream\) + +Writes raw bytes to storage from a stream. + +```csharp +public FlowIO WriteStream(Stream stream) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +Stream containing data to write + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +Effect that completes on successful write + +#### Remarks + +

+The stream will be read from its current position to the end. +The implementation should handle creating parent directories if needed. +

+

+Atomicity: +

+

+Implementations should strive for atomic writes (write to temp, then rename) +to avoid partial writes on failure. +

+

+Error Conditions: +

+
  • Insufficient disk space
  • Access denied (permissions)
  • Network failure (for remote storage)
  • I/O error
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Medium.MemoryStorageMedium.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Medium.MemoryStorageMedium.md new file mode 100644 index 00000000..c4056b05 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Medium.MemoryStorageMedium.md @@ -0,0 +1,225 @@ +# Class MemoryStorageMedium + +Namespace: [Flowthru.Data.Storage.Medium](Flowthru.Data.Storage.Medium.md) +Assembly: Flowthru.Core.dll + +Storage medium for in-memory byte storage. + +```csharp +public sealed class MemoryStorageMedium : IStorageMedium +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[MemoryStorageMedium](Flowthru.Data.Storage.Medium.MemoryStorageMedium.md) + +#### Implements + +[IStorageMedium](Flowthru.Data.Storage.IStorageMedium.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
var medium = new MemoryStorageMedium();
+
+// Write some data
+using var writeStream = new MemoryStream(Encoding.UTF8.GetBytes("Hello, World!"));
+await medium.WriteStream(writeStream).Run();
+
+// Read it back
+var readResult = await medium.ReadStream().Run();
+readResult.Match(
+    Succ: stream =>
+    {
+        using var reader = new StreamReader(stream);
+        var content = reader.ReadToEnd();
+        Console.WriteLine(content); // "Hello, World!"
+    },
+    Fail: error => Console.WriteLine($"Read failed: {error}")
+);
+ +## Remarks + +

+Responsibility: Store data in memory without any file I/O. +

+

+Use Cases: +

+
  • Testing without file system dependencies
  • Transient pipeline intermediates that don't need persistence
  • Fast prototyping and experimentation
  • In-memory caching of computed results
+

+Characteristics: +

+
  • IsPersistent: false - data lost when process exits
  • Very fast - no I/O overhead
  • Memory-bound - not suitable for large datasets
+

+Thread Safety: +

+

+This class uses locking to ensure thread-safe access to the internal buffer. +Multiple threads can safely read/write concurrently. +

+ +## Constructors + +### MemoryStorageMedium\(\) + +Creates a new memory storage medium with no initial data. + +```csharp +public MemoryStorageMedium() +``` + +### MemoryStorageMedium\(byte\[\]\) + +Creates a new memory storage medium with initial data. + +```csharp +public MemoryStorageMedium(byte[] initialData) +``` + +#### Parameters + +`initialData` [byte](https://learn.microsoft.com/dotnet/api/system.byte)\[\] + +Initial byte buffer + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +Thrown if initialData is null + +## Properties + +### BufferSize + +Gets the current buffer size in bytes, or null if no data is stored. + +```csharp +public int? BufferSize { get; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32)? + +### Traits + +Structural constraints and capabilities of this storage medium. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +Medium traits focus on WHERE data is stored and the access patterns it supports. +For composed adapters, these traits are merged with format and container traits. + +## Methods + +### Clear\(\) + +Clears the internal buffer, freeing memory. + +```csharp +public void Clear() +``` + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+This is used to determine if a catalog entry is a "seed" (Layer 0 input) +or if it's produced by a node in the pipeline. +

+ +### ReadStream\(\) + +Reads raw bytes from storage as a stream. + +```csharp +public FlowIO ReadStream() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[Stream](https://learn.microsoft.com/dotnet/api/system.io.stream)\> + +Effect that produces a readable stream on success + +#### Remarks + +

+The returned stream should be positioned at the beginning and ready to read. +The caller is responsible for disposing the stream. +

+

+Error Conditions: +

+
  • Storage location does not exist
  • Access denied (permissions)
  • Network failure (for remote storage)
  • I/O error
+ +### WriteStream\(Stream\) + +Writes raw bytes to storage from a stream. + +```csharp +public FlowIO WriteStream(Stream stream) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +Stream containing data to write + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +Effect that completes on successful write + +#### Remarks + +

+The stream will be read from its current position to the end. +The implementation should handle creating parent directories if needed. +

+

+Atomicity: +

+

+Implementations should strive for atomic writes (write to temp, then rename) +to avoid partial writes on failure. +

+

+Error Conditions: +

+
  • Insufficient disk space
  • Access denied (permissions)
  • Network failure (for remote storage)
  • I/O error
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Medium.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Medium.md new file mode 100644 index 00000000..720547bb --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Medium.md @@ -0,0 +1,12 @@ +# Namespace Flowthru.Data.Storage.Medium + +### Classes + + [FileStorageMedium](Flowthru.Data.Storage.Medium.FileStorageMedium.md) + +Storage medium for file-based I/O operations. + + [MemoryStorageMedium](Flowthru.Data.Storage.Medium.MemoryStorageMedium.md) + +Storage medium for in-memory byte storage. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.MemoryStorageAdapter-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.MemoryStorageAdapter-1.md new file mode 100644 index 00000000..507e9600 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.MemoryStorageAdapter-1.md @@ -0,0 +1,280 @@ +# Class MemoryStorageAdapter + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Direct memory storage adapter that bypasses serialization. + +```csharp +public sealed class MemoryStorageAdapter : IStorageAdapter +``` + +#### Type Parameters + +`T` + +The data type to store in memory + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[MemoryStorageAdapter](Flowthru.Data.Storage.MemoryStorageAdapter\-1.md) + +#### Implements + +[IStorageAdapter](Flowthru.Data.Storage.IStorageAdapter\-1.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
// Singleton usage
+var modelStorage = new MemoryStorageAdapter<LinearRegressionModel>();
+var modelEntry = new Item<LinearRegressionModel>("model", modelStorage);
+
+// Collection usage
+var dataStorage = new MemoryStorageAdapter<IEnumerable<FeatureRow>>();
+var dataEntry = new Item<IEnumerable<FeatureRow>>("features", dataStorage);
+ +## Remarks + +

+Design Rationale: Memory storage doesn't need byte serialization +since objects stay in-process. This adapter provides direct Load/Save without +medium/format/container composition. +

+

+Use Cases: +

+
  • Intermediate pipeline data that doesn't need persistence
  • Test data that doesn't require file I/O
  • Temporary results between pipeline stages
  • ML models, metrics, charts, or any ephemeral data
+

+Thread Safety: Thread-safe for concurrent Load/Save operations +

+

+Lifetime: Data persists only for the lifetime of this instance +

+

+Storage Traits: +

+
  • IsPersistent: false (data lost when process exits)
  • All other traits use filesystem baseline defaults
+ +## Constructors + +### MemoryStorageAdapter\(\) + +Creates a new in-memory storage adapter. + +```csharp +public MemoryStorageAdapter() +``` + +### MemoryStorageAdapter\(T\) + +Creates a new in-memory storage adapter with initial data. + +```csharp +public MemoryStorageAdapter(T initialData) +``` + +#### Parameters + +`initialData` T + +Initial data to store + +## Properties + +### Traits + +Structural constraints and capabilities of this storage implementation. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +

+Adapter authors must declare what their storage can and cannot do. +These are intrinsic properties of the storage medium, not runtime state. +

+

+Pipeline validation uses these traits to fail fast when a pipeline attempts +invalid operations (e.g., writing to a read-only source). +

+ +## Methods + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+Delegates to the underlying medium's Exists check. +Used to determine if a catalog entry is a seed (Layer 0 input). +

+ +### InspectDeep\(\) + +Performs deep validation by examining the entire dataset. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that all data is available, accessible, and valid. +

+

+Additional Checks Beyond Shallow: +

+
  • Validate ALL rows can be deserialized (not just sample)
  • Check data quality constraints across entire dataset
  • Detect corruption or inconsistencies throughout data
+

+Implementation Guidelines: +

+
  • File adapters: Read and validate entire file
  • Memory adapters: Validate all stored data
  • Database adapters: Full table scan with validation
  • Null adapters: Always return success (no data required)
+

+Performance: Potentially expensive - only use when data integrity is critical. +

+ +### InspectShallow\(int\) + +Performs shallow validation by checking data availability and sampling a subset of data. + +```csharp +public FlowIO InspectShallow(int sampleSize) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that data is available and accessible. +

+

+Typical Checks: +

+
  • Data source exists (file, table, etc.)
  • Data source is accessible (permissions, connectivity)
  • Sample rows can be read and deserialized successfully
  • Schema matches expected structure
+

+Implementation Guidelines: +

+
  • File adapters: Check file exists, read and validate sample rows
  • Memory adapters: Check if data has been initialized
  • Database adapters: Check table exists, query sample rows
  • Null adapters: Always return success (no data required)
+

+Performance: Should be fast (~10-100ms) - suitable for pre-flight validation. +

+ +### Load\(\) + +Loads data from storage. + +```csharp +public FlowIO Load() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +Effect that produces data on success + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. medium.ReadStream()           → Stream
+2. format.DeserializeRows()      → IAsyncEnumerable<TRow>
+3. container.FromRows()          → TContainer
+

+Error Handling: +

+

+Errors from any layer are propagated: +- Medium errors (file not found, access denied) +- Format errors (parse failures, schema mismatches) +- Container errors (memory allocation, type conversion) +

+ +### Save\(T\) + +Saves data to storage. + +```csharp +public FlowIO Save(T data) +``` + +#### Parameters + +`data` T + +The data to save + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +Effect that completes on successful save + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. container.ToRows()            → IAsyncEnumerable<TRow>
+2. format.SerializeRows()        → Stream
+3. medium.WriteStream()          → FlowUnit
+

+Atomicity: +

+

+Implementations should strive for atomic saves to avoid partial writes on failure. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.NullStorageAdapter-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.NullStorageAdapter-1.md new file mode 100644 index 00000000..e1277d87 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.NullStorageAdapter-1.md @@ -0,0 +1,235 @@ +# Class NullStorageAdapter + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Null storage adapter for side-effect-only nodes that produce no meaningful data. + +```csharp +public sealed class NullStorageAdapter : IStorageAdapter +``` + +#### Type Parameters + +`T` + +The data type (typically NoData) + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[NullStorageAdapter](Flowthru.Data.Storage.NullStorageAdapter\-1.md) + +#### Implements + +[IStorageAdapter](Flowthru.Data.Storage.IStorageAdapter\-1.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Use Case: Nodes that perform side effects (logging, visualization, alerts) +but don't produce data that downstream nodes need. +

+

+Storage Traits: +

+
  • CanWrite: false (Save is a no-op)
  • CanRead: false (Load throws NotSupportedException)
+ +## Properties + +### Traits + +Structural constraints and capabilities of this storage implementation. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +

+Adapter authors must declare what their storage can and cannot do. +These are intrinsic properties of the storage medium, not runtime state. +

+

+Pipeline validation uses these traits to fail fast when a pipeline attempts +invalid operations (e.g., writing to a read-only source). +

+ +## Methods + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+Delegates to the underlying medium's Exists check. +Used to determine if a catalog entry is a seed (Layer 0 input). +

+ +### InspectDeep\(\) + +Performs deep validation by examining the entire dataset. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that all data is available, accessible, and valid. +

+

+Additional Checks Beyond Shallow: +

+
  • Validate ALL rows can be deserialized (not just sample)
  • Check data quality constraints across entire dataset
  • Detect corruption or inconsistencies throughout data
+

+Implementation Guidelines: +

+
  • File adapters: Read and validate entire file
  • Memory adapters: Validate all stored data
  • Database adapters: Full table scan with validation
  • Null adapters: Always return success (no data required)
+

+Performance: Potentially expensive - only use when data integrity is critical. +

+ +### InspectShallow\(int\) + +Performs shallow validation by checking data availability and sampling a subset of data. + +```csharp +public FlowIO InspectShallow(int sampleSize) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that data is available and accessible. +

+

+Typical Checks: +

+
  • Data source exists (file, table, etc.)
  • Data source is accessible (permissions, connectivity)
  • Sample rows can be read and deserialized successfully
  • Schema matches expected structure
+

+Implementation Guidelines: +

+
  • File adapters: Check file exists, read and validate sample rows
  • Memory adapters: Check if data has been initialized
  • Database adapters: Check table exists, query sample rows
  • Null adapters: Always return success (no data required)
+

+Performance: Should be fast (~10-100ms) - suitable for pre-flight validation. +

+ +### Load\(\) + +Loads data from storage. + +```csharp +public FlowIO Load() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +Effect that produces data on success + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. medium.ReadStream()           → Stream
+2. format.DeserializeRows()      → IAsyncEnumerable<TRow>
+3. container.FromRows()          → TContainer
+

+Error Handling: +

+

+Errors from any layer are propagated: +- Medium errors (file not found, access denied) +- Format errors (parse failures, schema mismatches) +- Container errors (memory allocation, type conversion) +

+ +### Save\(T\) + +Saves data to storage. + +```csharp +public FlowIO Save(T data) +``` + +#### Parameters + +`data` T + +The data to save + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +Effect that completes on successful save + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. container.ToRows()            → IAsyncEnumerable<TRow>
+2. format.SerializeRows()        → Stream
+3. medium.WriteStream()          → FlowUnit
+

+Atomicity: +

+

+Implementations should strive for atomic saves to avoid partial writes on failure. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.PropertyMappingConfiguration.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.PropertyMappingConfiguration.md new file mode 100644 index 00000000..b2159c5c --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.PropertyMappingConfiguration.md @@ -0,0 +1,181 @@ +# Class PropertyMappingConfiguration + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Describes how a format serializer handles property-to-field name mapping. + +```csharp +public sealed class PropertyMappingConfiguration +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PropertyMappingConfiguration](Flowthru.Data.Storage.PropertyMappingConfiguration.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Purpose: Makes property mapping strategy explicit and discoverable for each serializer. +

+

+Strategy Types: +

+
  • SerializedLabel - Respects [SerializedLabel] attributes
  • NativeAttributes - Uses format-specific attributes (e.g., [LoadColumn])
  • - Underlying library controls mapping
  • Adapter - Bridges SerializedLabel with native attributes
+ +## Properties + +### Description + +Optional description of the mapping behavior. + +```csharp +public string? Description { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### MetadataType + +For NativeAttributes strategy: the name of the attribute type(s) used. +For Adapter strategy: the adapter type. + +```csharp +public Type? MetadataType { get; } +``` + +#### Property Value + + [Type](https://learn.microsoft.com/dotnet/api/system.type)? + +### Strategy + +The strategy used for property mapping. + +```csharp +public PropertyMappingStrategy Strategy { get; } +``` + +#### Property Value + + [PropertyMappingStrategy](Flowthru.Data.Storage.PropertyMappingStrategy.md) + +### SupportsSerializedLabel + +Checks if the serializer supports SerializedLabel attributes (directly or via adapter). + +```csharp +public bool SupportsSerializedLabel { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### FromAdapter\(\) + +Serializer uses an adapter to bridge SerializedLabel with native attributes. + +```csharp +public static PropertyMappingConfiguration FromAdapter() +``` + +#### Returns + + [PropertyMappingConfiguration](Flowthru.Data.Storage.PropertyMappingConfiguration.md) + +Configuration for adapter-based mapping + +#### Type Parameters + +`TAdapter` + +The adapter type that performs the bridging + +### FromNativeAttributes\(string\) + +Serializer uses format-specific attributes (e.g., ML.NET's [LoadColumn], [ColumnName]). + +```csharp +public static PropertyMappingConfiguration FromNativeAttributes(string attributeDescription) +``` + +#### Parameters + +`attributeDescription` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Description of native attributes used + +#### Returns + + [PropertyMappingConfiguration](Flowthru.Data.Storage.PropertyMappingConfiguration.md) + +Configuration for native attribute mapping + +### FromSerializedLabel\(\) + +Serializer uses SerializedLabel attributes via PropertyMappingHelper. + +```csharp +public static PropertyMappingConfiguration FromSerializedLabel() +``` + +#### Returns + + [PropertyMappingConfiguration](Flowthru.Data.Storage.PropertyMappingConfiguration.md) + +Configuration for SerializedLabel-based mapping + +#### Type Parameters + +`TRow` + +The schema type + +### LibraryControlled\(string?\) + +Underlying library controls mapping with no programmatic API. +Property names must match storage field names exactly. + +```csharp +public static PropertyMappingConfiguration LibraryControlled(string? limitation = null) +``` + +#### Parameters + +`limitation` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Optional description of the limitation + +#### Returns + + [PropertyMappingConfiguration](Flowthru.Data.Storage.PropertyMappingConfiguration.md) + +Configuration for library-controlled mapping + +### ToString\(\) + +Gets a human-readable description of the mapping strategy. + +```csharp +public override string ToString() +``` + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.PropertyMappingStrategy.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.PropertyMappingStrategy.md new file mode 100644 index 00000000..b2f04f0f --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.PropertyMappingStrategy.md @@ -0,0 +1,37 @@ +# Enum PropertyMappingStrategy + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Property mapping strategy used by a format serializer. + +```csharp +public enum PropertyMappingStrategy +``` + +## Fields + +`Adapter = 3` + +Serializer uses an adapter to translate SerializedLabel to native attributes. + + + +`LibraryControlled = 2` + +Underlying library controls mapping with no programmatic access. + + + +`NativeAttributes = 1` + +Serializer uses format-specific attributes (e.g., ML.NET [LoadColumn], CsvHelper [Name]). + + + +`SerializedLabel = 0` + +Serializer respects [SerializedLabel] attributes using PropertyMappingHelper. + + + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.SchemaActivator.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.SchemaActivator.md new file mode 100644 index 00000000..b62c4266 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.SchemaActivator.md @@ -0,0 +1,114 @@ +# Class SchemaActivator + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Factory for creating schema instances, supporting both traditional parameterless constructors +and modern C# features like required members and positional records. + +```csharp +public static class SchemaActivator +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[SchemaActivator](Flowthru.Data.Storage.SchemaActivator.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
// Traditional schema - uses fast path
+public record OldSchema(int Id, string Name) : IFlatSchema;
+var old = SchemaActivator.CreateInstance<OldSchema>();
+
+// Modern schema with required members - uses slow path
+public record NewSchema : IFlatSchema
+{
+  public required Guid Id { get; init; }
+  public required string Name { get; init; }
+}
+var modern = SchemaActivator.CreateInstance<NewSchema>();
+ +## Remarks + +

+Design Philosophy: +

+

+With Flowthru's strong node contracts, schemas with required members are guaranteed to contain +valid data because: +

+
  • Layer 0 (Seeds): Validation phase checks required fields exist before execution
  • Layers 1+ (Node outputs): C# compiler enforces required members when nodes construct output
+

+This activator's role is to enable deserialization by creating instances that will be populated +via property reflection. No validation is performed here - that happens at the pipeline boundaries. +

+

+Instantiation Strategy: +

+
  • Fast Path: Parameterless constructor (uses compiled expression tree)
  • Slow Path: No parameterless constructor (uses FormatterServices.GetUninitializedObject)
+

+Performance: +

+

+The fast path (parameterless constructor) is ~10x faster than Activator.CreateInstance. +The slow path (uninitialized object) is ~2x slower than Activator.CreateInstance but enables +required members and positional records. +

+

+Both paths cache metadata to minimize reflection overhead on subsequent calls. +

+ +## Methods + +### CreateInstance\(\) + +Creates an instance of the specified type, automatically selecting the optimal +instantiation strategy. + +```csharp +public static T CreateInstance() where T : notnull +``` + +#### Returns + + T + +A new instance of type T with uninitialized properties + +#### Type Parameters + +`T` + +The type to instantiate + +#### Remarks + +

+The returned instance will have: +- Reference type properties: null +- Value type properties: default values (0, false, etc.) +- Required members: uninitialized (will be set via reflection after this call) +

+

+This is safe because: +- Layer 0: Validation ensures required fields exist in data +- Layers 1+: Data came from valid node output +

+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if the type cannot be instantiated (e.g., abstract class, interface) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.SingletonJsonStorageAdapter-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.SingletonJsonStorageAdapter-1.md new file mode 100644 index 00000000..824cf1fe --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.SingletonJsonStorageAdapter-1.md @@ -0,0 +1,313 @@ +# Class SingletonJsonStorageAdapter + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Direct JSON file storage for singleton objects (not collections). + +```csharp +public sealed class SingletonJsonStorageAdapter : IStorageAdapter where T : IStructuredSerializable +``` + +#### Type Parameters + +`T` + +The object type to serialize + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[SingletonJsonStorageAdapter](Flowthru.Data.Storage.SingletonJsonStorageAdapter\-1.md) + +#### Implements + +[IStorageAdapter](Flowthru.Data.Storage.IStorageAdapter\-1.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
var storage = new SingletonJsonStorageAdapter<LinearRegressionModel>("model.json");
+var entry = new Item<LinearRegressionModel>("model", storage);
+
+// Save
+await entry.Save(model).RunAsync();
+
+// Load
+var loadedModel = await entry.Load().RunAsync();
+ +## Remarks + +

+Design Rationale: Singleton objects don't need the full +medium/format/container composition since they don't stream rows. This adapter +provides direct JSON serialization for single objects. +

+

+Use Cases: +

+
  • ML models (LinearRegressionModel)
  • Metrics objects (ModelMetrics, CrossValidationResults)
  • Configuration files
  • Any single object (not a collection)
+

+Serialization Format: JSON object (not wrapped in array) +

+

+Storage Traits: All traits use filesystem baseline defaults +

+ +## Constructors + +### SingletonJsonStorageAdapter\(string\) + +Creates a new singleton JSON storage adapter with default options. +Uses JsonFormatSerializer's default options to ensure consistent behavior, +including SerializedLabel attribute support. + +```csharp +public SingletonJsonStorageAdapter(string filePath) +``` + +#### Parameters + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to JSON file + +### SingletonJsonStorageAdapter\(string, JsonSerializerOptions\) + +Creates a new singleton JSON storage adapter with custom options. + +```csharp +public SingletonJsonStorageAdapter(string filePath, JsonSerializerOptions options) +``` + +#### Parameters + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to JSON file + +`options` [JsonSerializerOptions](https://learn.microsoft.com/dotnet/api/system.text.json.jsonserializeroptions) + +JSON serialization options + +## Properties + +### FilePath + +Gets the file path used by this adapter. + +```csharp +public string FilePath { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Options + +Gets the JSON serialization options. + +```csharp +public JsonSerializerOptions Options { get; } +``` + +#### Property Value + + [JsonSerializerOptions](https://learn.microsoft.com/dotnet/api/system.text.json.jsonserializeroptions) + +### Traits + +Structural constraints and capabilities of this storage implementation. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +

+Adapter authors must declare what their storage can and cannot do. +These are intrinsic properties of the storage medium, not runtime state. +

+

+Pipeline validation uses these traits to fail fast when a pipeline attempts +invalid operations (e.g., writing to a read-only source). +

+ +## Methods + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+Delegates to the underlying medium's Exists check. +Used to determine if a catalog entry is a seed (Layer 0 input). +

+ +### InspectDeep\(\) + +Performs deep validation by examining the entire dataset. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that all data is available, accessible, and valid. +

+

+Additional Checks Beyond Shallow: +

+
  • Validate ALL rows can be deserialized (not just sample)
  • Check data quality constraints across entire dataset
  • Detect corruption or inconsistencies throughout data
+

+Implementation Guidelines: +

+
  • File adapters: Read and validate entire file
  • Memory adapters: Validate all stored data
  • Database adapters: Full table scan with validation
  • Null adapters: Always return success (no data required)
+

+Performance: Potentially expensive - only use when data integrity is critical. +

+ +### InspectShallow\(int\) + +Performs shallow validation by checking data availability and sampling a subset of data. + +```csharp +public FlowIO InspectShallow(int sampleSize) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that data is available and accessible. +

+

+Typical Checks: +

+
  • Data source exists (file, table, etc.)
  • Data source is accessible (permissions, connectivity)
  • Sample rows can be read and deserialized successfully
  • Schema matches expected structure
+

+Implementation Guidelines: +

+
  • File adapters: Check file exists, read and validate sample rows
  • Memory adapters: Check if data has been initialized
  • Database adapters: Check table exists, query sample rows
  • Null adapters: Always return success (no data required)
+

+Performance: Should be fast (~10-100ms) - suitable for pre-flight validation. +

+ +### Load\(\) + +Loads data from storage. + +```csharp +public FlowIO Load() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +Effect that produces data on success + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. medium.ReadStream()           → Stream
+2. format.DeserializeRows()      → IAsyncEnumerable<TRow>
+3. container.FromRows()          → TContainer
+

+Error Handling: +

+

+Errors from any layer are propagated: +- Medium errors (file not found, access denied) +- Format errors (parse failures, schema mismatches) +- Container errors (memory allocation, type conversion) +

+ +### Save\(T\) + +Saves data to storage. + +```csharp +public FlowIO Save(T data) +``` + +#### Parameters + +`data` T + +The data to save + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +Effect that completes on successful save + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. container.ToRows()            → IAsyncEnumerable<TRow>
+2. format.SerializeRows()        → Stream
+3. medium.WriteStream()          → FlowUnit
+

+Atomicity: +

+

+Implementations should strive for atomic saves to avoid partial writes on failure. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.DatabaseStorageEntryFactory.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.DatabaseStorageEntryFactory.md new file mode 100644 index 00000000..e77ad455 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.DatabaseStorageEntryFactory.md @@ -0,0 +1,190 @@ +# Class DatabaseStorageEntryFactory + +Namespace: [Flowthru.Data.Storage.Strategies](Flowthru.Data.Storage.Strategies.md) +Assembly: Flowthru.Core.dll + +Database-backed storage strategy for production environments. + +```csharp +public sealed class DatabaseStorageEntryFactory : IStorageEntryFactory +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[DatabaseStorageEntryFactory](Flowthru.Data.Storage.Strategies.DatabaseStorageEntryFactory.md) + +#### Implements + +[IStorageEntryFactory](Flowthru.Data.Storage.Strategies.IStorageEntryFactory.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+⚠️ STUB IMPLEMENTATION - Phase 2 +

+

+This is a placeholder for future database support. Currently throws +NotImplementedException for all operations. +

+

+Planned Features: +

+
  • SQL Server, PostgreSQL, SQLite support
  • Connection pooling and retry logic
  • Schema migration support
  • Transaction coordination with pipelines
+

+Proposed Usage: +

+
services.AddFlowthru(flowthru =>
+{
+    flowthru.RegisterCatalog<MyCatalog>();
+
+    if (env.IsProduction())
+    {
+        flowthru.UseStorageStrategy<DatabaseStorageEntryFactory>();
+    }
+});
+ +## Constructors + +### DatabaseStorageEntryFactory\(IConfiguration\) + +Initializes a new database storage factory. + +```csharp +public DatabaseStorageEntryFactory(IConfiguration configuration) +``` + +#### Parameters + +`configuration` [IConfiguration](https://learn.microsoft.com/dotnet/api/microsoft.extensions.configuration.iconfiguration) + +Configuration containing connection string + +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if database connection string is not configured + +### DatabaseStorageEntryFactory\(string, string\) + +Initializes a new database storage factory with explicit settings. + +```csharp +public DatabaseStorageEntryFactory(string connectionString, string schema = "dbo") +``` + +#### Parameters + +`connectionString` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Database connection string + +`schema` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Default schema for tables + +## Methods + +### CreateEnumerable\(string, StorageOptions?\) + +Creates a catalog entry for an enumerable dataset. + +```csharp +public IItem> CreateEnumerable(string label, StorageOptions? options = null) where T : notnull, IFlatSchema, ITextSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Catalog label for the entry + +`options` [StorageOptions](Flowthru.Data.Storage.Strategies.StorageOptions.md)? + +Optional storage options + +#### Returns + + [IItem](Flowthru.Data.IItem\-1.md)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Configured catalog entry + +#### Type Parameters + +`T` + +Schema type (must implement IFlatSchema and ITextSerializable) + +#### Remarks + +

+Type constraints ensure compatibility with CSV and Parquet serialization. +Memory storage also works since it has no serialization requirements. +

+

+If options.Path is null, the label is used to derive a default path +(e.g., "Companies" → "Companies.csv" or "dbo.Companies"). +

+ +#### Exceptions + + [NotImplementedException](https://learn.microsoft.com/dotnet/api/system.notimplementedexception) + +Phase 2 stub - database support not yet implemented + +### CreateSingle\(string, StorageOptions?\) + +Creates a catalog entry for a singleton object. + +```csharp +public IItem CreateSingle(string label, StorageOptions? options = null) where T : IStructuredSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Catalog label for the entry + +`options` [StorageOptions](Flowthru.Data.Storage.Strategies.StorageOptions.md)? + +Optional storage options + +#### Returns + + [IItem](Flowthru.Data.IItem\-1.md) + +Configured catalog entry + +#### Type Parameters + +`T` + +Object type (must implement IStructuredSerializable) + +#### Remarks + +

+Type constraint ensures compatibility with JSON serialization. +Memory storage also works since it has no serialization requirements. +

+

+Typically uses structured formats (JSON, MessagePack) for singletons. +

+ +#### Exceptions + + [NotImplementedException](https://learn.microsoft.com/dotnet/api/system.notimplementedexception) + +Phase 2 stub - database support not yet implemented + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.IStorageEntryFactory.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.IStorageEntryFactory.md new file mode 100644 index 00000000..e0fdb794 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.IStorageEntryFactory.md @@ -0,0 +1,118 @@ +# Interface IStorageEntryFactory + +Namespace: [Flowthru.Data.Storage.Strategies](Flowthru.Data.Storage.Strategies.md) +Assembly: Flowthru.Core.dll + +Factory for creating catalog entries with environment-specific storage. + +```csharp +public interface IStorageEntryFactory +``` + +## Remarks + +

+The strategy pattern enables the same catalog to use different storage +backends based on the environment: +

+
  • Development: CSV files for easy inspection and version control
  • Production: Database tables for scalability and transactions
  • Testing: In-memory storage for fast, isolated tests
+

+Usage Pattern: +

+
public class MyCatalog : DataCatalogBase
+{
+    private readonly IStorageEntryFactory _storage;
+
+    public MyCatalog(IStorageEntryFactory storage)
+    {
+        _storage = storage;
+        InitializeCatalogProperties();
+    }
+
+    public IItem<IEnumerable<Company>> Companies =>
+        GetOrCreateEntry(() => _storage.CreateEnumerable<Company>("Companies"));
+}
+ +## Methods + +### CreateEnumerable\(string, StorageOptions?\) + +Creates a catalog entry for an enumerable dataset. + +```csharp +IItem> CreateEnumerable(string label, StorageOptions? options = null) where T : notnull, IFlatSchema, ITextSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Catalog label for the entry + +`options` [StorageOptions](Flowthru.Data.Storage.Strategies.StorageOptions.md)? + +Optional storage options + +#### Returns + + [IItem](Flowthru.Data.IItem\-1.md)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Configured catalog entry + +#### Type Parameters + +`T` + +Schema type (must implement IFlatSchema and ITextSerializable) + +#### Remarks + +

+Type constraints ensure compatibility with CSV and Parquet serialization. +Memory storage also works since it has no serialization requirements. +

+

+If options.Path is null, the label is used to derive a default path +(e.g., "Companies" → "Companies.csv" or "dbo.Companies"). +

+ +### CreateSingle\(string, StorageOptions?\) + +Creates a catalog entry for a singleton object. + +```csharp +IItem CreateSingle(string label, StorageOptions? options = null) where T : IStructuredSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Catalog label for the entry + +`options` [StorageOptions](Flowthru.Data.Storage.Strategies.StorageOptions.md)? + +Optional storage options + +#### Returns + + [IItem](Flowthru.Data.IItem\-1.md) + +Configured catalog entry + +#### Type Parameters + +`T` + +Object type (must implement IStructuredSerializable) + +#### Remarks + +

+Type constraint ensures compatibility with JSON serialization. +Memory storage also works since it has no serialization requirements. +

+

+Typically uses structured formats (JSON, MessagePack) for singletons. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.MemoryStorageEntryFactory.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.MemoryStorageEntryFactory.md new file mode 100644 index 00000000..4c209690 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.MemoryStorageEntryFactory.md @@ -0,0 +1,132 @@ +# Class MemoryStorageEntryFactory + +Namespace: [Flowthru.Data.Storage.Strategies](Flowthru.Data.Storage.Strategies.md) +Assembly: Flowthru.Core.dll + +In-memory storage strategy for unit tests. + +```csharp +public sealed class MemoryStorageEntryFactory : IStorageEntryFactory +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[MemoryStorageEntryFactory](Flowthru.Data.Storage.Strategies.MemoryStorageEntryFactory.md) + +#### Implements + +[IStorageEntryFactory](Flowthru.Data.Storage.Strategies.IStorageEntryFactory.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Stores all data in memory for: +

+
  • Fast test execution (no I/O)
  • Test isolation (no shared state between tests)
  • Simple setup (no files or databases)
+

+Usage in Tests: +

+
[Test]
+public async Task MyTest()
+{
+    var storage = new MemoryStorageEntryFactory();
+    var catalog = new MyCatalog(storage);
+
+    // All data stays in memory - no files created
+    await catalog.Companies.Save(companies).Run();
+    var result = await catalog.Companies.Load().Run();
+}
+ +## Methods + +### CreateEnumerable\(string, StorageOptions?\) + +Creates a catalog entry for an enumerable dataset. + +```csharp +public IItem> CreateEnumerable(string label, StorageOptions? options = null) where T : notnull, IFlatSchema, ITextSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Catalog label for the entry + +`options` [StorageOptions](Flowthru.Data.Storage.Strategies.StorageOptions.md)? + +Optional storage options + +#### Returns + + [IItem](Flowthru.Data.IItem\-1.md)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Configured catalog entry + +#### Type Parameters + +`T` + +Schema type (must implement IFlatSchema and ITextSerializable) + +#### Remarks + +

+Type constraints ensure compatibility with CSV and Parquet serialization. +Memory storage also works since it has no serialization requirements. +

+

+If options.Path is null, the label is used to derive a default path +(e.g., "Companies" → "Companies.csv" or "dbo.Companies"). +

+ +### CreateSingle\(string, StorageOptions?\) + +Creates a catalog entry for a singleton object. + +```csharp +public IItem CreateSingle(string label, StorageOptions? options = null) where T : IStructuredSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Catalog label for the entry + +`options` [StorageOptions](Flowthru.Data.Storage.Strategies.StorageOptions.md)? + +Optional storage options + +#### Returns + + [IItem](Flowthru.Data.IItem\-1.md) + +Configured catalog entry + +#### Type Parameters + +`T` + +Object type (must implement IStructuredSerializable) + +#### Remarks + +

+Type constraint ensures compatibility with JSON serialization. +Memory storage also works since it has no serialization requirements. +

+

+Typically uses structured formats (JSON, MessagePack) for singletons. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.StorageOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.StorageOptions.md new file mode 100644 index 00000000..e5213011 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.StorageOptions.md @@ -0,0 +1,92 @@ +# Class StorageOptions + +Namespace: [Flowthru.Data.Storage.Strategies](Flowthru.Data.Storage.Strategies.md) +Assembly: Flowthru.Core.dll + +Options for configuring storage entry creation. + +```csharp +public sealed record StorageOptions : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[StorageOptions](Flowthru.Data.Storage.Strategies.StorageOptions.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Provides flexibility for strategy-specific configuration without +requiring changes to the factory interface. + +## Properties + +### Metadata + +Additional strategy-specific metadata. + +```csharp +public Dictionary? Metadata { get; init; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [object](https://learn.microsoft.com/dotnet/api/system.object)\>? + +#### Remarks + +Examples: +- Excel: {"SheetName": "Data"} +- Database: {"Schema": "analytics", "Timeout": 30} +- Parquet: {"CompressionCodec": "SNAPPY"} + +### Path + +Relative path or identifier for the storage location. + +```csharp +public string? Path { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +Interpretation depends on the strategy: +- CSV: File path relative to base directory (e.g., "_01_Raw/data.csv") +- Database: Table name or qualified identifier (e.g., "dbo.Companies") +- Memory: Ignored (memory storage has no path) + +## Methods + +### WithPath\(string\) + +Creates storage options with a path. + +```csharp +public static StorageOptions WithPath(string path) +``` + +#### Parameters + +`path` [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Returns + + [StorageOptions](Flowthru.Data.Storage.Strategies.StorageOptions.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.md new file mode 100644 index 00000000..f312ae89 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.Strategies.md @@ -0,0 +1,22 @@ +# Namespace Flowthru.Data.Storage.Strategies + +### Classes + + [DatabaseStorageEntryFactory](Flowthru.Data.Storage.Strategies.DatabaseStorageEntryFactory.md) + +Database-backed storage strategy for production environments. + + [MemoryStorageEntryFactory](Flowthru.Data.Storage.Strategies.MemoryStorageEntryFactory.md) + +In-memory storage strategy for unit tests. + + [StorageOptions](Flowthru.Data.Storage.Strategies.StorageOptions.md) + +Options for configuring storage entry creation. + +### Interfaces + + [IStorageEntryFactory](Flowthru.Data.Storage.Strategies.IStorageEntryFactory.md) + +Factory for creating catalog entries with environment-specific storage. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.TextFileStorageAdapter.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.TextFileStorageAdapter.md new file mode 100644 index 00000000..b977427f --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.TextFileStorageAdapter.md @@ -0,0 +1,239 @@ +# Class TextFileStorageAdapter + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Core.dll + +Storage adapter for plain text files with string content. + +```csharp +public sealed class TextFileStorageAdapter : IStorageAdapter +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[TextFileStorageAdapter](Flowthru.Data.Storage.TextFileStorageAdapter.md) + +#### Implements + +[IStorageAdapter](Flowthru.Data.Storage.IStorageAdapter\-1.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Use Cases: Markdown reports, configuration files, plain text logs +

+

+Storage Traits: All traits use filesystem baseline defaults +

+ +## Constructors + +### TextFileStorageAdapter\(string\) + +```csharp +public TextFileStorageAdapter(string filePath) +``` + +#### Parameters + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +## Properties + +### Traits + +Structural constraints and capabilities of this storage implementation. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md) + +#### Remarks + +

+Adapter authors must declare what their storage can and cannot do. +These are intrinsic properties of the storage medium, not runtime state. +

+

+Pipeline validation uses these traits to fail fast when a pipeline attempts +invalid operations (e.g., writing to a read-only source). +

+ +## Methods + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+Delegates to the underlying medium's Exists check. +Used to determine if a catalog entry is a seed (Layer 0 input). +

+ +### InspectDeep\(\) + +Performs deep validation by examining the entire dataset. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that all data is available, accessible, and valid. +

+

+Additional Checks Beyond Shallow: +

+
  • Validate ALL rows can be deserialized (not just sample)
  • Check data quality constraints across entire dataset
  • Detect corruption or inconsistencies throughout data
+

+Implementation Guidelines: +

+
  • File adapters: Read and validate entire file
  • Memory adapters: Validate all stored data
  • Database adapters: Full table scan with validation
  • Null adapters: Always return success (no data required)
+

+Performance: Potentially expensive - only use when data integrity is critical. +

+ +### InspectShallow\(int\) + +Performs shallow validation by checking data availability and sampling a subset of data. + +```csharp +public FlowIO InspectShallow(int sampleSize) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that data is available and accessible. +

+

+Typical Checks: +

+
  • Data source exists (file, table, etc.)
  • Data source is accessible (permissions, connectivity)
  • Sample rows can be read and deserialized successfully
  • Schema matches expected structure
+

+Implementation Guidelines: +

+
  • File adapters: Check file exists, read and validate sample rows
  • Memory adapters: Check if data has been initialized
  • Database adapters: Check table exists, query sample rows
  • Null adapters: Always return success (no data required)
+

+Performance: Should be fast (~10-100ms) - suitable for pre-flight validation. +

+ +### Load\(\) + +Loads data from storage. + +```csharp +public FlowIO Load() +``` + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +Effect that produces data on success + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. medium.ReadStream()           → Stream
+2. format.DeserializeRows()      → IAsyncEnumerable<TRow>
+3. container.FromRows()          → TContainer
+

+Error Handling: +

+

+Errors from any layer are propagated: +- Medium errors (file not found, access denied) +- Format errors (parse failures, schema mismatches) +- Container errors (memory allocation, type conversion) +

+ +### Save\(string\) + +Saves data to storage. + +```csharp +public FlowIO Save(string data) +``` + +#### Parameters + +`data` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The data to save + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[FlowUnit](Flowthru.Effects.FlowUnit.md)\> + +Effect that completes on successful save + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. container.ToRows()            → IAsyncEnumerable<TRow>
+2. format.SerializeRows()        → Stream
+3. medium.WriteStream()          → FlowUnit
+

+Atomicity: +

+

+Implementations should strive for atomic saves to avoid partial writes on failure. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.md new file mode 100644 index 00000000..91631031 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Storage.md @@ -0,0 +1,71 @@ +# Namespace Flowthru.Data.Storage + +### Namespaces + + [Flowthru.Data.Storage.Container](Flowthru.Data.Storage.Container.md) + + [Flowthru.Data.Storage.Format](Flowthru.Data.Storage.Format.md) + + [Flowthru.Data.Storage.Medium](Flowthru.Data.Storage.Medium.md) + + [Flowthru.Data.Storage.Strategies](Flowthru.Data.Storage.Strategies.md) + +### Classes + + [BinaryFileStorageAdapter](Flowthru.Data.Storage.BinaryFileStorageAdapter.md) + +Storage adapter for binary files with byte array content. + + [ComposedStorageAdapter](Flowthru.Data.Storage.ComposedStorageAdapter\-2.md) + +Composed storage adapter that delegates to medium, format, and container layers. + + [MemoryStorageAdapter](Flowthru.Data.Storage.MemoryStorageAdapter\-1.md) + +Direct memory storage adapter that bypasses serialization. + + [NullStorageAdapter](Flowthru.Data.Storage.NullStorageAdapter\-1.md) + +Null storage adapter for side-effect-only nodes that produce no meaningful data. + + [PropertyMappingConfiguration](Flowthru.Data.Storage.PropertyMappingConfiguration.md) + +Describes how a format serializer handles property-to-field name mapping. + + [SchemaActivator](Flowthru.Data.Storage.SchemaActivator.md) + +Factory for creating schema instances, supporting both traditional parameterless constructors +and modern C# features like required members and positional records. + + [SingletonJsonStorageAdapter](Flowthru.Data.Storage.SingletonJsonStorageAdapter\-1.md) + +Direct JSON file storage for singleton objects (not collections). + + [TextFileStorageAdapter](Flowthru.Data.Storage.TextFileStorageAdapter.md) + +Storage adapter for plain text files with string content. + +### Interfaces + + [IContainerAdapter](Flowthru.Data.Storage.IContainerAdapter\-2.md) + +Interface for container adaptation - converts between streaming rows and in-memory containers. + + [IFormatSerializer](Flowthru.Data.Storage.IFormatSerializer\-1.md) + +Interface for format serialization - handles row-based serialization/deserialization. + + [IStorageAdapter](Flowthru.Data.Storage.IStorageAdapter\-1.md) + +Interface for high-level storage operations - abstracts Load/Save with any storage implementation. + + [IStorageMedium](Flowthru.Data.Storage.IStorageMedium.md) + +Interface for storage medium - handles raw byte stream I/O. + +### Enums + + [PropertyMappingStrategy](Flowthru.Data.Storage.PropertyMappingStrategy.md) + +Property mapping strategy used by a format serializer. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.InspectionLevel.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.InspectionLevel.md new file mode 100644 index 00000000..b7104f61 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.InspectionLevel.md @@ -0,0 +1,80 @@ +# Enum InspectionLevel + +Namespace: [Flowthru.Data.Validation](Flowthru.Data.Validation.md) +Assembly: Flowthru.Core.dll + +Defines the level of inspection to perform on catalog entries before pipeline execution. + +```csharp +public enum InspectionLevel +``` + +## Fields + +`Deep = 2` + +Perform deep inspection: all checks from Shallow plus validation of ALL rows. + +

+What Deep Inspection Adds: +

+
  • All checks from Shallow inspection
  • Validates EVERY row deserializes successfully
  • Checks for data quality issues throughout entire dataset
+

+Performance: Potentially significant overhead (seconds to minutes for large datasets) +

+

+Use Cases: +

+
  • Critical production deployments
  • After external data updates
  • CI/CD regression testing
  • When data corruption is suspected
+

+Must be explicitly opted-in by the pipeline creator. +

+ +`None = 0` + +Skip inspection entirely. + +Use when: +- Data source is trusted and validated externally +- Validation overhead is prohibitive for large datasets +- You're explicitly opting out of safety checks + +`Shallow = 1` + +Perform shallow inspection: existence, format, headers, and sample rows. + +

+What Shallow Inspection Checks: +

+
  • File/resource exists
  • Format is valid (parseable as CSV/Excel/Parquet/etc.)
  • Headers match expected schema (column names, property mappings)
  • First N rows (default: 100) deserialize successfully
  • Data types are compatible with schema
+

+Performance: Minimal overhead (~10-100ms for typical files) +

+

+This is the default for all Layer 0 inputs. +

+ +## Remarks + +

+Inspection levels are used to validate external data sources (Layer 0 inputs) before +the pipeline begins execution, following the "fail-fast" principle. +

+

+Default Behavior: +- External inputs (Layer 0): Shallow inspection (all storage adapters support inspection) +- Intermediate outputs (Layer 1+): Always None (not inspected, they're created by the pipeline) +

+

+When to Use Each Level: +

+
  • None +Skip validation entirely. Use for trusted data sources or when validation overhead is prohibitive. +
  • Shallow +Validate existence, format, headers, and a sample of rows. Fast and catches most common issues. +This is the default for Layer 0 inputs. +
  • Deep +Validate all rows in the dataset. Thorough but expensive. Use for critical data or after +external updates. Must be explicitly opted-in. +
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationError.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationError.md new file mode 100644 index 00000000..6e1d8e5a --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationError.md @@ -0,0 +1,132 @@ +# Class ValidationError + +Namespace: [Flowthru.Data.Validation](Flowthru.Data.Validation.md) +Assembly: Flowthru.Core.dll + +Represents a single validation error discovered during catalog entry inspection. + +```csharp +public class ValidationError +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ValidationError](Flowthru.Data.Validation.ValidationError.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +ValidationError provides structured information about what went wrong during +inspection, making it easier to diagnose and fix data issues. + +## Constructors + +### ValidationError\(string, ValidationErrorType, string, string?\) + +Creates a new validation error. + +```csharp +public ValidationError(string catalogKey, ValidationErrorType errorType, string message, string? details = null) +``` + +#### Parameters + +`catalogKey` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The catalog entry key where the error occurred + +`errorType` [ValidationErrorType](Flowthru.Data.Validation.ValidationErrorType.md) + +The category of error + +`message` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Human-readable description of the error + +`details` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Optional additional context (file path, row number, column name, etc.) + +## Properties + +### CatalogKey + +The catalog entry key where the error occurred. + +```csharp +public string CatalogKey { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Details + +Optional additional context about the error. + +```csharp +public string? Details { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +May contain: +- File path +- Row number +- Column name +- Expected vs actual values +- Stack trace (for exceptions) + +### ErrorType + +The category of error that occurred. + +```csharp +public ValidationErrorType ErrorType { get; } +``` + +#### Property Value + + [ValidationErrorType](Flowthru.Data.Validation.ValidationErrorType.md) + +### Message + +Human-readable description of the error. + +```csharp +public string Message { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +## Methods + +### ToString\(\) + +Returns a formatted string representation of the error. + +```csharp +public override string ToString() +``` + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationErrorType.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationErrorType.md new file mode 100644 index 00000000..acf4346a --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationErrorType.md @@ -0,0 +1,55 @@ +# Enum ValidationErrorType + +Namespace: [Flowthru.Data.Validation](Flowthru.Data.Validation.md) +Assembly: Flowthru.Core.dll + +Categories of validation errors that can occur during catalog entry inspection. + +```csharp +public enum ValidationErrorType +``` + +## Fields + +`DeserializationError = 4` + +A row failed to deserialize (missing required field, invalid value, etc.). + + + +`EmptyDataset = 5` + +The data source is empty when data was expected. + + + +`InspectionFailure = 6` + +An unexpected exception occurred during inspection. + + + +`InvalidFormat = 1` + +The data format is invalid or corrupted (malformed CSV, corrupt Parquet, etc.). + + + +`NotFound = 0` + +The data source does not exist (file not found, URL unreachable, etc.). + + + +`SchemaMismatch = 2` + +Headers or column names don't match the expected schema. + + + +`TypeMismatch = 3` + +Data types in the source don't match the expected types. + + + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationException.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationException.md new file mode 100644 index 00000000..378837cc --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationException.md @@ -0,0 +1,78 @@ +# Class ValidationException + +Namespace: [Flowthru.Data.Validation](Flowthru.Data.Validation.md) +Assembly: Flowthru.Core.dll + +Exception thrown when catalog entry validation fails. + +```csharp +public class ValidationException : Exception, ISerializable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Exception](https://learn.microsoft.com/dotnet/api/system.exception) ← +[ValidationException](Flowthru.Data.Validation.ValidationException.md) + +#### Implements + +[ISerializable](https://learn.microsoft.com/dotnet/api/system.runtime.serialization.iserializable) + +#### Inherited Members + +[Exception.GetBaseException\(\)](https://learn.microsoft.com/dotnet/api/system.exception.getbaseexception), +[Exception.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.exception.gettype), +[Exception.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.exception.tostring), +[Exception.Data](https://learn.microsoft.com/dotnet/api/system.exception.data), +[Exception.HelpLink](https://learn.microsoft.com/dotnet/api/system.exception.helplink), +[Exception.HResult](https://learn.microsoft.com/dotnet/api/system.exception.hresult), +[Exception.InnerException](https://learn.microsoft.com/dotnet/api/system.exception.innerexception), +[Exception.Message](https://learn.microsoft.com/dotnet/api/system.exception.message), +[Exception.Source](https://learn.microsoft.com/dotnet/api/system.exception.source), +[Exception.StackTrace](https://learn.microsoft.com/dotnet/api/system.exception.stacktrace), +[Exception.TargetSite](https://learn.microsoft.com/dotnet/api/system.exception.targetsite), +[Exception.SerializeObjectState](https://learn.microsoft.com/dotnet/api/system.exception.serializeobjectstate), +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +This exception is thrown by to halt +pipeline execution when external data validation fails. + +## Constructors + +### ValidationException\(ValidationResult\) + +Creates a new validation exception. + +```csharp +public ValidationException(ValidationResult validationResult) +``` + +#### Parameters + +`validationResult` [ValidationResult](Flowthru.Data.Validation.ValidationResult.md) + +The validation result containing errors + +## Properties + +### ValidationResult + +The validation result containing all errors. + +```csharp +public ValidationResult ValidationResult { get; } +``` + +#### Property Value + + [ValidationResult](Flowthru.Data.Validation.ValidationResult.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationResult.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationResult.md new file mode 100644 index 00000000..c0afe962 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.ValidationResult.md @@ -0,0 +1,200 @@ +# Class ValidationResult + +Namespace: [Flowthru.Data.Validation](Flowthru.Data.Validation.md) +Assembly: Flowthru.Core.dll + +Represents the result of inspecting one or more catalog entries. + +```csharp +public class ValidationResult +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ValidationResult](Flowthru.Data.Validation.ValidationResult.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+ValidationResult provides a structured way to collect and report validation errors +discovered during catalog entry inspection. It supports both single-entry and +multi-entry validation scenarios. +

+ +## Constructors + +### ValidationResult\(\) + +Creates a successful validation result with no errors. + +```csharp +public ValidationResult() +``` + +### ValidationResult\(IEnumerable\) + +Creates a validation result with the specified errors. + +```csharp +public ValidationResult(IEnumerable errors) +``` + +#### Parameters + +`errors` [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)<[ValidationError](Flowthru.Data.Validation.ValidationError.md)\> + +Collection of validation errors + +## Properties + +### ErrorCount + +Number of validation errors found. + +```csharp +public int ErrorCount { get; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### Errors + +Read-only collection of all validation errors. + +```csharp +public IReadOnlyList Errors { get; } +``` + +#### Property Value + + [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[ValidationError](Flowthru.Data.Validation.ValidationError.md)\> + +### HasErrors + +True if one or more validation errors were found. + +```csharp +public bool HasErrors { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### IsValid + +True if no validation errors were found. + +```csharp +public bool IsValid { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### Failure\(string, ValidationErrorType, string, string?\) + +Creates a failed validation result with a single error. + +```csharp +public static ValidationResult Failure(string catalogKey, ValidationErrorType errorType, string message, string? details = null) +``` + +#### Parameters + +`catalogKey` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The catalog entry key where the error occurred + +`errorType` [ValidationErrorType](Flowthru.Data.Validation.ValidationErrorType.md) + +The category of error + +`message` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Human-readable description of the error + +`details` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Optional additional context + +#### Returns + + [ValidationResult](Flowthru.Data.Validation.ValidationResult.md) + +### FromException\(string, Exception\) + +Creates a failed validation result from an exception. + +```csharp +public static ValidationResult FromException(string catalogKey, Exception exception) +``` + +#### Parameters + +`catalogKey` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The catalog entry key where the error occurred + +`exception` [Exception](https://learn.microsoft.com/dotnet/api/system.exception) + +The exception that occurred during inspection + +#### Returns + + [ValidationResult](Flowthru.Data.Validation.ValidationResult.md) + +### Success\(\) + +Creates a successful validation result. + +```csharp +public static ValidationResult Success() +``` + +#### Returns + + [ValidationResult](Flowthru.Data.Validation.ValidationResult.md) + +### ThrowIfInvalid\(\) + +Throws a ValidationException if this result has errors. + +```csharp +public void ThrowIfInvalid() +``` + +#### Exceptions + + [ValidationException](Flowthru.Data.Validation.ValidationException.md) + +Thrown if validation failed + +### ToString\(\) + +Returns a formatted string representation of all errors. + +```csharp +public override string ToString() +``` + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.md new file mode 100644 index 00000000..bb688bd6 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Validation.md @@ -0,0 +1,26 @@ +# Namespace Flowthru.Data.Validation + +### Classes + + [ValidationError](Flowthru.Data.Validation.ValidationError.md) + +Represents a single validation error discovered during catalog entry inspection. + + [ValidationException](Flowthru.Data.Validation.ValidationException.md) + +Exception thrown when catalog entry validation fails. + + [ValidationResult](Flowthru.Data.Validation.ValidationResult.md) + +Represents the result of inspecting one or more catalog entries. + +### Enums + + [InspectionLevel](Flowthru.Data.Validation.InspectionLevel.md) + +Defines the level of inspection to perform on catalog entries before pipeline execution. + + [ValidationErrorType](Flowthru.Data.Validation.ValidationErrorType.md) + +Categories of validation errors that can occur during catalog entry inspection. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.md new file mode 100644 index 00000000..4f6882af --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.md @@ -0,0 +1,43 @@ +# Namespace Flowthru.Data + +### Namespaces + + [Flowthru.Data.Capabilities](Flowthru.Data.Capabilities.md) + + [Flowthru.Data.Storage](Flowthru.Data.Storage.md) + + [Flowthru.Data.Validation](Flowthru.Data.Validation.md) + +### Classes + + [CatalogAbstract](Flowthru.Data.CatalogAbstract.md) + +Base class for strongly-typed catalog implementations with automatic property caching. + + [EnumerableItems](Flowthru.Data.EnumerableItems.md) + +Extension point for factory methods. + + [Item](Flowthru.Data.Item\-1.md) + +Standard catalog item implementation that delegates to a storage adapter. + + [Items](Flowthru.Data.Items.md) + +Static factory methods for creating catalog entries with common configurations. + + [Items.Single](Flowthru.Data.Items.Single.md) + +Factory methods for single (non-collection) values. + +### Interfaces + + [IItem](Flowthru.Data.IItem\-1.md) + +Unified catalog item with cardinality encoded in the type parameter. + + [IItem](Flowthru.Data.IItem.md) + +Non-generic base interface for catalog items. +Provides untyped operations for internal use by the flow executor and mapping layer. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.FlowIO-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.FlowIO-1.md new file mode 100644 index 00000000..917ee9c9 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.FlowIO-1.md @@ -0,0 +1,338 @@ +# Struct FlowIO + +Namespace: [Flowthru.Effects](Flowthru.Effects.md) +Assembly: Flowthru.Core.dll + +Represents a cancellable asynchronous effect that produces a value of type A. + +```csharp +public readonly struct FlowIO +``` + +#### Type Parameters + +`A` + +The type of value produced by this effect. + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+ is a lightweight effect monad for representing I/O operations. +It provides: +

+
  • Lazy evaluation - effects don't run until is called
  • Cancellation support - all effects accept a
  • Functor/Monad operations - ,
  • LINQ comprehension syntax - via and
+

+Example - Basic usage: +

+
FlowIO<string> ReadFile(string path) =>
+    FlowIO.LiftAsync(ct => File.ReadAllTextAsync(path, ct));
+
+FlowIO<int> GetWordCount(string path) =>
+    from content in ReadFile(path)
+    select content.Split(' ').Length;
+
+int count = await GetWordCount("data.txt").Run();
+

+Example - Error handling: +

+
FlowIO<Data> LoadData() =>
+    FlowIO.LiftAsync(async ct => {
+        if (!File.Exists("data.json"))
+            throw new FileNotFoundException("Data file missing");
+        return await JsonSerializer.DeserializeAsync<Data>(...);
+    });
+
+try {
+    var data = await LoadData().Run();
+}
+catch (FileNotFoundException ex) {
+    // Handle error
+}
+ +## Methods + +###
Bind\(Func\>\) + +Binds this effect to a function that produces another effect. + +```csharp +public FlowIO Bind(Func> f) +``` + +#### Parameters + +`f` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +The function that takes this effect's result and produces a new effect. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +A new effect representing the sequential composition of both effects. + +#### Type Parameters + +`B` + +The result type of the produced effect. + +#### Remarks + +This is the monad's bind (or >>=) operation. It enables sequential +composition of effects, forming a computational pipeline. + +### Catch\(Func\>\) + +Catches exceptions thrown during effect execution and recovers with an alternative effect. + +```csharp +public FlowIO Catch(Func> handler) +``` + +#### Parameters + +`handler` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[Exception](https://learn.microsoft.com/dotnet/api/system.exception), [FlowIO](Flowthru.Effects.FlowIO\-1.md)\> + +Function that receives the exception and produces a recovery effect. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +A new effect that recovers from failures using handler. + +### Catch\(Func\>\) + +Catches exceptions of a specific type during effect execution. + +```csharp +public FlowIO Catch(Func> handler) where TException : Exception +``` + +#### Parameters + +`handler` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Function that receives the exception and produces a recovery effect. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +A new effect that recovers from specific exceptions. + +#### Type Parameters + +`TException` + +The type of exception to catch. + +### IfFail\(A\) + +Returns a default value if this effect fails. + +```csharp +public FlowIO IfFail(A defaultValue) +``` + +#### Parameters + +`defaultValue` A + +The value to return on failure. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +A new effect that never fails, using defaultValue on error. + +### IfFail\(FlowIO\) + +Returns an alternative effect if this effect fails. + +```csharp +public FlowIO IfFail(FlowIO alternative) +``` + +#### Parameters + +`alternative` [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +The effect to run on failure. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +A new effect that falls back to alternative on error. + +### Map\(Func\) + +Maps the result of this effect using the specified function. + +```csharp +public FlowIO Map(Func f) +``` + +#### Parameters + +`f` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +The function to apply to the effect's result. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +A new effect that applies f to this effect's result. + +#### Type Parameters + +`B` + +The result type after mapping. + +#### Remarks + +This is the functor's fmap operation. It transforms the value inside the effect +without changing the effect structure. + +### MapAsync\(Func\>\) + +Maps the result of this effect using an async function. + +```csharp +public FlowIO MapAsync(Func> f) +``` + +#### Parameters + +`f` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +The async function to apply to the effect's result. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +A new effect that applies f to this effect's result. + +#### Type Parameters + +`B` + +The result type after mapping. + +### Run\(CancellationToken\) + +Executes this effect and returns the result. + +```csharp +public ValueTask Run(CancellationToken token = default) +``` + +#### Parameters + +`token` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Optional cancellation token to cancel the effect. + +#### Returns + + [ValueTask](https://learn.microsoft.com/dotnet/api/system.threading.tasks.valuetask\-1) + +A representing the asynchronous operation. + +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if the effect is uninitialized. + +### Select\(Func\) + +Transforms this effect using the specified function (alias for ). +Enables LINQ select syntax. + +```csharp +public FlowIO Select(Func selector) +``` + +#### Parameters + +`selector` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +The function to apply to the effect's result. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +A new effect with the transformed result. + +#### Type Parameters + +`B` + +The result type after transformation. + +#### Remarks + +Example: +
var result = from x in GetValue()
+             select x * 2;
+ +### SelectMany\(Func\>, Func\) + +Projects this effect through a function that produces another effect, then combines +both results using a projection function. Enables LINQ from...from...select syntax. + +```csharp +public FlowIO SelectMany(Func> bind, Func project) +``` + +#### Parameters + +`bind` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +The function that produces the next effect based on this effect's result. + +`project` [Func](https://learn.microsoft.com/dotnet/api/system.func\-3) + +The function that combines both results into the final result. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +A new effect representing the composition. + +#### Type Parameters + +`B` + +The intermediate result type. + +`C` + +The final result type. + +#### Remarks + +Example: +
var result = from x in GetFirstValue()
+             from y in GetSecondValue(x)
+             select x + y;
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.FlowIO.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.FlowIO.md new file mode 100644 index 00000000..783065ed --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.FlowIO.md @@ -0,0 +1,243 @@ +# Class FlowIO + +Namespace: [Flowthru.Effects](Flowthru.Effects.md) +Assembly: Flowthru.Core.dll + +Provides factory methods and combinators for creating effects. + +```csharp +public static class FlowIO +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowIO](Flowthru.Effects.FlowIO.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### Fail\(Exception\) + +Creates an effect that immediately fails with the given exception. + +```csharp +public static FlowIO Fail(Exception error) +``` + +#### Parameters + +`error` [Exception](https://learn.microsoft.com/dotnet/api/system.exception) + +The exception to throw. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +An effect that fails with error. + +#### Type Parameters + +`A` + +The expected result type. + +### Fail\(string\) + +Creates an effect that immediately fails with an exception containing the given message. + +```csharp +public static FlowIO Fail(string message) +``` + +#### Parameters + +`message` [string](https://learn.microsoft.com/dotnet/api/system.string) + +The error message. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +An effect that fails with an exception containing message. + +#### Type Parameters + +`A` + +The expected result type. + +### Lift\(Func\) + +Lifts a synchronous function into an effect. + +```csharp +public static FlowIO Lift(Func f) +``` + +#### Parameters + +`f` [Func](https://learn.microsoft.com/dotnet/api/system.func\-1) + +The function to lift. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +An effect that executes f. + +#### Type Parameters + +`A` + +The return type. + +#### Remarks + +The function is still executed lazily - only when is called. + +### LiftAsync\(Func\>\) + +Lifts a cancellation-aware -returning function into an effect. + +```csharp +public static FlowIO LiftAsync(Func> f) +``` + +#### Parameters + +`f` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken), [ValueTask](https://learn.microsoft.com/dotnet/api/system.threading.tasks.valuetask\-1)\> + +The function that accepts a cancellation token and returns a . + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +An effect that executes f. + +#### Type Parameters + +`A` + +The return type. + +#### Remarks + +

+All async I/O operations should observe the cancellation token to support graceful shutdown. +If your operation is truly synchronous, use instead. +

+

+For Task-based APIs, convert using .AsTask(): LiftAsync(async ct => await SomeTaskAsync(ct).AsTask()) +or rely on implicit conversion from Task to ValueTask. +

+ +###
Pure\(A\) + +Creates an effect that immediately returns the given value. + +```csharp +public static FlowIO Pure(A value) +``` + +#### Parameters + +`value` A + +The value to return. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +An effect that produces value. + +#### Type Parameters + +`A` + +The type of value. + +#### Remarks + +This is the monad's return or pure operation. + +### Sequence\(IEnumerable\>\) + +Sequences a collection of effects, running them all and collecting the results. + +```csharp +public static FlowIO> Sequence(IEnumerable> effects) +``` + +#### Parameters + +`effects` [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)<[FlowIO](Flowthru.Effects.FlowIO\-1.md)\> + +The collection of effects to sequence. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +An effect that produces an enumerable of all results. + +#### Type Parameters + +`A` + +The result type of each effect. + +#### Remarks + +Effects are executed sequentially in the order they appear in the collection. +If any effect fails, the entire sequence fails. + +### Traverse\(IEnumerable, Func\>\) + +Traverses a collection, applying an effect-producing function to each element +and collecting the results. + +```csharp +public static FlowIO> Traverse(IEnumerable source, Func> f) +``` + +#### Parameters + +`source` [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1) + +The collection to traverse. + +`f` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +The function that produces an effect for each element. + +#### Returns + + [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +An effect that produces an enumerable of results. + +#### Type Parameters + +`A` + +The source element type. + +`B` + +The result type. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.FlowUnit.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.FlowUnit.md new file mode 100644 index 00000000..5777d3df --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.FlowUnit.md @@ -0,0 +1,292 @@ +# Struct FlowUnit + +Namespace: [Flowthru.Effects](Flowthru.Effects.md) +Assembly: Flowthru.Core.dll + +Represents a void-like value for effect operations with no meaningful return value. +Similar to Unit in functional programming or void in imperative programming, +but usable as a type parameter. + +```csharp +public readonly struct FlowUnit : IEquatable, IComparable +``` + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1), +[IComparable](https://learn.microsoft.com/dotnet/api/system.icomparable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Use when an effect produces side effects but no meaningful result. +For example, FlowIO<FlowUnit> represents an I/O operation that doesn't return data. +

+

+Example: +

+
FlowIO<FlowUnit> WriteLog(string message) =>
+    FlowIO.LiftAsync(async ct => {
+        await File.WriteAllTextAsync("log.txt", message, ct);
+        return FlowFlowUnit.Default;
+    });
+ +## Fields + +### Default + +The single instance of . + +```csharp +public static readonly FlowUnit Default +``` + +#### Field Value + + [FlowUnit](Flowthru.Effects.FlowUnit.md) + +## Methods + +### CompareTo\(FlowUnit\) + +Compares the current instance with another . + +```csharp +public int CompareTo(FlowUnit other) +``` + +#### Parameters + +`other` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The to compare. + +#### Returns + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Always 0, as all FlowUnit instances are equal. + +### Equals\(FlowUnit\) + +Determines whether the specified is equal to the current instance. + +```csharp +public bool Equals(FlowUnit other) +``` + +#### Parameters + +`other` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The to compare. + +#### Returns + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +Always true, as all FlowUnit instances are equal. + +### Equals\(object?\) + +Determines whether the specified object is equal to the current instance. + +```csharp +public override bool Equals(object? obj) +``` + +#### Parameters + +`obj` [object](https://learn.microsoft.com/dotnet/api/system.object)? + +The object to compare. + +#### Returns + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +true if obj is a ; otherwise, false. + +### GetHashCode\(\) + +Returns the hash code for this instance. + +```csharp +public override int GetHashCode() +``` + +#### Returns + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Always returns 0, as all FlowUnit instances are equal. + +### ToString\(\) + +Returns a string representation of this instance. + +```csharp +public override string ToString() +``` + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +Always returns "unit". + +## Operators + +### operator ==\(FlowUnit, FlowUnit\) + +Determines whether two instances are equal. + +```csharp +public static bool operator ==(FlowUnit left, FlowUnit right) +``` + +#### Parameters + +`left` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The first instance to compare. + +`right` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The second instance to compare. + +#### Returns + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +Always true. + +### operator \>\(FlowUnit, FlowUnit\) + +Compares two instances. + +```csharp +public static bool operator >(FlowUnit left, FlowUnit right) +``` + +#### Parameters + +`left` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The first instance to compare. + +`right` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The second instance to compare. + +#### Returns + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +Always false. + +### operator \>=\(FlowUnit, FlowUnit\) + +Compares two instances. + +```csharp +public static bool operator >=(FlowUnit left, FlowUnit right) +``` + +#### Parameters + +`left` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The first instance to compare. + +`right` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The second instance to compare. + +#### Returns + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +Always true. + +### operator \!=\(FlowUnit, FlowUnit\) + +Determines whether two instances are not equal. + +```csharp +public static bool operator !=(FlowUnit left, FlowUnit right) +``` + +#### Parameters + +`left` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The first instance to compare. + +`right` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The second instance to compare. + +#### Returns + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +Always false. + +### operator <\(FlowUnit, FlowUnit\) + +Compares two instances. + +```csharp +public static bool operator <(FlowUnit left, FlowUnit right) +``` + +#### Parameters + +`left` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The first instance to compare. + +`right` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The second instance to compare. + +#### Returns + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +Always false. + +### operator <=\(FlowUnit, FlowUnit\) + +Compares two instances. + +```csharp +public static bool operator <=(FlowUnit left, FlowUnit right) +``` + +#### Parameters + +`left` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The first instance to compare. + +`right` [FlowUnit](Flowthru.Effects.FlowUnit.md) + +The second instance to compare. + +#### Returns + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +Always true. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.md new file mode 100644 index 00000000..455a5630 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Effects.md @@ -0,0 +1,20 @@ +# Namespace Flowthru.Effects + +### Classes + + [FlowIO](Flowthru.Effects.FlowIO.md) + +Provides factory methods and combinators for creating effects. + +### Structs + + [FlowIO](Flowthru.Effects.FlowIO\-1.md) + +Represents a cancellable asynchronous effect that produces a value of type A. + + [FlowUnit](Flowthru.Effects.FlowUnit.md) + +Represents a void-like value for effect operations with no meaningful return value. +Similar to Unit in functional programming or void in imperative programming, +but usable as a type parameter. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.DryRunOption.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.DryRunOption.md new file mode 100644 index 00000000..db999e61 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.DryRunOption.md @@ -0,0 +1,89 @@ +# Struct DryRunOption + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Represents a dry-run configuration. Can be assigned from a +or a value. + +```csharp +public readonly struct DryRunOption +``` + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Assigning true enables a full dry run (all pre-flight checks, no execution). +Assigning false disables dry-run mode entirely. +Assigning a enables dry-run at the specified depth. + +## Properties + +### Depth + +The validation depth applied when dry-run is enabled. + +```csharp +public ValidationDepth Depth { get; } +``` + +#### Property Value + + [ValidationDepth](Flowthru.Flows.ValidationDepth.md) + +### Enabled + +Whether dry-run mode is enabled. + +```csharp +public bool Enabled { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Operators + +### implicit operator DryRunOption\(bool\) + +Implicitly converts a to a . +true enables full dry-run; false disables it. + +```csharp +public static implicit operator DryRunOption(bool value) +``` + +#### Parameters + +`value` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Returns + + [DryRunOption](Flowthru.Flows.DryRunOption.md) + +### implicit operator DryRunOption\(ValidationDepth\) + +Implicitly converts a to a , +enabling dry-run at the specified depth. + +```csharp +public static implicit operator DryRunOption(ValidationDepth depth) +``` + +#### Parameters + +`depth` [ValidationDepth](Flowthru.Flows.ValidationDepth.md) + +#### Returns + + [DryRunOption](Flowthru.Flows.DryRunOption.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.ExecutionOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.ExecutionOptions.md new file mode 100644 index 00000000..af95fbf9 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.ExecutionOptions.md @@ -0,0 +1,120 @@ +# Class ExecutionOptions + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Configuration options for pipeline execution. + +```csharp +public class ExecutionOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ExecutionOptions](Flowthru.Flows.ExecutionOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Controls how pipelines are executed and how results are presented. + +## Properties + +### DryRun + +Whether to perform a dry run, and at what validation depth. + +```csharp +public DryRunOption DryRun { get; set; } +``` + +#### Property Value + + [DryRunOption](Flowthru.Flows.DryRunOption.md) + +#### Remarks + +Assign true to perform all pre-flight operations (structure validation, +validation hooks, and external data source inspection) without executing nodes. +Assign a value to control how deeply the pre-flight +checks run — for example, validates +the pipeline graph and runs extension hooks without probing any data sources. +Assign false (default) to run normally without a dry-run stop. + +### EnableParallelExecution + +Whether to enable parallel execution of nodes within the same layer. + +```csharp +public bool EnableParallelExecution { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Phase 2 feature - currently not implemented. +When true, nodes in the same execution layer run concurrently. + +### ResultFormatter + +The result formatter to use for displaying execution results. + +```csharp +public IFlowResultFormatter? ResultFormatter { get; set; } +``` + +#### Property Value + + [IFlowResultFormatter](Flowthru.Results.IFlowResultFormatter.md)? + +#### Remarks + +Defaults to ConsoleResultFormatter if not specified. + +### SliceStrategy + +Optional slicing strategy to apply when executing pipelines. + +```csharp +public FlowSliceStrategy? SliceStrategy { get; set; } +``` + +#### Property Value + + [FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md)? + +#### Remarks + +When provided, only nodes matching the slice strategy will be executed. +Used when slicing flags are provided without a specific pipeline name. + +### StopOnFirstError + +Whether to stop execution on the first node failure. + +```csharp +public bool StopOnFirstError { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +When true (default), pipeline execution stops immediately when a node fails. +When false, execution continues to independent nodes (Phase 2 feature). + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Flow.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Flow.md new file mode 100644 index 00000000..04e8ad4c --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Flow.md @@ -0,0 +1,418 @@ +# Class Flow + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Represents a complete data flow with steps, dependencies, and execution order. + +```csharp +public class Flow +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Flow](Flowthru.Flows.Flow.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+A flow is a directed acyclic graph (DAG) of transformation steps. +Each step reads data from catalog entries, performs transformations, +and writes results back to catalog entries. +

+

+Execution Model: +

+
  • Steps are organized into layers via topological sort
  • Steps in layer 0 have no dependencies (read external data only)
  • Steps in layer N depend only on steps in layers 0..N-1
  • Sequential execution: Execute all steps in layer order
  • Parallel execution (Phase 2): Execute steps within same layer concurrently
+

+Single Producer Rule: Each catalog entry can be written by at most +one step. This ensures deterministic execution order and prevents race conditions. +

+ +## Properties + +### Description + +Optional description of what this flow does. + +```csharp +public string? Description { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### IsBuilt + +Indicates whether the flow has been built (dependencies analyzed and layers assigned). + +```csharp +public bool IsBuilt { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### Logger + +Optional logger for flow execution. + +```csharp +public ILogger? Logger { get; set; } +``` + +#### Property Value + + [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger)? + +### Name + +Flow name for identification and logging. + +```csharp +public string? Name { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +Set by FlowRegistry during flow registration. + +### ServiceProvider + +Optional service provider for dependency injection into steps. + +```csharp +public IServiceProvider? ServiceProvider { get; set; } +``` + +#### Property Value + + [IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider)? + +#### Remarks + +Set by the service layer before flow execution to enable steps +to resolve services (e.g., database connections, external APIs). + +### Steps + +All steps in this flow, in the order they were added. + +```csharp +public IReadOnlyList Steps { get; } +``` + +#### Property Value + + [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[FlowStep](Flowthru.Flows.FlowStep.md)\> + +#### Remarks + +Exposed as public to enable validation hooks (Phase 4) to inspect steps. +The collection is read-only - steps can only be added via FlowBuilder. + +### ValidationHooks + +Validation hooks that run during pre-flight checks. + +```csharp +public List ValidationHooks { get; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[IFlowValidationHook](Flowthru.Flows.Validation.IFlowValidationHook.md)\> + +#### Remarks + +

+Extensions can register hooks to validate their own step types during pre-flight. +Hooks are invoked after DAG analysis but before external input inspection. +

+

+Hook execution order: +

+
  1. Flow.Build() - DAG construction and layer assignment
  2. ValidationHooks.ValidateAsync() - Extension-specific validation
  3. Flow.ValidateExternalInputsAsync() - External input inspection
+

+Example (Python extension): +

+
flow.ValidationHooks.Add(new PythonStepValidator(executor, runtime));
+ +### ValidationOptions + +Validation options for this flow. + +```csharp +public ValidationOptions ValidationOptions { get; } +``` + +#### Property Value + + [ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md) + +#### Remarks + +Configures how external data sources (Layer 0 inputs) are validated +before flow execution begins. + +## Methods + +### Build\(FlowSliceStrategy?\) + +Builds the flow by analyzing dependencies and assigning execution layers. +Must be called before executing the flow. + +```csharp +public void Build(FlowSliceStrategy? sliceStrategy = null) +``` + +#### Parameters + +`sliceStrategy` [FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md)? + +Optional slicing strategy to filter steps before execution + +#### Remarks + +

+Slicing: If a slicing strategy is provided, only steps matching +the strategy will be included in the execution. The slice always forms a valid +sub-DAG with all required dependencies. +

+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if: +- Multiple steps write to the same catalog entry (single producer rule) +- Circular dependency is detected +- Slice strategy references non-existent steps or catalog entries + +### ExecuteAsync\(CancellationToken\) + +Executes the flow sequentially, layer by layer. + +```csharp +public Task ExecuteAsync(CancellationToken cancellationToken) +``` + +#### Parameters + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token to signal graceful shutdown + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task) + +Task representing the flow execution + +#### Remarks + +

+This method executes flow in topological order: +1. Execute all flow in layer 0 sequentially +2. Execute all flow in layer 1 sequentially +3. Continue until all layers are complete +

+

+Note: This method throws exceptions on failure. For result-based +execution with error handling, use RunAsync() instead. +

+

+In Phase 2, this will be replaced with a parallel executor that can run +steps within the same layer concurrently. +

+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if flow has not been built + +### ExportDag\(\) + +Exports DAG metadata for this Flow. + +```csharp +public DagMetadata ExportDag() +``` + +#### Returns + + [DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +Complete DAG metadata including steps, catalog entries, and edges + +#### Remarks + +

+This method extracts structural metadata from the built flow , creating +a complete representation of the DAG (Directed Acyclic Graph) that can be +serialized to JSON for visualization in Flowthru.Viz. +

+

+Prerequisites: Flow must be built before calling this method. +Call Build() first if IsBuilt is false. +

+

+Usage: +

+
var flow = DataProcessingFlow.Create(catalog);
+flow.Build();
+
+var dag = flow.ExportDag();
+var json = dag.ToJson();
+File.WriteAllText("dag.json", json);
+

+This method is non-destructive and idempotent - it can be called multiple +times without affecting the flow state. +

+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if flow has not been built + +### Merge\(Dictionary\) + +Merges multiple flows into a single flow by combining all their steps. + +```csharp +public static Flow Merge(Dictionary flows) +``` + +#### Parameters + +`flows` [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [Flow](Flowthru.Flows.Flow.md)\> + +Dictionary of flow names to flow instances + +#### Returns + + [Flow](Flowthru.Flows.Flow.md) + +A new flow containing all steps from all input flows + +#### Remarks + +

+This method creates a new flow by combining all steps from the input flows. +Step names are prefixed with their source flow name (e.g., "data_processing.PreprocessCompanies") +to ensure uniqueness and maintain traceability in logs. +

+

+The existing DependencyAnalyzer will automatically resolve cross-flow dependencies +based on catalog entries. The single producer rule is enforced - if multiple flows +attempt to write to the same catalog entry, Build() will throw an InvalidOperationException. +

+ +### RunAsync\(CancellationToken\) + +/// Builds and executes the flow, returning comprehensive execution results. + +```csharp +public Task RunAsync(CancellationToken cancellationToken) +``` + +#### Parameters + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token to signal graceful shutdown + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[FlowResult](Flowthru.Flows.FlowResult.md)\> + +FlowResult containing execution status, timing, and flow results + +#### Remarks + +

+This is the primary high-level API for executing flows. It automatically +calls Build() if the flow hasn't been built yet, then executes and tracks results. +

+ +### ValidateExternalInputsAsync\(CancellationToken\) + +Validates all external inputs before flow execution. + +```csharp +public Task ValidateExternalInputsAsync(CancellationToken cancellationToken = default) +``` + +#### Parameters + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token for validation I/O operations + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +ValidationResult containing any errors found + +#### Remarks + +

+This method inspects catalog entries that are consumed by the flow but not +produced by any step in the execution set. These are pre-existing external data +sources (files, databases, APIs) that must exist and be valid before the flow +can execute. +

+

+Slicing Support: In sliced flows, catalog entries that were +produced by steps outside the slice are correctly identified as external inputs +and validated. This prevents runtime failures from missing intermediate data. +

+

+Inspection Levels: +

+
  • None: Skip inspection entirely
  • Shallow: Validate file exists, check headers/schema, deserialize sample rows
  • Deep: Validate all rows in the dataset (expensive!)
+

+Default Behavior: +

+
  • If explicitly configured via WithValidation() → use that level
  • If entry has PreferredInspectionLevel set → use that level
  • Otherwise → Shallow (all storage adapters support inspection)
+

+Important: Only external inputs are inspected. Intermediate flow +outputs produced within the execution set are never inspected, as they don't exist yet. +

+

+Usage: +

+
flow.Build();
+var validationResult = await flow.ValidateExternalInputsAsync();
+if (!validationResult.IsValid) {
+  // Handle validation errors before execution
+  validationResult.ThrowIfInvalid();
+}
+await flow.RunAsync();
+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if flow has not been built + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowBuilder.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowBuilder.md new file mode 100644 index 00000000..b41007ee --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowBuilder.md @@ -0,0 +1,9683 @@ +# Class FlowBuilder + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Fluent builder for constructing type-safe flows with function-based steps. + +```csharp +public class FlowBuilder +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Function-Based Design (v0.5.0): +Steps are pure transformation functions with compile-time type safety. +Both synchronous and asynchronous functions are supported: +- Sync: Func<TInput, TOutput> +- Async: Func<TInput, Task<TOutput>> +- Multi-input: Func<(TIn1, TIn2, ...), TOutput> or Task<TOutput> +- Multi-output: Func<TInput, (TOut1, TOut2, ...)> or Task<(TOut1, TOut2, ...)> +

+

+Use synchronous functions for pure data transformations. Use asynchronous functions +only when your step performs I/O operations (external APIs, databases, etc.). +

+

+The compiler infers all types from function signatures and validates catalog entry +types at flow construction time, catching type mismatches before execution. +

+

+Usage Patterns: +

+
var flow = FlowBuilder.CreateFlow(builder =>
+{
+    // Simple synchronous step
+    builder.AddStep(
+        name: "Preprocess",
+        transform: PreprocessStep.Create(),
+        input: catalog.RawData,
+        output: catalog.ProcessedData
+    );
+
+    // Multi-input step: tuple → single output
+    builder.AddStep(
+        name: "TrainModel",
+        transform: TrainModelStep.Create(),
+        input: (catalog.XTrain, catalog.YTrain),
+        output: catalog.Model
+    );
+
+    // Multi-output step: single input → tuple
+    builder.AddStep(
+        name: "SplitData",
+        transform: SplitDataStep.Create(),
+        input: catalog.Data,
+        output: (catalog.XTrain, catalog.XTest, catalog.YTrain, catalog.YTest)
+    );
+
+    // Asynchronous step (only when needed for I/O)
+    builder.AddStep(
+        name: "FetchExternalData",
+        transform: ExternalDataStep.Create(),
+        input: catalog.Config,
+        output: catalog.ExternalData
+    );
+});
+
+flow.Build();
+await flow.ExecuteAsync();
+ +## Methods + +### AddStep\(string, Func\>, IItem, IItem, string\) + +Adds a step with single input and single output (asynchronous transformation). +All types are inferred from the transformation function signature. + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function from input to output + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description for this step + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TInput` + +Input type (inferred from transform) + +`TOutput` + +Output type (inferred from transform) + +### AddStep\(string, Func\>, IItem, IItem, string\) + +Adds a step with single input and single output (asynchronous transformation with cancellation support). +All types are inferred from the transformation function signature. + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-3)\> + +Asynchronous transformation function from input to output with cancellation token + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description for this step + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TInput` + +Input type (inferred from transform) + +`TOutput` + +Output type (inferred from transform) + +### AddStep\(string, Func, IItem, IItem, string\) + +Adds a step with single input and single output (synchronous transformation). +All types are inferred from the transformation function signature. + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function from input to output + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description for this step + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TInput` + +Input type (inferred from transform) + +`TOutput` + +Output type (inferred from transform) + +### AddStep\(string, IReadOnlyList\>, IItem, Func, TOut\>, string\) + +Adds a homogeneous fan-in step: N catalog entries of the same element type collapse +into a single step whose transform receives all N loaded collections as a typed list. + +```csharp +public FlowBuilder AddStep(string label, IReadOnlyList> inputs, IItem output, Func, TOut> step, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`inputs` [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[IItem](Flowthru.Data.IItem\-1.md)\> + +Variable-length list of same-typed input entries + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry to store the merged result + +`step` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1), TOut\> + +Transform function receiving all N loaded values as a typed read-only list + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional human-readable description + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn` + +Element type of each input catalog entry + +`TOut` + +Output type produced by the transform + +#### Remarks + +Use this when the number of inputs is not known at compile time — for example, +aggregating per-partition catalogs constructed in a loop. The function receives +IReadOnlyList<TIn> where each element corresponds to one input entry +in declaration order. An empty inputs list is allowed but produces an empty list argument. + +### AddStep\(string, Func\>, IItem, \(IItem, IItem\), string\) + +Adds a step with 1 input and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func, IItem, \(IItem, IItem\), string\) + +Adds a step with 1 input and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 1 input and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task\>, \(IItem, IItem\), IItem, string\) + +Adds a step with 2 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task> transform, (IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2\), TOut1\>, \(IItem, IItem\), IItem, string\) + +Adds a step with 2 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), TOut1> transform, (IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 2 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2)>> transform, (IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2\)\>, \(IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 2 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2)> transform, (IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3)> transform, (IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 2 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task\>, \(IItem, IItem, IItem\), IItem, string\) + +Adds a step with 3 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task> transform, (IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), TOut1\>, \(IItem, IItem, IItem\), IItem, string\) + +Adds a step with 3 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), TOut1> transform, (IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 3 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 3 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2)> transform, (IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 3 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task\>, \(IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a step with 4 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task> transform, (IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), TOut1\>, \(IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a step with 4 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), TOut1> transform, (IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 4 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 4 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 4 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task\>, \(IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a step with 5 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task> transform, (IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), TOut1\>, \(IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a step with 5 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), TOut1> transform, (IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 5 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 5 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 5 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a step with 6 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), TOut1\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a step with 6 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), TOut1> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 6 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 6 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 6 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a step with 7 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), TOut1\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a step with 7 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), TOut1> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 7 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 7 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 7 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a step with 8 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), TOut1\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a step with 8 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), TOut1> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 8 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a step with 8 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a step with 8 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog item or tuple of catalog items to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the step's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### CreateFlow\(Action\) + +Creates and configures a new flow using the builder pattern. + +```csharp +public static Flow CreateFlow(Action configure) +``` + +#### Parameters + +`configure` [Action](https://learn.microsoft.com/dotnet/api/system.action\-1)<[FlowBuilder](Flowthru.Flows.FlowBuilder.md)\> + +Action to configure the flow by adding steps + +#### Returns + + [Flow](Flowthru.Flows.Flow.md) + +Configured but not yet built flow + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowResult.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowResult.md new file mode 100644 index 00000000..323eac40 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowResult.md @@ -0,0 +1,214 @@ +# Class FlowResult + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Represents the result of a flow execution. + +```csharp +public class FlowResult +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowResult](Flowthru.Flows.FlowResult.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This class provides comprehensive execution information including success status, +timing, individual step results, and error details. +

+

+Usage Pattern: +

+
var result = await flow.RunAsync();
+
+if (result.Success)
+{
+    Console.WriteLine($"Flow completed in {result.ExecutionTime.TotalSeconds:F2}s");
+    foreach (var stepResult in result.StepResults.Values)
+    {
+        Console.WriteLine($"  {stepResult.StepName}: {stepResult.ExecutionTime.TotalSeconds:F2}s");
+    }
+}
+else
+{
+    Console.WriteLine($"Flow failed: {result.Exception?.Message}");
+}
+ +## Properties + +### Exception + +Exception that caused flow failure, if any. + +```csharp +public Exception? Exception { get; init; } +``` + +#### Property Value + + [Exception](https://learn.microsoft.com/dotnet/api/system.exception)? + +#### Remarks + +Null if Success is true. Contains the first exception that caused +flow execution to halt if Success is false. + +### ExecutionTime + +Total execution time for the entire flow. + +```csharp +public TimeSpan ExecutionTime { get; init; } +``` + +#### Property Value + + [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +### FlowName + +The name of the flow that was executed. + +```csharp +public string? FlowName { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### IsDryRun + +Indicates whether this was a dry run (pre-flight checks only). + +```csharp +public bool IsDryRun { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### StepResults + +Results for individual steps, keyed by step name. + +```csharp +public Dictionary StepResults { get; init; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [StepResult](Flowthru.Flows.StepResult.md)\> + +#### Remarks + +Dictionary keys are the step names as specified in the flow definition. +Values contain execution details for each step. + +### Success + +Indicates whether the flow executed successfully. + +```csharp +public bool Success { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### CreateDryRunSuccess\(TimeSpan, int, int, int, string?\) + +Creates a successful dry run result. + +```csharp +public static FlowResult CreateDryRunSuccess(TimeSpan preFlightDuration, int stepCount, int layerCount, int validatedInputCount, string? flowName = null) +``` + +#### Parameters + +`preFlightDuration` [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +Time spent on pre-flight checks + +`stepCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Total number of steps in the flow + +`layerCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of execution layers + +`validatedInputCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of external inputs validated + +`flowName` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Name of the flow + +#### Returns + + [FlowResult](Flowthru.Flows.FlowResult.md) + +A successful dry run result + +### CreateFailure\(TimeSpan, Exception, Dictionary?, string?\) + +Creates a failed flow result. + +```csharp +public static FlowResult CreateFailure(TimeSpan executionTime, Exception exception, Dictionary? stepResults = null, string? flowName = null) +``` + +#### Parameters + +`executionTime` [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +`exception` [Exception](https://learn.microsoft.com/dotnet/api/system.exception) + +`stepResults` [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [StepResult](Flowthru.Flows.StepResult.md)\>? + +`flowName` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Returns + + [FlowResult](Flowthru.Flows.FlowResult.md) + +### CreateSuccess\(TimeSpan, Dictionary, string?\) + +Creates a successful flow result. + +```csharp +public static FlowResult CreateSuccess(TimeSpan executionTime, Dictionary stepResults, string? flowName = null) +``` + +#### Parameters + +`executionTime` [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +`stepResults` [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [StepResult](Flowthru.Flows.StepResult.md)\> + +`flowName` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Returns + + [FlowResult](Flowthru.Flows.FlowResult.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowSliceStrategy.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowSliceStrategy.md new file mode 100644 index 00000000..b6e05cfb --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowSliceStrategy.md @@ -0,0 +1,183 @@ +# Class FlowSliceStrategy + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Defines a strategy for slicing a flow to execute a subset of nodes. + +```csharp +public sealed class FlowSliceStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+flow slicing allows executing only specific portions of a DAG while maintaining +execution validity. All slicing operations preserve the runnability guarantee: +the resulting sub-DAG must be executable without missing dependencies. +

+

+Slicing Strategies: +

+
  • flows: Filter to nodes from specific named flows (in merged DAGs)
  • FromNodes: Include specified nodes and all downstream dependents
  • ToNodes: Include specified nodes and all upstream dependencies (run "up to" these nodes)
  • FromData: Include nodes consuming specified catalog entries and all downstream dependents
  • ToData: Include nodes producing specified catalog entries and all upstream dependencies
  • OnlyNodes: Explicit allowlist of nodes plus minimal required dependencies
+

+Composition: Multiple strategies compose via intersection (additive filtering). +For example, --from-nodes A --to-data B produces nodes in the downstream dependency +tree of A that are also required to produce data B. +

+

+Runnability Guarantee: Slicing operations are additive only. Subtractive +operations (--from-nodes A --except B) would break the runnability guarantee and +are not supported. +

+ +## Properties + +### Flows + +Filter to nodes from these named flows (applies to merged flows). + +```csharp +public IReadOnlySet? Flows { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +In merged flows, nodes are prefixed with their flow name (e.g., "DataScience.TrainModel"). +This filter includes only nodes from the specified flows. +flow names are case-insensitive. + +### FromData + +Start from nodes that consume these catalog entry labels, including all downstream dependents. + +```csharp +public IReadOnlySet? FromData { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +Finds all nodes that read the specified catalog entries, then expands downstream. +Useful for impact analysis - "what breaks if I change this data?" + +TODO: Remove this, as it is now sufficient to reference both steps and data entries as nodes. + +### FromNodes + +Start from these nodes, including all downstream dependents. + +```csharp +public IReadOnlySet? FromNodes { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +Expands to include all nodes that depend on these nodes (transitively). +Useful for impact analysis - "what breaks if I change this node?" + +### IsSliced + +Whether any slicing is configured. + +```csharp +public bool IsSliced { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### OnlyNodes + +Explicit allowlist of node names (dependencies auto-included). + +```csharp +public IReadOnlySet? OnlyNodes { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +Specifies exactly which nodes to execute, then automatically includes any +required dependencies to maintain DAG validity. + +### ToData + +End at nodes that produce these catalog entry labels, including all upstream dependencies. + +```csharp +public IReadOnlySet? ToData { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +Finds the nodes that write the specified catalog entries, then expands upstream. +Useful for targeted execution - "run everything needed to produce this data". + +TODO: Remove this, as it is now sufficient to reference both steps and data entries as nodes. + +### ToNodes + +End at these nodes, including all upstream dependencies needed to produce them. + +```csharp +public IReadOnlySet? ToNodes { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +Expands to include all transitive dependencies needed to run these nodes. +Equivalent to "run everything up to and including these nodes". +Useful for testing specific outputs without running the entire flow. + +## Methods + +### All\(\) + +No filtering - execute entire flow. + +```csharp +public static FlowSliceStrategy All() +``` + +#### Returns + + [FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowStep.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowStep.md new file mode 100644 index 00000000..4af126b8 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.FlowStep.md @@ -0,0 +1,219 @@ +# Class FlowStep + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Represents a step within a flow, wrapping the transformation function with metadata +about its inputs, outputs, and dependencies. + +```csharp +public class FlowStep +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowStep](Flowthru.Flows.FlowStep.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+FlowStep serves as the internal representation of a step during flow +construction and execution. It tracks: +- The transformation function (Func<TInput, Task<TOutput>>) +- Input catalog entries (what data it reads) +- Output catalog entries (what data it writes) +- Dependencies (other steps that must run first) +

+

+Single Producer Rule: Each catalog entry can be written by at most +one step in a flow. This constraint ensures deterministic dependency resolution +and enables simple DAG construction via topological sort. +

+

+Made public to enable validation hooks to inspect step properties. +This is necessary for extensions (e.g., Python) to validate their own step types. +

+ +## Constructors + +### FlowStep\(string, string?, Delegate, IReadOnlyList, IReadOnlyList\) + +Creates a new flow step with a transformation function. + +```csharp +public FlowStep(string label, string? description, Delegate step, IReadOnlyList inputs, IReadOnlyList outputs) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Optional description of this step + +`step` [Delegate](https://learn.microsoft.com/dotnet/api/system.delegate) + +The transformation function (Func<TInput, Task<TOutput>>) + +`inputs` [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[IItem](Flowthru.Data.IItem.md)\> + +Catalog entries this step reads + +`outputs` [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[IItem](Flowthru.Data.IItem.md)\> + +Catalog entries this step writes + +## Properties + +### Dependencies + +Other flow steps that must execute before this step. +Populated during dependency analysis by checking which steps produce our inputs. + +```csharp +public List Dependencies { get; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[FlowStep](Flowthru.Flows.FlowStep.md)\> + +#### Remarks + +This forms the edges of the execution DAG: +- If step A produces output X, and step B consumes input X, then B depends on A. +- Topological sort uses these dependencies to determine execution order. + +### Description + +String description of the step's purpose. + +```csharp +public string Description { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Inputs + +Catalog entries that this step reads as input. +These may be produced by other steps (dependencies) or be external prerequisites. + +```csharp +public IReadOnlyList Inputs { get; } +``` + +#### Property Value + + [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[IItem](Flowthru.Data.IItem.md)\> + +### Label + +Unique identifier for this step within the flow. +Typically the step type name or user-provided name. + +```csharp +public string Label { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Layer + +Execution layer determined by topological sort. +Steps in layer 0 have no dependencies. Steps in layer N depend on steps in layers 0..N-1. + +```csharp +public int Layer { get; set; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### Outputs + +Catalog entries that this step writes as output. +Per the single producer rule, each entry here must be unique across all steps. + +```csharp +public IReadOnlyList Outputs { get; } +``` + +#### Property Value + + [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[IItem](Flowthru.Data.IItem.md)\> + +### TransformFunction + +The transformation function that performs the step's work. +Type-erased to Delegate since we need to store different function signatures together. + +```csharp +public Delegate TransformFunction { get; } +``` + +#### Property Value + + [Delegate](https://learn.microsoft.com/dotnet/api/system.delegate) + +#### Remarks + +

+At execution time, this delegate will be invoked via DynamicInvoke with the +appropriate input parameter(s). The function signature can be either synchronous +or asynchronous: +- Sync single: Func<TInput, TOutput> +- Async single: Func<TInput, Task<TOutput>> +- Sync multi-input: Func<(TIn1, TIn2, ...), TOutput> +- Async multi-input: Func<(TIn1, TIn2, ...), Task<TOutput>> +- Sync multi-output: Func<TInput, (TOut1, TOut2, ...)> +- Async multi-output: Func<TInput, Task<(TOut1, TOut2, ...)>> +

+

+Optional Cancellation Support: Steps can opt-in to cancellation awareness +by accepting a CancellationToken as the last parameter: +- Func<TInput, CancellationToken, Task<TOutput>> +- Func<(TIn1, TIn2), CancellationToken, Task<TOutput>> +

+

+When a Step accepts a CancellationToken, the flow will pass the runtime token during +execution, allowing the step to cancel long-running operations cooperatively. Steps that +do not accept a CancellationToken will only be cancelled between step executions. +

+

+The execution engine detects whether the result is a Task and awaits it if needed. +

+ +## Methods + +### ToString\(\) + +Returns a string representation for debugging. + +```csharp +public override string ToString() +``` + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.StepResult.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.StepResult.md new file mode 100644 index 00000000..b5fbab03 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.StepResult.md @@ -0,0 +1,161 @@ +# Class StepResult + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Represents the execution result of a single flow step. + +```csharp +public class StepResult +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[StepResult](Flowthru.Flows.StepResult.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### Exception + +Exception that occurred during step execution, if any. + +```csharp +public Exception? Exception { get; init; } +``` + +#### Property Value + + [Exception](https://learn.microsoft.com/dotnet/api/system.exception)? + +#### Remarks + +Null if Success is true. Contains the exception that caused +the step to fail if Success is false. + +### ExecutionTime + +Execution time for this specific step. + +```csharp +public TimeSpan ExecutionTime { get; init; } +``` + +#### Property Value + + [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +### InputCount + +Number of input items processed by this step. + +```csharp +public int InputCount { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +For multi-input steps, this represents the total count across +all input catalog entries. + +### OutputCount + +Number of output items produced by this step. + +```csharp +public int OutputCount { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +For multi-output steps, this represents the total count across +all output catalog entries. + +### StepName + +The name of the step that was executed. + +```csharp +public required string StepName { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Success + +Indicates whether the step executed successfully. + +```csharp +public bool Success { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### CreateFailure\(string, TimeSpan, Exception, int\) + +Creates a failed step result. + +```csharp +public static StepResult CreateFailure(string stepName, TimeSpan executionTime, Exception exception, int inputCount = 0) +``` + +#### Parameters + +`stepName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +`executionTime` [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +`exception` [Exception](https://learn.microsoft.com/dotnet/api/system.exception) + +`inputCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Returns + + [StepResult](Flowthru.Flows.StepResult.md) + +### CreateSuccess\(string, TimeSpan, int, int\) + +Creates a successful step result. + +```csharp +public static StepResult CreateSuccess(string stepName, TimeSpan executionTime, int inputCount, int outputCount) +``` + +#### Parameters + +`stepName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +`executionTime` [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +`inputCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +`outputCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Returns + + [StepResult](Flowthru.Flows.StepResult.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Validation.IFlowValidationHook.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Validation.IFlowValidationHook.md new file mode 100644 index 00000000..82f1b933 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Validation.IFlowValidationHook.md @@ -0,0 +1,86 @@ +# Interface IFlowValidationHook + +Namespace: [Flowthru.Flows.Validation](Flowthru.Flows.Validation.md) +Assembly: Flowthru.Core.dll + +Validation hook that runs during flow pre-flight checks. + +```csharp +public interface IFlowValidationHook +``` + +## Remarks + +

+Validation hooks provide an extensibility point for extensions to contribute +their own validation logic during the pre-flight phase. Hooks are invoked after +DAG analysis but before external input inspection, allowing extensions to +validate their own step types. +

+

+Example use cases: +

  • Python extension validates @step decorators match C# types
  • Custom extensions validate step-specific configuration
  • Third-party plugins validate external dependencies
+

+

+Hook execution order: +

+
  1. Flow.Build() - DAG construction and layer assignment
  2. ValidationHooks.ValidateAsync() - Extension-specific validation
  3. Flow.ValidateExternalInputsAsync() - External input inspection
+

+Error handling: +Hooks should return ValidationResult with errors, never throw exceptions. +Multiple hooks may run, and all errors are aggregated into a single result. +

+ +## Methods + +### ValidateAsync\(Flow, CancellationToken\) + +Validates flow steps during pre-flight checks. + +```csharp +Task ValidateAsync(Flow flow, CancellationToken cancellationToken) +``` + +#### Parameters + +`flow` [Flow](Flowthru.Flows.Flow.md) + +The flow being validated + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token for async operations + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Validation result containing any errors found + +#### Remarks + +

+Implementations should: +

  • Never throw exceptions (return errors in ValidationResult)
  • Be idempotent (safe to call multiple times)
  • Be reasonably fast (executed during pre-flight, blocks flow start)
  • Only validate steps they understand (ignore other step types)
+

+

+Example implementation (Python extension): +

+
public async Task<ValidationResult> ValidateAsync(
+  Flow flow,
+  CancellationToken cancellationToken)
+{
+  var result = ValidationResult.Success();
+
+  foreach (var step in flow.Steps)
+  {
+    if (IsPythonStep(step))
+    {
+      var stepResult = await ValidatePythonStep(step, cancellationToken);
+      result.Merge(stepResult);
+    }
+  }
+
+  return result;
+}
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Validation.ValidationOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Validation.ValidationOptions.md new file mode 100644 index 00000000..4bd73c73 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Validation.ValidationOptions.md @@ -0,0 +1,106 @@ +# Class ValidationOptions + +Namespace: [Flowthru.Flows.Validation](Flowthru.Flows.Validation.md) +Assembly: Flowthru.Core.dll + +Configuration for pipeline validation behavior. + +```csharp +public class ValidationOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+ValidationOptions provides pipeline-level overrides for validation configuration. +The primary mechanism for validation configuration is catalog-level via the + property and the fluent +.WithInspectionLevel() API. +

+

+Default Behavior (if not configured): +

+
  • Catalog entry has PreferredInspectionLevel set → use that level
  • Layer 0 inputs → Shallow inspection (all storage adapters support inspection)
  • All intermediate outputs (Layer 1+) → None (never inspected)
+

+Catalog-Level Configuration (Recommended): +

+
public ICatalogDataset<Company> Companies =>
+  GetOrCreateDataset(() => new CsvCatalogDataset<Company>("companies", "data/companies.csv")
+    .WithInspectionLevel(InspectionLevel.Deep));
+

+Pipeline-Level Override (Advanced): +

+
builder
+  .RegisterPipeline<MyCatalog>("data_processing", MyPipeline.Create)
+  .WithValidation(validation => {
+    // Override catalog-level setting for this specific pipeline
+    validation.Inspect(catalog.Companies, InspectionLevel.Shallow); // Temporarily use shallow
+  });
+

+Design Rationale: +

+

+Validation configuration is primarily a property of the data source itself, not the pipeline +consuming it. Critical external datasets should always be deeply validated, regardless of +which pipeline uses them. Pipeline-level overrides exist for rare cases where different +validation is needed temporarily (e.g., performance testing, debugging). +

+ +## Methods + +### Default\(\) + +Creates a new ValidationOptions instance with default settings. + +```csharp +public static ValidationOptions Default() +``` + +#### Returns + + [ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md) + +### Inspect\(IItem, InspectionLevel\) + +Specifies the inspection level for a specific catalog entry. + +```csharp +public ValidationOptions Inspect(IItem catalogEntry, InspectionLevel level) +``` + +#### Parameters + +`catalogEntry` [IItem](Flowthru.Data.IItem.md) + +The catalog entry to configure + +`level` [InspectionLevel](Flowthru.Data.Validation.InspectionLevel.md) + +The inspection level to use for this entry + +#### Returns + + [ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md) + +This ValidationOptions instance for fluent chaining + +#### Remarks + +This configuration only applies to Layer 0 inputs (external data). +Intermediate outputs are never inspected regardless of this setting. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Validation.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Validation.md new file mode 100644 index 00000000..ccd36dc4 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.Validation.md @@ -0,0 +1,14 @@ +# Namespace Flowthru.Flows.Validation + +### Classes + + [ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md) + +Configuration for pipeline validation behavior. + +### Interfaces + + [IFlowValidationHook](Flowthru.Flows.Validation.IFlowValidationHook.md) + +Validation hook that runs during flow pre-flight checks. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.ValidationDepth.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.ValidationDepth.md new file mode 100644 index 00000000..f327331f --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.ValidationDepth.md @@ -0,0 +1,27 @@ +# Enum ValidationDepth + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Controls how deeply a dry run validates the pipeline before stopping. + +```csharp +public enum ValidationDepth +``` + +## Fields + +`Full = 1` + +Structure validation plus external data presence checks (default dry-run behaviour). + + + +`StructureOnly = 0` + +Validates graph structure only: no cycles, all node type contracts satisfied, +all catalog entry dependencies wired, and all validation hooks run. +No data source access. + + + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.md new file mode 100644 index 00000000..64e76941 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Flows.md @@ -0,0 +1,50 @@ +# Namespace Flowthru.Flows + +### Namespaces + + [Flowthru.Flows.Validation](Flowthru.Flows.Validation.md) + +### Classes + + [ExecutionOptions](Flowthru.Flows.ExecutionOptions.md) + +Configuration options for pipeline execution. + + [Flow](Flowthru.Flows.Flow.md) + +Represents a complete data flow with steps, dependencies, and execution order. + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +Fluent builder for constructing type-safe flows with function-based steps. + + [FlowResult](Flowthru.Flows.FlowResult.md) + +Represents the result of a flow execution. + + [FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md) + +Defines a strategy for slicing a flow to execute a subset of nodes. + + [FlowStep](Flowthru.Flows.FlowStep.md) + +Represents a step within a flow, wrapping the transformation function with metadata +about its inputs, outputs, and dependencies. + + [StepResult](Flowthru.Flows.StepResult.md) + +Represents the execution result of a single flow step. + +### Structs + + [DryRunOption](Flowthru.Flows.DryRunOption.md) + +Represents a dry-run configuration. Can be assigned from a +or a value. + +### Enums + + [ValidationDepth](Flowthru.Flows.ValidationDepth.md) + +Controls how deeply a dry run validates the pipeline before stopping. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.FlowthruMetadataBuilder.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.FlowthruMetadataBuilder.md new file mode 100644 index 00000000..7eca12e9 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.FlowthruMetadataBuilder.md @@ -0,0 +1,123 @@ +# Class FlowthruMetadataBuilder + +Namespace: [Flowthru.Meta](Flowthru.Meta.md) +Assembly: Flowthru.Core.dll + +Fluent builder for configuring metadata providers and export settings. + +```csharp +public class FlowthruMetadataBuilder +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowthruMetadataBuilder](Flowthru.Meta.FlowthruMetadataBuilder.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Use this builder to register metadata providers with custom configuration. +Providers are executed in registration order during metadata export. +

+

+Example usage: +

+
builder.ConfigureMetadata(meta => meta
+    .AddProvider<JsonMetadataProvider, JsonMetadataProviderBuilder>(json => json
+        .WithOutputDirectory("metadata")
+        .WithTimestamp("yyyy-MM-dd_HH-mm-ss")
+        .UseCompactFormat())
+    .AddProvider<MermaidMetadataProvider, MermaidMetadataProviderBuilder>(mermaid => mermaid
+        .WithOutputDirectory("metadata")
+        .WithDirection(MermaidMetadataProvider.MermaidFlowchartDirection.LeftToRight))
+);
+ +## Methods + +### AddProvider\(Action?\) + +Adds a metadata provider with optional configuration. + +```csharp +public FlowthruMetadataBuilder AddProvider(Action? configure = null) where TProvider : IMetadataProvider where TBuilder : new() +``` + +#### Parameters + +`configure` [Action](https://learn.microsoft.com/dotnet/api/system.action\-1)? + +Optional configuration action for the provider's builder + +#### Returns + + [FlowthruMetadataBuilder](Flowthru.Meta.FlowthruMetadataBuilder.md) + +This builder for fluent chaining + +#### Type Parameters + +`TProvider` + +The metadata provider type (must implement and have ) + +`TBuilder` + +The builder type for the provider + +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown when provider type lacks or builder type mismatch + +### AddProvider\(IMetadataProvider\) + +Adds a custom metadata provider instance directly. + +```csharp +public FlowthruMetadataBuilder AddProvider(IMetadataProvider provider) +``` + +#### Parameters + +`provider` [IMetadataProvider](Flowthru.Meta.Providers.IMetadataProvider.md) + +The metadata provider to register + +#### Returns + + [FlowthruMetadataBuilder](Flowthru.Meta.FlowthruMetadataBuilder.md) + +This builder for fluent chaining + +### WithAutoExport\(bool\) + +Enables or disables automatic metadata export during pipeline execution. + +```csharp +public FlowthruMetadataBuilder WithAutoExport(bool enabled = true) +``` + +#### Parameters + +`enabled` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +True to auto-export (default), false to require manual export + +#### Returns + + [FlowthruMetadataBuilder](Flowthru.Meta.FlowthruMetadataBuilder.md) + +This builder for fluent chaining + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.FlowthruMetadataConfiguration.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.FlowthruMetadataConfiguration.md new file mode 100644 index 00000000..a7f87888 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.FlowthruMetadataConfiguration.md @@ -0,0 +1,181 @@ +# Class FlowthruMetadataConfiguration + +Namespace: [Flowthru.Meta](Flowthru.Meta.md) +Assembly: Flowthru.Core.dll + +Configuration for Flowthru metadata collection and export. + +```csharp +public class FlowthruMetadataConfiguration +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowthruMetadataConfiguration](Flowthru.Meta.FlowthruMetadataConfiguration.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This configuration controls whether and how pipeline metadata is collected +and persisted. Metadata includes DAG structure (nodes, catalog entries, edges) +that can be consumed by Flowthru.Viz for visualization. +

+

+Usage: +

+
builder.IncludeMetadata(metadata => {
+    metadata
+        .WithOutputDirectory("Data/Metadata")
+        .EnableAutoExport();
+});
+ +## Properties + +### AutoExportDag + +Whether to automatically export DAG metadata after Pipeline.Build(). + +```csharp +public bool AutoExportDag { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Default: true +When enabled, DAG JSON files are automatically created after each pipeline build. + +### ExportMermaid + +Whether to export Mermaid diagram files (.md) alongside JSON files. + +```csharp +public bool ExportMermaid { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Default: true +When enabled, a Markdown file with an embedded Mermaid diagram is created +alongside each JSON file for immediate visualization. + +### OutputDirectory + +Directory where metadata JSON files will be written. + +```csharp +public string OutputDirectory { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Default: "Data/Metadata" + +## Methods + +### DisableAutoExport\(\) + +Disables automatic DAG export. + +```csharp +public FlowthruMetadataConfiguration DisableAutoExport() +``` + +#### Returns + + [FlowthruMetadataConfiguration](Flowthru.Meta.FlowthruMetadataConfiguration.md) + +This configuration for fluent chaining + +#### Remarks + +Use this when you want manual control over metadata export via Pipeline.ExportDag(). + +### DisableMermaid\(\) + +Disables Mermaid diagram export. + +```csharp +public FlowthruMetadataConfiguration DisableMermaid() +``` + +#### Returns + + [FlowthruMetadataConfiguration](Flowthru.Meta.FlowthruMetadataConfiguration.md) + +This configuration for fluent chaining + +#### Remarks + +Use this when you only want JSON output without Mermaid diagrams. + +### EnableAutoExport\(\) + +Enables automatic DAG export after pipeline builds. + +```csharp +public FlowthruMetadataConfiguration EnableAutoExport() +``` + +#### Returns + + [FlowthruMetadataConfiguration](Flowthru.Meta.FlowthruMetadataConfiguration.md) + +This configuration for fluent chaining + +### EnableMermaid\(\) + +Enables Mermaid diagram export. + +```csharp +public FlowthruMetadataConfiguration EnableMermaid() +``` + +#### Returns + + [FlowthruMetadataConfiguration](Flowthru.Meta.FlowthruMetadataConfiguration.md) + +This configuration for fluent chaining + +### WithOutputDirectory\(string\) + +Sets the output directory for metadata files. + +```csharp +public FlowthruMetadataConfiguration WithOutputDirectory(string directory) +``` + +#### Parameters + +`directory` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Directory path (absolute or relative to working directory) + +#### Returns + + [FlowthruMetadataConfiguration](Flowthru.Meta.FlowthruMetadataConfiguration.md) + +This configuration for fluent chaining + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.JsonMetadataProviderBuilder.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.JsonMetadataProviderBuilder.md new file mode 100644 index 00000000..7257764e --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.JsonMetadataProviderBuilder.md @@ -0,0 +1,150 @@ +# Class JsonMetadataProviderBuilder + +Namespace: [Flowthru.Meta](Flowthru.Meta.md) +Assembly: Flowthru.Core.dll + +Builder for configuring JSON metadata provider options. + +```csharp +public class JsonMetadataProviderBuilder +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[JsonMetadataProviderBuilder](Flowthru.Meta.JsonMetadataProviderBuilder.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### Build\(\) + +Builds the JSON metadata provider with the configured options. + +```csharp +public JsonMetadataProvider Build() +``` + +#### Returns + + [JsonMetadataProvider](Flowthru.Meta.Providers.JsonMetadataProvider.md) + +A configured instance + +### UseCompactFormat\(\) + +Enables compact JSON format (no indentation). + +```csharp +public JsonMetadataProviderBuilder UseCompactFormat() +``` + +#### Returns + + [JsonMetadataProviderBuilder](Flowthru.Meta.JsonMetadataProviderBuilder.md) + +This builder for fluent chaining + +### UseIndentedFormat\(\) + +Enables indented JSON format (default). + +```csharp +public JsonMetadataProviderBuilder UseIndentedFormat() +``` + +#### Returns + + [JsonMetadataProviderBuilder](Flowthru.Meta.JsonMetadataProviderBuilder.md) + +This builder for fluent chaining + +### WithFilenameTemplate\(string\) + +Sets the filename template for metadata files. + +```csharp +public JsonMetadataProviderBuilder WithFilenameTemplate(string template) +``` + +#### Parameters + +`template` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Template with placeholders: {FlowName}, {Timestamp}, {SliceType} + +#### Returns + + [JsonMetadataProviderBuilder](Flowthru.Meta.JsonMetadataProviderBuilder.md) + +This builder for fluent chaining + +### WithLogger\(ILogger\) + +Sets a custom logger for this provider. + +```csharp +public JsonMetadataProviderBuilder WithLogger(ILogger logger) +``` + +#### Parameters + +`logger` [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger) + +Logger instance + +#### Returns + + [JsonMetadataProviderBuilder](Flowthru.Meta.JsonMetadataProviderBuilder.md) + +This builder for fluent chaining + +### WithOutputDirectory\(string\) + +Sets the output directory for metadata files. + +```csharp +public JsonMetadataProviderBuilder WithOutputDirectory(string directory) +``` + +#### Parameters + +`directory` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Directory path (relative or absolute) + +#### Returns + + [JsonMetadataProviderBuilder](Flowthru.Meta.JsonMetadataProviderBuilder.md) + +This builder for fluent chaining + +### WithTimestamp\(string?\) + +Sets the timestamp format for filename generation. + +```csharp +public JsonMetadataProviderBuilder WithTimestamp(string? format = null) +``` + +#### Parameters + +`format` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Timestamp format string (e.g., "yyyy-MM-dd_HH-mm-ss") + +#### Returns + + [JsonMetadataProviderBuilder](Flowthru.Meta.JsonMetadataProviderBuilder.md) + +This builder for fluent chaining + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MermaidMetadataExtensions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MermaidMetadataExtensions.md new file mode 100644 index 00000000..cea43a55 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MermaidMetadataExtensions.md @@ -0,0 +1,96 @@ +# Class MermaidMetadataExtensions + +Namespace: [Flowthru.Meta](Flowthru.Meta.md) +Assembly: Flowthru.Core.dll + +Extension methods for generating Mermaid diagram representations of DAG metadata. + +```csharp +public static class MermaidMetadataExtensions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[MermaidMetadataExtensions](Flowthru.Meta.MermaidMetadataExtensions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Mermaid diagrams provide immediate visualization in Markdown-compatible tools +(GitHub, VS Code, etc.) without requiring a separate web application. +

+

+The generated diagram uses Mermaid flowchart syntax with: +- Steps as rectangles with rounded corners +- Catalog items as cylindrical database shapes +- Flow subgraphs grouping nodes by their origin flow +- External data (no producer) shown with special styling +- Produced data (has producer) inside their producer's flow subgraph +

+ +## Methods + +### ToMermaidDiagram\(DagMetadata, string, string, string\) + +Generates a Mermaid flowchart representation of the DAG, wrapped in a code fence. + +```csharp +public static string ToMermaidDiagram(this DagMetadata dag, string direction = "TB", string activeStepColor = "#2E7D32", string activeItemColor = "#2E7D32") +``` + +#### Parameters + +`dag` [DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +The DAG metadata to visualize + +`direction` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Flow direction code (TB, LR, BT, RL). Defaults to TB (Top to Bottom). + +`activeStepColor` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Hex color for active (sliced) steps. Defaults to #2E7D32. + +`activeItemColor` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Hex color for active (sliced) catalog items. Defaults to #2E7D32. + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +Complete Markdown document with Mermaid code fence + +#### Remarks + +

+The output is a valid Markdown document that can be saved as a .md file +and rendered by any Mermaid-compatible viewer. +

+

+Example output: +

+
```mermaid
+flowchart TB
+    RawCompanies[("Raw Companies")]
+
+    subgraph DataProcessing["DataProcessing"]
+        PreprocessCompanies["Preprocess Companies"]
+        CleanedCompanies[("Cleaned Companies")]
+        RawCompanies --> PreprocessCompanies
+        PreprocessCompanies --> CleanedCompanies
+    end
+```
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MermaidMetadataProviderBuilder.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MermaidMetadataProviderBuilder.md new file mode 100644 index 00000000..5d278d27 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MermaidMetadataProviderBuilder.md @@ -0,0 +1,182 @@ +# Class MermaidMetadataProviderBuilder + +Namespace: [Flowthru.Meta](Flowthru.Meta.md) +Assembly: Flowthru.Core.dll + +Builder for configuring Mermaid diagram provider options. + +```csharp +public class MermaidMetadataProviderBuilder +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[MermaidMetadataProviderBuilder](Flowthru.Meta.MermaidMetadataProviderBuilder.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### Build\(\) + +Builds the Mermaid metadata provider with the configured options. + +```csharp +public MermaidMetadataProvider Build() +``` + +#### Returns + + [MermaidMetadataProvider](Flowthru.Meta.Providers.MermaidMetadataProvider.md) + +A configured instance + +### WithActiveDataColor\(string\) + +Sets the color for active (sliced) catalog entries. + +```csharp +public MermaidMetadataProviderBuilder WithActiveDataColor(string color) +``` + +#### Parameters + +`color` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Hex color code (e.g., "#2E7D32") + +#### Returns + + [MermaidMetadataProviderBuilder](Flowthru.Meta.MermaidMetadataProviderBuilder.md) + +This builder for fluent chaining + +### WithActiveStepColor\(string\) + +Sets the color for active (sliced) nodes. + +```csharp +public MermaidMetadataProviderBuilder WithActiveStepColor(string color) +``` + +#### Parameters + +`color` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Hex color code (e.g., "#2E7D32") + +#### Returns + + [MermaidMetadataProviderBuilder](Flowthru.Meta.MermaidMetadataProviderBuilder.md) + +This builder for fluent chaining + +### WithDirection\(MermaidFlowchartDirection\) + +Sets the flowchart direction. + +```csharp +public MermaidMetadataProviderBuilder WithDirection(MermaidMetadataProvider.MermaidFlowchartDirection direction) +``` + +#### Parameters + +`direction` [MermaidMetadataProvider](Flowthru.Meta.Providers.MermaidMetadataProvider.md).[MermaidFlowchartDirection](Flowthru.Meta.Providers.MermaidMetadataProvider.MermaidFlowchartDirection.md) + +Direction for the flowchart (TB, LR, BT, RL) + +#### Returns + + [MermaidMetadataProviderBuilder](Flowthru.Meta.MermaidMetadataProviderBuilder.md) + +This builder for fluent chaining + +### WithFilenameTemplate\(string\) + +Sets the filename template for metadata files. + +```csharp +public MermaidMetadataProviderBuilder WithFilenameTemplate(string template) +``` + +#### Parameters + +`template` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Template with placeholders: {FlowName}, {Timestamp}, {SliceType} + +#### Returns + + [MermaidMetadataProviderBuilder](Flowthru.Meta.MermaidMetadataProviderBuilder.md) + +This builder for fluent chaining + +### WithLogger\(ILogger\) + +Sets a custom logger for this provider. + +```csharp +public MermaidMetadataProviderBuilder WithLogger(ILogger logger) +``` + +#### Parameters + +`logger` [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger) + +Logger instance + +#### Returns + + [MermaidMetadataProviderBuilder](Flowthru.Meta.MermaidMetadataProviderBuilder.md) + +This builder for fluent chaining + +### WithOutputDirectory\(string\) + +Sets the output directory for metadata files. + +```csharp +public MermaidMetadataProviderBuilder WithOutputDirectory(string directory) +``` + +#### Parameters + +`directory` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Directory path (relative or absolute) + +#### Returns + + [MermaidMetadataProviderBuilder](Flowthru.Meta.MermaidMetadataProviderBuilder.md) + +This builder for fluent chaining + +### WithTimestamp\(string?\) + +Sets the timestamp format for filename generation. + +```csharp +public MermaidMetadataProviderBuilder WithTimestamp(string? format = null) +``` + +#### Parameters + +`format` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Timestamp format string (e.g., "yyyy-MM-dd_HH-mm-ss") + +#### Returns + + [MermaidMetadataProviderBuilder](Flowthru.Meta.MermaidMetadataProviderBuilder.md) + +This builder for fluent chaining + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MetadataJsonExtensions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MetadataJsonExtensions.md new file mode 100644 index 00000000..0a82239d --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MetadataJsonExtensions.md @@ -0,0 +1,109 @@ +# Class MetadataJsonExtensions + +Namespace: [Flowthru.Meta](Flowthru.Meta.md) +Assembly: Flowthru.Core.dll + +Extension methods for serializing metadata to JSON. + +```csharp +public static class MetadataJsonExtensions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[MetadataJsonExtensions](Flowthru.Meta.MetadataJsonExtensions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### FromJson\(string\) + +Deserializes DagMetadata from JSON string. + +```csharp +public static DagMetadata FromJson(string json) +``` + +#### Parameters + +`json` [string](https://learn.microsoft.com/dotnet/api/system.string) + +JSON string to deserialize + +#### Returns + + [DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +Deserialized DagMetadata object + +#### Exceptions + + [JsonException](https://learn.microsoft.com/dotnet/api/system.text.json.jsonexception) + +Thrown if JSON is invalid or doesn't match schema + +### ToCompactJson\(DagMetadata\) + +Serializes DagMetadata to compact JSON string (no indentation). + +```csharp +public static string ToCompactJson(this DagMetadata metadata) +``` + +#### Parameters + +`metadata` [DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +The DAG metadata to serialize + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +Compact JSON string representation + +#### Remarks + +Use this for minimizing file size when human readability is not a concern, +such as API responses or embedded metadata. + +### ToJson\(DagMetadata\) + +Serializes DagMetadata to pretty-printed JSON string. + +```csharp +public static string ToJson(this DagMetadata metadata) +``` + +#### Parameters + +`metadata` [DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +The DAG metadata to serialize + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +JSON string representation + +#### Remarks + +

+Output format uses: +

+
  • camelCase property names (pipelineName, not FlowName)
  • Indented formatting for readability
  • Null properties omitted
  • Enums serialized as strings
+

+This format is optimized for Flowthru.Viz consumption and human readability. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MetadataProviderBuilderAttribute.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MetadataProviderBuilderAttribute.md new file mode 100644 index 00000000..364837c4 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.MetadataProviderBuilderAttribute.md @@ -0,0 +1,113 @@ +# Class MetadataProviderBuilderAttribute + +Namespace: [Flowthru.Meta](Flowthru.Meta.md) +Assembly: Flowthru.Core.dll + +Specifies the builder type for a metadata provider. + +```csharp +[AttributeUsage(AttributeTargets.Class, Inherited = false, AllowMultiple = false)] +public sealed class MetadataProviderBuilderAttribute : Attribute +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Attribute](https://learn.microsoft.com/dotnet/api/system.attribute) ← +[MetadataProviderBuilderAttribute](Flowthru.Meta.MetadataProviderBuilderAttribute.md) + +#### Inherited Members + +[Attribute.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.attribute.equals), +[Attribute.GetCustomAttribute\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-assembly\-system\-type\)), +[Attribute.GetCustomAttribute\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.GetCustomAttribute\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-module\-system\-type\)), +[Attribute.GetCustomAttribute\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttribute\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.GetCustomAttribute\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Assembly\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\)), +[Attribute.GetCustomAttributes\(Assembly, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-type\)), +[Attribute.GetCustomAttributes\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(MemberInfo\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\)), +[Attribute.GetCustomAttributes\(MemberInfo, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-boolean\)), +[Attribute.GetCustomAttributes\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.GetCustomAttributes\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Module\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\)), +[Attribute.GetCustomAttributes\(Module, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-boolean\)), +[Attribute.GetCustomAttributes\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-type\)), +[Attribute.GetCustomAttributes\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.GetCustomAttributes\(ParameterInfo\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\)), +[Attribute.GetCustomAttributes\(ParameterInfo, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-boolean\)), +[Attribute.GetCustomAttributes\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.GetCustomAttributes\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.attribute.gethashcode), +[Attribute.IsDefaultAttribute\(\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefaultattribute), +[Attribute.IsDefined\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-assembly\-system\-type\)), +[Attribute.IsDefined\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-assembly\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-memberinfo\-system\-type\)), +[Attribute.IsDefined\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-module\-system\-type\)), +[Attribute.IsDefined\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-module\-system\-type\-system\-boolean\)), +[Attribute.IsDefined\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-parameterinfo\-system\-type\)), +[Attribute.IsDefined\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)), +[Attribute.Match\(object?\)](https://learn.microsoft.com/dotnet/api/system.attribute.match), +[Attribute.TypeId](https://learn.microsoft.com/dotnet/api/system.attribute.typeid), +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This attribute enables type-safe provider registration via the generic +FlowthruMetadataBuilder.AddProvider<TProvider>(Action<object>?) method. +

+

+The builder type must: +

+
  • Have a public parameterless constructor
  • Expose a public Build() method returning
+

+Example: +

+
[MetadataProviderBuilder(typeof(JsonMetadataProviderBuilder))]
+public class JsonMetadataProvider : IMetadataProvider
+{
+  // ...
+}
+ +## Constructors + +### MetadataProviderBuilderAttribute\(Type\) + +Initializes a new instance of the class. + +```csharp +public MetadataProviderBuilderAttribute(Type builderType) +``` + +#### Parameters + +`builderType` [Type](https://learn.microsoft.com/dotnet/api/system.type) + +The type of the provider's builder class + +## Properties + +### BuilderType + +Gets the builder type for this metadata provider. + +```csharp +public Type BuilderType { get; } +``` + +#### Property Value + + [Type](https://learn.microsoft.com/dotnet/api/system.type) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.CatalogEntryMetadata.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.CatalogEntryMetadata.md new file mode 100644 index 00000000..7f228408 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.CatalogEntryMetadata.md @@ -0,0 +1,160 @@ +# Class ItemMetadata + +Namespace: [Flowthru.Meta.Models](Flowthru.Meta.Models.md) +Assembly: Flowthru.Core.dll + +Metadata describing a single catalog entry (dataset) in the pipeline. + +```csharp +public class ItemMetadata +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ItemMetadata](Flowthru.Meta.Models.ItemMetadata.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Catalog entries represent data sources and sinks. They can be external files, +intermediate pipeline outputs, or final results. Each entry is uniquely identified +by its key. + +## Properties + +### Consumers + +List of node IDs that consume (read from) this catalog entry. + +```csharp +[JsonPropertyName("consumers")] +public List Consumers { get; init; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +#### Remarks + +Empty for pipeline outputs that aren't consumed by other nodes. +Example: ["CreateModelInputTable", "ValidateData"] + +### DataType + +The C# type name of data stored in this catalog entry. + +```csharp +[JsonPropertyName("dataType")] +public required string DataType { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Simple type name without namespace. +Example: "Company", "Shuttle", "ModelInput" + +### Fields + +Additional metadata fields specific to the catalog entry type. + +```csharp +[JsonPropertyName("fields")] +public Dictionary Fields { get; init; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [object](https://learn.microsoft.com/dotnet/api/system.object)\> + +#### Remarks + +

Examples of fields:

+
  • filepath: Path to file for file-based datasets
  • catalogType: Type of catalog dataset (CsvCatalogDataset, ParquetCatalogDataset, etc.)
  • isReadOnly: Whether the dataset is read-only
  • inspectionLevel: Validation inspection level (None, Shallow, Deep)
+ +### Key + +Unique key identifying this catalog entry. + +```csharp +[JsonPropertyName("key")] +public required string Key { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Corresponds to the catalog property name or explicitly set key. +Example: "Companies", "CleanedCompanies", "ModelInputTable" + +### Label + +Human-readable display label for this catalog entry. + +```csharp +[JsonPropertyName("label")] +public required string Label { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +May be formatted for better display in Flowthru.Viz. +Example: "Companies", "Cleaned Companies", "Model Input Table" + +### Producer + +Node ID that produces (writes to) this catalog entry. + +```csharp +[JsonPropertyName("producer")] +public string? Producer { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +Null for external inputs (Layer 0 inputs that exist before pipeline execution). +Example: "PreprocessCompanies" + +### Schema + +Schema information inferred from the data type. + +```csharp +[JsonPropertyName("schema")] +public SchemaMetadata? Schema { get; init; } +``` + +#### Property Value + + [SchemaMetadata](Flowthru.Meta.Models.SchemaMetadata.md)? + +#### Remarks + +Null for simple types or when schema inference fails. +Contains property names, types, and nullability for complex types. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.DagMetadata.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.DagMetadata.md new file mode 100644 index 00000000..eacd572d --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.DagMetadata.md @@ -0,0 +1,172 @@ +# Class DagMetadata + +Namespace: [Flowthru.Meta.Models](Flowthru.Meta.Models.md) +Assembly: Flowthru.Core.dll + +Root metadata model representing a complete FlowthruService DAG (Directed Acyclic Graph). + +```csharp +public class DagMetadata +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +#### Extension Methods + +[MetadataJsonExtensions.ToCompactJson\(DagMetadata\)](Flowthru.Meta.MetadataJsonExtensions.md\#Flowthru\_Meta\_MetadataJsonExtensions\_ToCompactJson\_Flowthru\_Meta\_Models\_DagMetadata\_), +[MetadataJsonExtensions.ToJson\(DagMetadata\)](Flowthru.Meta.MetadataJsonExtensions.md\#Flowthru\_Meta\_MetadataJsonExtensions\_ToJson\_Flowthru\_Meta\_Models\_DagMetadata\_), +[MermaidMetadataExtensions.ToMermaidDiagram\(DagMetadata, string, string, string\)](Flowthru.Meta.MermaidMetadataExtensions.md\#Flowthru\_Meta\_MermaidMetadataExtensions\_ToMermaidDiagram\_Flowthru\_Meta\_Models\_DagMetadata\_System\_String\_System\_String\_System\_String\_) + +## Remarks + +This model captures the structure of a built flow, including all steps, +catalog items, and their relationships. It serves as the backbone for +Flowthru.Viz visualization. + +## Properties + +### AppliedSlice + +Slice criteria applied to generate this DAG, if any. + +```csharp +[JsonPropertyName("appliedSlice")] +[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] +public DagSliceMetadata? AppliedSlice { get; init; } +``` + +#### Property Value + + [DagSliceMetadata](Flowthru.Meta.Models.DagSliceMetadata.md)? + +#### Remarks + +Present when the DAG represents a sliced subset of the full Flow. +Null when the DAG represents the complete, unsliced flow. +Used for reproducibility, debugging, and filename generation. + +### CatalogItems + +All catalog items involved in the flow. + +```csharp +[JsonPropertyName("catalogItems")] +public List CatalogItems { get; init; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[ItemMetadata](Flowthru.Meta.Models.ItemMetadata.md)\> + +### Edges + +All edges representing data flow in the DAG. + +```csharp +[JsonPropertyName("edges")] +public List Edges { get; init; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[EdgeMetadata](Flowthru.Meta.Models.EdgeMetadata.md)\> + +#### Remarks + +Edges connect catalog items to steps and steps to catalog items, +forming the complete graph. + +### FlowName + +Name of the flow this DAG represents. + +```csharp +[JsonPropertyName("flowName")] +public required string FlowName { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### GeneratedAt + +Timestamp when this metadata was generated. + +```csharp +[JsonPropertyName("generatedAt")] +public DateTime GeneratedAt { get; init; } +``` + +#### Property Value + + [DateTime](https://learn.microsoft.com/dotnet/api/system.datetime) + +### SlicedCatalogItemIds + +Catalog item IDsthat are produced by steps in the active execution slice. + +```csharp +[JsonPropertyName("slicedCatalogItemIds")] +[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] +public HashSet? SlicedCatalogItemIds { get; init; } +``` + +#### Property Value + + [HashSet](https://learn.microsoft.com/dotnet/api/system.collections.generic.hashset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +When a slice is applied, this contains the keys of catalog items (data) that +will be written during execution. Derived from the outputs of sliced steps. +Null when no slice was applied (all data may be updated). +Enables visualization tools to highlight both steps and the data they produce. + +### SlicedStepIds + +Step IDs that are in the active execution slice, if a slice was applied. + +```csharp +[JsonPropertyName("slicedStepIds")] +[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] +public HashSet? SlicedStepIds { get; init; } +``` + +#### Property Value + + [HashSet](https://learn.microsoft.com/dotnet/api/system.collections.generic.hashset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +When a slice is applied, this contains the IDs of steps that will actually execute. +The Steps collection contains the full DAG, while this set identifies the subset. +Null when no slice was applied (all steps execute). +Enables visualization tools to highlight execution paths while showing full context. + +### Steps + +All steps in the flow with their metadata. + +```csharp +[JsonPropertyName("steps")] +public List Steps { get; init; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[StepMetadata](Flowthru.Meta.Models.StepMetadata.md)\> + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.DagSliceMetadata.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.DagSliceMetadata.md new file mode 100644 index 00000000..579cf6ad --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.DagSliceMetadata.md @@ -0,0 +1,118 @@ +# Class DagSliceMetadata + +Namespace: [Flowthru.Meta.Models](Flowthru.Meta.Models.md) +Assembly: Flowthru.Core.dll + +Metadata describing how a pipeline was sliced during execution. + +```csharp +public class DagSliceMetadata +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[DagSliceMetadata](Flowthru.Meta.Models.DagSliceMetadata.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Captures the criteria used to select a subset of nodes from the full pipeline DAG. +This information is essential for: +
  • Reproducibility - rerun the exact same slice
  • Debugging - understand what was included/excluded when failures occur
  • Auditing - track which pipeline subsets were executed in production
  • Visualization - indicate sliced vs. full DAG in metadata exports
+ +## Properties + +### FromData + +Catalog entry labels whose consumers are included (expanded downstream). + +```csharp +[JsonPropertyName("fromData")] +[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] +public string[]? FromData { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)\[\]? + +### FromNodes + +Node names from which the slice expanded downstream (dependents included). + +```csharp +[JsonPropertyName("fromNodes")] +[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] +public string[]? FromNodes { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)\[\]? + +### OnlyNodes + +Explicit allowlist of node names (with dependencies auto-included). + +```csharp +[JsonPropertyName("onlyNodes")] +[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] +public string[]? OnlyNodes { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)\[\]? + +### Pipelines + +Pipeline names to include in the merged DAG. + +```csharp +[JsonPropertyName("pipelines")] +[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] +public string[]? Pipelines { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)\[\]? + +### ToData + +Catalog entry labels whose producers are included (expanded upstream). + +```csharp +[JsonPropertyName("toData")] +[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] +public string[]? ToData { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)\[\]? + +### ToNodes + +Node names to which the slice expanded upstream (dependencies included). + +```csharp +[JsonPropertyName("toNodes")] +[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] +public string[]? ToNodes { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)\[\]? + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.EdgeMetadata.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.EdgeMetadata.md new file mode 100644 index 00000000..b12f1eac --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.EdgeMetadata.md @@ -0,0 +1,94 @@ +# Class EdgeMetadata + +Namespace: [Flowthru.Meta.Models](Flowthru.Meta.Models.md) +Assembly: Flowthru.Core.dll + +Metadata describing an edge in the pipeline DAG. + +```csharp +public class EdgeMetadata +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EdgeMetadata](Flowthru.Meta.Models.EdgeMetadata.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Edges represent data flow between catalog entries and nodes. The DAG contains +two types of edges: +

+
  • Catalog → Node: A node reads from a catalog entry
  • Node → Catalog: A node writes to a catalog entry
+

+Together, these edges form the complete data flow: +Item → Node → Item → Node → ... +

+ +## Properties + +### DataType + +C# type name of data flowing through this edge. + +```csharp +[JsonPropertyName("dataType")] +public required string DataType { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Simple type name without namespace. +Example: "Company", "Shuttle", "ModelInput" + +### Source + +Source identifier (either a catalog entry key or node ID). + +```csharp +[JsonPropertyName("source")] +public required string Source { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +For Catalog → Node edges, this is a catalog entry key. +For Node → Catalog edges, this is a node ID. + +### Target + +Target identifier (either a node ID or catalog entry key). + +```csharp +[JsonPropertyName("target")] +public required string Target { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +For Catalog → Node edges, this is a node ID. +For Node → Catalog edges, this is a catalog entry key. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.ItemMetadata.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.ItemMetadata.md new file mode 100644 index 00000000..d9141f16 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.ItemMetadata.md @@ -0,0 +1,160 @@ +# Class ItemMetadata + +Namespace: [Flowthru.Meta.Models](Flowthru.Meta.Models.md) +Assembly: Flowthru.Core.dll + +Metadata describing a single catalog item (dataset, json object, API response, whatever) in the flow. + +```csharp +public class ItemMetadata +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ItemMetadata](Flowthru.Meta.Models.ItemMetadata.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Catalog items represent data sources and sinks. They can be external files, +intermediate flow outputs, or final results. Each item is uniquely identified +by its key. + +## Properties + +### Consumers + +List of step IDs that consume (read from) this catalog item. + +```csharp +[JsonPropertyName("consumers")] +public List Consumers { get; init; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +#### Remarks + +Empty for item outputs that aren't consumed by other steps. +Example: ["CreateModelInputTable", "ValidateData"] + +### DataType + +The C# type name of data stored in this catalog item. + +```csharp +[JsonPropertyName("dataType")] +public required string DataType { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Simple type name without namespace. +Example: "Company", "Shuttle", "ModelInput" + +### Fields + +Additional metadata fields specific to the catalog item type. + +```csharp +[JsonPropertyName("fields")] +public Dictionary Fields { get; init; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [object](https://learn.microsoft.com/dotnet/api/system.object)\> + +#### Remarks + +

Examples of fields:

+
  • filepath: Path to file for file-based datasets
  • catalogType: Type of catalog dataset (CsvCatalogDataset, ParquetCatalogDataset, etc.)
  • isReadOnly: Whether the dataset is read-only
  • inspectionLevel: Validation inspection level (None, Shallow, Deep)
+ +### Key + +Unique key identifying this catalog item. + +```csharp +[JsonPropertyName("key")] +public required string Key { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Corresponds to the catalog property name or explicitly set key. +Example: "Companies", "CleanedCompanies", "ModelInputTable" + +### Label + +Human-readable display label for this catalog item. + +```csharp +[JsonPropertyName("label")] +public required string Label { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +May be formatted for better display in Flowthru.Viz. +Example: "Companies", "Cleaned Companies", "Model Input Table" + +### Producer + +Step ID that produces (writes to) this catalog item. + +```csharp +[JsonPropertyName("producer")] +public string? Producer { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +Null for external inputs (Layer 0 inputs that exist before flow execution). +Example: "PreprocessCompanies" + +### Schema + +Schema information inferred from the data type. + +```csharp +[JsonPropertyName("schema")] +public SchemaMetadata? Schema { get; init; } +``` + +#### Property Value + + [SchemaMetadata](Flowthru.Meta.Models.SchemaMetadata.md)? + +#### Remarks + +Null for simple types or when schema inference fails. +Contains property names, types, and nullability for complex types. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.NodeMetadata.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.NodeMetadata.md new file mode 100644 index 00000000..15982aad --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.NodeMetadata.md @@ -0,0 +1,160 @@ +# Class NodeMetadata + +Namespace: [Flowthru.Meta.Models](Flowthru.Meta.Models.md) +Assembly: Flowthru.Core.dll + +Metadata describing a single node in the pipeline DAG. + +```csharp +public class NodeMetadata +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[NodeMetadata](Flowthru.Meta.Models.NodeMetadata.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Nodes are the processing units in a pipeline. Each node reads from one or more +catalog entries (inputs), performs a transformation, and writes to one or more +catalog entries (outputs). + +## Properties + +### Id + +Unique identifier for this node within the pipeline. + +```csharp +[JsonPropertyName("id")] +public required string Id { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Typically the node name as defined when adding it to the pipeline. +Example: "PreprocessCompanies", "TrainModel" + +### Inputs + +List of catalog entry keys this node reads from. + +```csharp +[JsonPropertyName("inputs")] +public List Inputs { get; init; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +#### Remarks + +For multi-input nodes using CatalogMap, this contains all mapped entries. +Example: ["Companies", "Shuttles", "Reviews"] + +### Label + +Human-readable display label for this node. + +```csharp +[JsonPropertyName("label")] +public required string Label { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +May be formatted for better display in Flowthru.Viz. +Example: "Preprocess Companies", "Train Model" + +### Layer + +Execution layer assigned by the dependency analyzer. + +```csharp +[JsonPropertyName("layer")] +public int Layer { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +Layer 0 nodes have no dependencies (read external data only). +Layer N nodes depend only on nodes in layers 0..N-1. + +### NodeType + +The C# class type name implementing this node. + +```csharp +[JsonPropertyName("nodeType")] +public required string NodeType { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Simple type name without namespace or generic parameters. +Example: "PreprocessCompaniesNode", "TrainModelNode" + +### Outputs + +List of catalog entry keys this node writes to. + +```csharp +[JsonPropertyName("outputs")] +public List Outputs { get; init; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +#### Remarks + +For multi-output nodes using CatalogMap, this contains all mapped entries. +Example: ["XTrain", "XTest", "YTrain", "YTest"] + +### FlowName + +Name of the parent pipeline this node belongs to. + +```csharp +[JsonPropertyName("pipelineName")] +public required string FlowName { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Important for merged pipelines where nodes from multiple pipelines +are combined into a single DAG. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.SchemaField.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.SchemaField.md new file mode 100644 index 00000000..b3d2d426 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.SchemaField.md @@ -0,0 +1,80 @@ +# Class SchemaField + +Namespace: [Flowthru.Meta.Models](Flowthru.Meta.Models.md) +Assembly: Flowthru.Core.dll + +A single field (property) in a schema. + +```csharp +public class SchemaField +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[SchemaField](Flowthru.Meta.Models.SchemaField.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### IsNullable + +Whether the property can be null. + +```csharp +[JsonPropertyName("isNullable")] +public bool IsNullable { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Determined by nullable reference types (string?) or nullable value types (int?). + +### Name + +Name of the property. + +```csharp +[JsonPropertyName("name")] +public required string Name { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Example: "Id", "Name", "IataApproved" + +### Type + +C# type name of the property. + +```csharp +[JsonPropertyName("type")] +public required string Type { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Simple type name without namespace. +Example: "string", "int", "DateTime", "double" + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.SchemaMetadata.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.SchemaMetadata.md new file mode 100644 index 00000000..4c32b9b1 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.SchemaMetadata.md @@ -0,0 +1,47 @@ +# Class SchemaMetadata + +Namespace: [Flowthru.Meta.Models](Flowthru.Meta.Models.md) +Assembly: Flowthru.Core.dll + +Schema information for a data type. + +```csharp +public class SchemaMetadata +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[SchemaMetadata](Flowthru.Meta.Models.SchemaMetadata.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Extracted from C# type definitions using reflection. Describes the structure +of data flowing through catalog entries, enabling Flowthru.Viz to display +data schemas and validate type compatibility. + +## Properties + +### Fields + +List of fields (properties) in the schema. + +```csharp +[JsonPropertyName("fields")] +public List Fields { get; init; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[SchemaField](Flowthru.Meta.Models.SchemaField.md)\> + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.StepMetadata.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.StepMetadata.md new file mode 100644 index 00000000..fc2d2087 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.StepMetadata.md @@ -0,0 +1,160 @@ +# Class StepMetadata + +Namespace: [Flowthru.Meta.Models](Flowthru.Meta.Models.md) +Assembly: Flowthru.Core.dll + +Metadata describing a single step in the flow DAG. + +```csharp +public class StepMetadata +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[StepMetadata](Flowthru.Meta.Models.StepMetadata.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Steps are the processing units in a flow. Each step reads from one or more +catalog entries (inputs), performs a transformation, and writes to one or more +catalog entries (outputs). + +## Properties + +### FlowName + +Name of the parent flow this step belongs to. + +```csharp +[JsonPropertyName("flowName")] +public required string FlowName { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Important for merged flows where steps from multiple flows +are combined into a single DAG. + +### Id + +Unique identifier for this step within the flow. + +```csharp +[JsonPropertyName("id")] +public required string Id { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Typically the step name as defined when adding it to the flow. +Example: "PreprocessCompanies", "TrainModel" + +### Inputs + +List of catalog entry keys this step reads from. + +```csharp +[JsonPropertyName("inputs")] +public List Inputs { get; init; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +#### Remarks + +For multi-input steps using CatalogMap, this contains all mapped entries. +Example: ["Companies", "Shuttles", "Reviews"] + +### Label + +Human-readable display label for this step. + +```csharp +[JsonPropertyName("label")] +public required string Label { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +May be formatted for better display in Flowthru.Viz. +Example: "Preprocess Companies", "Train Model" + +### Layer + +Execution layer assigned by the dependency analyzer. + +```csharp +[JsonPropertyName("layer")] +public int Layer { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +Layer 0 steps have no dependencies (read external data only). +Layer N steps depend only on steps in layers 0..N-1. + +### Outputs + +List of catalog entry keys this step writes to. + +```csharp +[JsonPropertyName("outputs")] +public List Outputs { get; init; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +#### Remarks + +For multi-output steps using CatalogMap, this contains all mapped entries. +Example: ["XTrain", "XTest", "YTrain", "YTest"] + +### StepType + +The C# class type name implementing this step. + +```csharp +[JsonPropertyName("stepType")] +public required string StepType { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Simple type name without namespace or generic parameters. +Example: "PreprocessCompaniesStep", "TrainModelStep" + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.md new file mode 100644 index 00000000..3a3badf0 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Models.md @@ -0,0 +1,32 @@ +# Namespace Flowthru.Meta.Models + +### Classes + + [DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +Root metadata model representing a complete FlowthruService DAG (Directed Acyclic Graph). + + [DagSliceMetadata](Flowthru.Meta.Models.DagSliceMetadata.md) + +Metadata describing how a pipeline was sliced during execution. + + [EdgeMetadata](Flowthru.Meta.Models.EdgeMetadata.md) + +Metadata describing an edge in the pipeline DAG. + + [ItemMetadata](Flowthru.Meta.Models.ItemMetadata.md) + +Metadata describing a single catalog item (dataset, json object, API response, whatever) in the flow. + + [SchemaField](Flowthru.Meta.Models.SchemaField.md) + +A single field (property) in a schema. + + [SchemaMetadata](Flowthru.Meta.Models.SchemaMetadata.md) + +Schema information for a data type. + + [StepMetadata](Flowthru.Meta.Models.StepMetadata.md) + +Metadata describing a single step in the flow DAG. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.IMetadataProvider.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.IMetadataProvider.md new file mode 100644 index 00000000..9c09f3c4 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.IMetadataProvider.md @@ -0,0 +1,79 @@ +# Interface IMetadataProvider + +Namespace: [Flowthru.Meta.Providers](Flowthru.Meta.Providers.md) +Assembly: Flowthru.Core.dll + +Interface for metadata consumers. + +```csharp +public interface IMetadataProvider +``` + +## Remarks + +

+Metadata providers receive DAG metadata after pipeline builds and can +process it in any way: write files, send to APIs, store in memory, etc. +

+

+Built-in Providers: +

+
  • - Exports JSON files
  • - Exports Mermaid diagrams
+

+Custom Provider Example: +

+
public class DashboardMetadataProvider : IMetadataProvider
+{
+  private readonly IDashboardClient _client;
+
+  public DashboardMetadataProvider(IDashboardClient client)
+  {
+    _client = client;
+  }
+
+  public string Name => "Dashboard";
+
+  public void Consume(DagMetadata dag)
+  {
+    _client.SendVisualization(dag);
+  }
+}
+ +## Properties + +### Name + +Gets the unique name of this provider. + +```csharp +string Name { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +## Methods + +### Consume\(DagMetadata\) + +Consumes DAG metadata. + +```csharp +void Consume(DagMetadata dag) +``` + +#### Parameters + +`dag` [DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +The DAG metadata to consume + +#### Remarks + +This method is called after pipeline builds. Providers can process +the metadata in any way: write files, send to APIs, store in memory, etc. + +Implementations should handle their own error recovery - exceptions thrown +from this method will be logged but will not fail the pipeline execution. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.JsonMetadataProvider.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.JsonMetadataProvider.md new file mode 100644 index 00000000..af07616e --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.JsonMetadataProvider.md @@ -0,0 +1,107 @@ +# Class JsonMetadataProvider + +Namespace: [Flowthru.Meta.Providers](Flowthru.Meta.Providers.md) +Assembly: Flowthru.Core.dll + +Exports DAG metadata as JSON files. + +```csharp +[MetadataProviderBuilder(typeof(JsonMetadataProviderBuilder))] +public class JsonMetadataProvider : IMetadataProvider +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[JsonMetadataProvider](Flowthru.Meta.Providers.JsonMetadataProvider.md) + +#### Implements + +[IMetadataProvider](Flowthru.Meta.Providers.IMetadataProvider.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +This provider creates timestamped JSON files containing the complete DAG structure +(nodes, catalog entries, edges, schema information) for consumption by Flowthru.Viz +or other visualization tools. + +## Constructors + +### JsonMetadataProvider\(string, string, TimestampConfiguration, bool, ILogger?\) + +Initializes a new JSON metadata provider. + +```csharp +public JsonMetadataProvider(string outputDirectory, string filenameTemplate, TimestampConfiguration timestampConfig, bool useCompactFormat = false, ILogger? logger = null) +``` + +#### Parameters + +`outputDirectory` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Directory to write JSON files to + +`filenameTemplate` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Template for generating output filenames + +`timestampConfig` [TimestampConfiguration](Flowthru.Meta.TimestampConfiguration.md) + +Configuration for timestamp handling in filenames + +`useCompactFormat` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +Whether to use compact (minified) JSON format + +`logger` [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger)? + +Optional logger for diagnostic messages + +## Properties + +### Name + +Gets the unique name of this provider. + +```csharp +public string Name { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +## Methods + +### Consume\(DagMetadata\) + +Consumes DAG metadata. + +```csharp +public void Consume(DagMetadata dag) +``` + +#### Parameters + +`dag` [DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +The DAG metadata to consume + +#### Remarks + +This method is called after pipeline builds. Providers can process +the metadata in any way: write files, send to APIs, store in memory, etc. + +Implementations should handle their own error recovery - exceptions thrown +from this method will be logged but will not fail the pipeline execution. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.MermaidMetadataProvider.MermaidFlowchartDirection.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.MermaidMetadataProvider.MermaidFlowchartDirection.md new file mode 100644 index 00000000..1d11d685 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.MermaidMetadataProvider.MermaidFlowchartDirection.md @@ -0,0 +1,37 @@ +# Enum MermaidMetadataProvider.MermaidFlowchartDirection + +Namespace: [Flowthru.Meta.Providers](Flowthru.Meta.Providers.md) +Assembly: Flowthru.Core.dll + +Flow direction for Mermaid flowcharts. + +```csharp +public enum MermaidMetadataProvider.MermaidFlowchartDirection +``` + +## Fields + +`BottomToTop = 2` + +Bottom to Top + + + +`LeftToRight = 1` + +Left to Right + + + +`RightToLeft = 3` + +Right to Left + + + +`TopToBottom = 0` + +Top to Bottom (default) + + + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.MermaidMetadataProvider.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.MermaidMetadataProvider.md new file mode 100644 index 00000000..9cac2ee7 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.MermaidMetadataProvider.md @@ -0,0 +1,114 @@ +# Class MermaidMetadataProvider + +Namespace: [Flowthru.Meta.Providers](Flowthru.Meta.Providers.md) +Assembly: Flowthru.Core.dll + +Exports DAG metadata as Mermaid flowchart diagrams. + +```csharp +[MetadataProviderBuilder(typeof(MermaidMetadataProviderBuilder))] +public class MermaidMetadataProvider : IMetadataProvider +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[MermaidMetadataProvider](Flowthru.Meta.Providers.MermaidMetadataProvider.md) + +#### Implements + +[IMetadataProvider](Flowthru.Meta.Providers.IMetadataProvider.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +This provider creates Markdown files containing Mermaid flowchart diagrams +for immediate visualization in GitHub, VS Code, and other Mermaid-compatible viewers. + +## Constructors + +### MermaidMetadataProvider\(string, string, TimestampConfiguration, MermaidFlowchartDirection, string, string, ILogger?\) + +Initializes a new Mermaid metadata provider. + +```csharp +public MermaidMetadataProvider(string outputDirectory, string filenameTemplate, TimestampConfiguration timestampConfig, MermaidMetadataProvider.MermaidFlowchartDirection direction = MermaidFlowchartDirection.TopToBottom, string activeNodeColor = "#2E7D32", string activeDataColor = "#2E7D32", ILogger? logger = null) +``` + +#### Parameters + +`outputDirectory` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Directory to write Mermaid files to + +`filenameTemplate` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Template for generating output filenames + +`timestampConfig` [TimestampConfiguration](Flowthru.Meta.TimestampConfiguration.md) + +Configuration for timestamp handling in filenames + +`direction` [MermaidMetadataProvider](Flowthru.Meta.Providers.MermaidMetadataProvider.md).[MermaidFlowchartDirection](Flowthru.Meta.Providers.MermaidMetadataProvider.MermaidFlowchartDirection.md) + +Flow direction for the diagram + +`activeNodeColor` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Hex color for active (sliced) nodes + +`activeDataColor` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Hex color for active (sliced) catalog entries + +`logger` [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger)? + +Optional logger for diagnostic messages + +## Properties + +### Name + +Gets the unique name of this provider. + +```csharp +public string Name { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +## Methods + +### Consume\(DagMetadata\) + +Consumes DAG metadata. + +```csharp +public void Consume(DagMetadata dag) +``` + +#### Parameters + +`dag` [DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +The DAG metadata to consume + +#### Remarks + +This method is called after pipeline builds. Providers can process +the metadata in any way: write files, send to APIs, store in memory, etc. + +Implementations should handle their own error recovery - exceptions thrown +from this method will be logged but will not fail the pipeline execution. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.md new file mode 100644 index 00000000..958ddda7 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.Providers.md @@ -0,0 +1,24 @@ +# Namespace Flowthru.Meta.Providers + +### Classes + + [JsonMetadataProvider](Flowthru.Meta.Providers.JsonMetadataProvider.md) + +Exports DAG metadata as JSON files. + + [MermaidMetadataProvider](Flowthru.Meta.Providers.MermaidMetadataProvider.md) + +Exports DAG metadata as Mermaid flowchart diagrams. + +### Interfaces + + [IMetadataProvider](Flowthru.Meta.Providers.IMetadataProvider.md) + +Interface for metadata consumers. + +### Enums + + [MermaidMetadataProvider.MermaidFlowchartDirection](Flowthru.Meta.Providers.MermaidMetadataProvider.MermaidFlowchartDirection.md) + +Flow direction for Mermaid flowcharts. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.TimestampConfiguration.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.TimestampConfiguration.md new file mode 100644 index 00000000..e46b4c8e --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.TimestampConfiguration.md @@ -0,0 +1,82 @@ +# Class TimestampConfiguration + +Namespace: [Flowthru.Meta](Flowthru.Meta.md) +Assembly: Flowthru.Core.dll + +Configuration for timestamp handling in metadata file exports. + +```csharp +public class TimestampConfiguration +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[TimestampConfiguration](Flowthru.Meta.TimestampConfiguration.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Controls whether and how timestamps are included in metadata filenames. +This configuration applies to all metadata providers (JSON, Mermaid, etc.) +to ensure consistent filename generation. +

+

+Default behavior: Timestamps are included with format "yyyyMMdd-HHmmss" +

+

+Example filenames: +

+
  • With timestamp: dag-DataProcessing-20251024-143052.json
  • Without timestamp: dag-DataProcessing.json
+

+Warning: When timestamps are disabled, subsequent exports will overwrite +previous files with the same pipeline name. +

+ +## Properties + +### Format + +Gets or sets the timestamp format string. + +```csharp +public string Format { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +Default: "yyyyMMdd-HHmmss" (e.g., "20251024-143052") +Must be a valid DateTime format string compatible with DateTime.ToString(). +Only used when IncludeTimestamp is true. + +### IncludeTimestamp + +Gets or sets whether to include timestamps in metadata filenames. + +```csharp +public bool IncludeTimestamp { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Default: true +When false, files will be named without timestamps and will overwrite on each export. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.md new file mode 100644 index 00000000..33c9bb1a --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Meta.md @@ -0,0 +1,42 @@ +# Namespace Flowthru.Meta + +### Namespaces + + [Flowthru.Meta.Models](Flowthru.Meta.Models.md) + + [Flowthru.Meta.Providers](Flowthru.Meta.Providers.md) + +### Classes + + [FlowthruMetadataBuilder](Flowthru.Meta.FlowthruMetadataBuilder.md) + +Fluent builder for configuring metadata providers and export settings. + + [FlowthruMetadataConfiguration](Flowthru.Meta.FlowthruMetadataConfiguration.md) + +Configuration for Flowthru metadata collection and export. + + [JsonMetadataProviderBuilder](Flowthru.Meta.JsonMetadataProviderBuilder.md) + +Builder for configuring JSON metadata provider options. + + [MermaidMetadataExtensions](Flowthru.Meta.MermaidMetadataExtensions.md) + +Extension methods for generating Mermaid diagram representations of DAG metadata. + + [MermaidMetadataProviderBuilder](Flowthru.Meta.MermaidMetadataProviderBuilder.md) + +Builder for configuring Mermaid diagram provider options. + + [MetadataJsonExtensions](Flowthru.Meta.MetadataJsonExtensions.md) + +Extension methods for serializing metadata to JSON. + + [MetadataProviderBuilderAttribute](Flowthru.Meta.MetadataProviderBuilderAttribute.md) + +Specifies the builder type for a metadata provider. + + [TimestampConfiguration](Flowthru.Meta.TimestampConfiguration.md) + +Configuration for timestamp handling in metadata file exports. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.Factory.NodeFactory.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.Factory.NodeFactory.md new file mode 100644 index 00000000..d6cb369e --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.Factory.NodeFactory.md @@ -0,0 +1,74 @@ +# Class NodeFactory + +Namespace: [Flowthru.Steps.Factory](Flowthru.Steps.Factory.md) +Assembly: Flowthru.Core.dll + +Factory for creating node instances using TypeActivator. + +```csharp +public static class NodeFactory +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[NodeFactory](Flowthru.Steps.Factory.NodeFactory.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Design Pattern: Factory Pattern - provides a centralized location for +node instantiation logic. +

+

+This is a thin wrapper around TypeActivator, providing a domain-specific API for +creating nodes. Could be extended in the future with: +- Node validation logic +- Pre/post-creation hooks +- Node decoration/wrapping +

+ +## Methods + +### Create\(\) + +Creates a new instance of the specified node type. + +```csharp +public static TNode Create() where TNode : new() +``` + +#### Returns + + TNode + +A new node instance + +#### Type Parameters + +`TNode` + +The node type to instantiate + +#### Remarks + +

+Requirements: +- TNode must inherit from NodeBase<TInput, TOutput> +- TNode must have a parameterless constructor +

+

+These requirements are enforced at compile-time via generic constraints in +FlowBuilder.AddStep methods. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.Factory.TypeActivator.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.Factory.TypeActivator.md new file mode 100644 index 00000000..a3aa03b5 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.Factory.TypeActivator.md @@ -0,0 +1,114 @@ +# Class TypeActivator + +Namespace: [Flowthru.Steps.Factory](Flowthru.Steps.Factory.md) +Assembly: Flowthru.Core.dll + +Factory for creating instances of types using compiled expression trees for performance. + +```csharp +public static class TypeActivator +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[TypeActivator](Flowthru.Steps.Factory.TypeActivator.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Design Pattern: Factory Pattern with caching - creates instances of types +using reflection on first call, then caches compiled expression trees for subsequent calls. +

+

+Performance: +- First call uses Expression.Compile() which has overhead +- Subsequent calls use cached delegate which is nearly as fast as `new T()` +- Significantly faster than Activator.CreateInstance<T>() for repeated calls +

+

+Inspiration: ChainSharp uses similar pattern for node instantiation. +

+

+Thread Safety: This class is thread-safe. Multiple threads can safely +call Create<T>() concurrently. +

+ +## Properties + +### CacheCount + +Gets the number of cached factory functions. + +```csharp +public static int CacheCount { get; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +## Methods + +### ClearCache\(\) + +Clears the factory cache. + +```csharp +public static void ClearCache() +``` + +#### Remarks + +Useful for testing or memory management in long-running applications +that dynamically load/unload types. + +### Create\(\) + +Creates an instance of type T using a cached factory. + +```csharp +public static T Create() where T : new() +``` + +#### Returns + + T + +A new instance of type T + +#### Type Parameters + +`T` + +The type to instantiate (must have parameterless constructor) + +#### Remarks + +

+Compile-Time Safety: The `new()` constraint ensures that T has a +parameterless constructor. This is enforced at compile-time by the C# compiler. +

+

+Caching Strategy: +- First call: Compiles an expression tree and caches the resulting delegate +- Subsequent calls: Reuses the cached delegate +- One cache entry per type T +

+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if type T does not have a parameterless constructor + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.Factory.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.Factory.md new file mode 100644 index 00000000..1474665c --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.Factory.md @@ -0,0 +1,12 @@ +# Namespace Flowthru.Steps.Factory + +### Classes + + [NodeFactory](Flowthru.Steps.Factory.NodeFactory.md) + +Factory for creating node instances using TypeActivator. + + [TypeActivator](Flowthru.Steps.Factory.TypeActivator.md) + +Factory for creating instances of types using compiled expression trees for performance. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.NoData.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.NoData.md new file mode 100644 index 00000000..750098f2 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.NoData.md @@ -0,0 +1,182 @@ +# Class NoData + +Namespace: [Flowthru.Steps](Flowthru.Steps.md) +Assembly: Flowthru.Core.dll + +Marker type representing "no meaningful data" for nodes with side-effects or data generation. +Used as input/output type in NodeBase when a node doesn't consume or produce meaningful data. + +```csharp +public sealed class NoData +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[NoData](Flowthru.Steps.NoData.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Design Rationale: NoData provides a type-safe way to represent nodes that: +- Generate data without inputs (e.g., synthetic data generation, seeding) +- Perform side-effects without outputs (e.g., validation, logging, diagnostics) +

+

+This pattern is inspired by functional programming's "Unit" type but uses more intuitive +naming for .NET developers unfamiliar with functional terminology. +

+

+Usage Examples: +

+
// Node with no inputs (data generation)
+public class GenerateDataNode : NodeBase<NoData, OutputSchema>
+{
+    protected override Task<IEnumerable<OutputSchema>> Transform(IEnumerable<NoData> input)
+    {
+        // Generate data from scratch...
+        return Task.FromResult(generatedData);
+    }
+}
+
+// Node with no outputs (side-effects only)
+public class ValidateNode : NodeBase<InputSchema, NoData>
+{
+    protected override Task<IEnumerable<NoData>> Transform(IEnumerable<InputSchema> input)
+    {
+        // Perform validation, logging, etc...
+        return Task.FromResult(Enumerable.Repeat(NoData.Value, 1));
+    }
+}
+

+Pipeline Registration: Use NoData type directly - it automatically converts +to a unique NullCatalogDataset instance: +

+
// Simple syntax with automatic unique key generation
+pipeline.AddStep<ValidationNode>(
+    input: catalog.InputData,
+    output: NoData.Output  // or just: NoData.Discard
+);
+
+pipeline.AddStep<GenerateDataNode>(
+    input: NoData.Input,  // or just: NoData.None
+    output: catalog.GeneratedData
+);
+ +## Fields + +### Value + +Singleton instance of NoData. +Use this value when returning NoData from node transformations. + +```csharp +public static readonly NoData Value +``` + +#### Field Value + + [NoData](Flowthru.Steps.NoData.md) + +## Properties + +### Discard + +Creates a unique null catalog entry for use as a node output (side-effect-only nodes). +Semantic alias for Output - use whichever reads better in context. + +```csharp +public static IItem Discard { get; } +``` + +#### Property Value + + [IItem](Flowthru.Data.IItem\-1.md)<[NoData](Flowthru.Steps.NoData.md)\> + +### Input + +Creates a unique null catalog entry for use as a node input (no-input nodes). +Each call generates a new instance with a unique key to avoid DAG conflicts. + +```csharp +public static IItem Input { get; } +``` + +#### Property Value + + [IItem](Flowthru.Data.IItem\-1.md)<[NoData](Flowthru.Steps.NoData.md)\> + +#### Remarks + +Alias for readability in pipeline declarations where nodes don't consume external inputs. + +### None + +Creates a unique null catalog entry for use as a node input (no-input nodes). +Semantic alias for Input - use whichever reads better in context. + +```csharp +public static IItem None { get; } +``` + +#### Property Value + + [IItem](Flowthru.Data.IItem\-1.md)<[NoData](Flowthru.Steps.NoData.md)\> + +### Output + +Creates a unique null catalog entry for use as a node output (side-effect-only nodes). +Each call generates a new instance with a unique key to avoid DAG conflicts. + +```csharp +public static IItem Output { get; } +``` + +#### Property Value + + [IItem](Flowthru.Data.IItem\-1.md)<[NoData](Flowthru.Steps.NoData.md)\> + +#### Remarks + +Alias for readability in pipeline declarations where nodes produce no meaningful output. + +## Methods + +### Result\(\) + +Returns the standard NoData result for side-effect-only nodes. +Use this at the end of Transform() methods that return NoData. + +```csharp +public static Task> Result() +``` + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)<[NoData](Flowthru.Steps.NoData.md)\>\> + +Singleton collection containing NoData.Value + +#### Remarks + +

+This helper eliminates the verbose Task.FromResult(Enumerable.Repeat(NoData.Value, 1)) +boilerplate. Simply return NoData.Result(). +

+ +
// Instead of:
+return Task.FromResult(Enumerable.Repeat(NoData.Value, 1));
+
+// Use:
+return NoData.Result();
+
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.NoParams.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.NoParams.md new file mode 100644 index 00000000..0c65f916 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.NoParams.md @@ -0,0 +1,49 @@ +# Class NoParams + +Namespace: [Flowthru.Steps](Flowthru.Steps.md) +Assembly: Flowthru.Core.dll + +Marker type for nodes that don't require parameters. +Used as the default TParameters type in NodeBase<TInput, TOutput, TParameters>. + +```csharp +public sealed class NoParams +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[NoParams](Flowthru.Steps.NoParams.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This is a simple empty class that serves as a shorthand for users and the library +when no parameters are needed for a node. Nodes that don't need configuration can +omit the third type parameter by using the two-parameter NodeBase<TInput, TOutput> +convenience base class. +

+

+Usage Examples: +

+
// Explicit NoParams (rarely needed)
+public class MyNode : NodeBase<Input, Output, NoParams> { }
+
+// Recommended: Use two-parameter base class
+public class MyNode : NodeBase<Input, Output> { }
+
+// With parameters
+public class ConfigurableNode : NodeBase<Input, Output, MyParameters>
+{
+    // Parameters property is automatically available
+}
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.md new file mode 100644 index 00000000..78f123b1 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Nodes.md @@ -0,0 +1,18 @@ +# Namespace Flowthru.Steps + +### Namespaces + + [Flowthru.Steps.Factory](Flowthru.Steps.Factory.md) + +### Classes + + [NoData](Flowthru.Steps.NoData.md) + +Marker type representing "no meaningful data" for nodes with side-effects or data generation. +Used as input/output type in NodeBase when a node doesn't consume or produce meaningful data. + + [NoParams](Flowthru.Steps.NoParams.md) + +Marker type for nodes that don't require parameters. +Used as the default TParameters type in NodeBase<TInput, TOutput, TParameters>. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.DryRunOption.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.DryRunOption.md new file mode 100644 index 00000000..49f4e9a6 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.DryRunOption.md @@ -0,0 +1,89 @@ +# Struct DryRunOption + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Represents a dry-run configuration. Can be assigned from a +or a value. + +```csharp +public readonly struct DryRunOption +``` + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Assigning true enables a full dry run (all pre-flight checks, no execution). +Assigning false disables dry-run mode entirely. +Assigning a enables dry-run at the specified depth. + +## Properties + +### Depth + +The validation depth applied when dry-run is enabled. + +```csharp +public ValidationDepth Depth { get; } +``` + +#### Property Value + + [ValidationDepth](Flowthru.Flows.ValidationDepth.md) + +### Enabled + +Whether dry-run mode is enabled. + +```csharp +public bool Enabled { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Operators + +### implicit operator DryRunOption\(bool\) + +Implicitly converts a to a . +true enables full dry-run; false disables it. + +```csharp +public static implicit operator DryRunOption(bool value) +``` + +#### Parameters + +`value` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Returns + + [DryRunOption](Flowthru.Flows.DryRunOption.md) + +### implicit operator DryRunOption\(ValidationDepth\) + +Implicitly converts a to a , +enabling dry-run at the specified depth. + +```csharp +public static implicit operator DryRunOption(ValidationDepth depth) +``` + +#### Parameters + +`depth` [ValidationDepth](Flowthru.Flows.ValidationDepth.md) + +#### Returns + + [DryRunOption](Flowthru.Flows.DryRunOption.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.ExecutionOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.ExecutionOptions.md new file mode 100644 index 00000000..51ad8ce8 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.ExecutionOptions.md @@ -0,0 +1,120 @@ +# Class ExecutionOptions + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Configuration options for pipeline execution. + +```csharp +public class ExecutionOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ExecutionOptions](Flowthru.Flows.ExecutionOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Controls how pipelines are executed and how results are presented. + +## Properties + +### DryRun + +Whether to perform a dry run, and at what validation depth. + +```csharp +public DryRunOption DryRun { get; set; } +``` + +#### Property Value + + [DryRunOption](Flowthru.Flows.DryRunOption.md) + +#### Remarks + +Assign true to perform all pre-flight operations (structure validation, +validation hooks, and external data source inspection) without executing nodes. +Assign a value to control how deeply the pre-flight +checks run — for example, validates +the pipeline graph and runs extension hooks without probing any data sources. +Assign false (default) to run normally without a dry-run stop. + +### EnableParallelExecution + +Whether to enable parallel execution of nodes within the same layer. + +```csharp +public bool EnableParallelExecution { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Phase 2 feature - currently not implemented. +When true, nodes in the same execution layer run concurrently. + +### ResultFormatter + +The result formatter to use for displaying execution results. + +```csharp +public IFlowResultFormatter? ResultFormatter { get; set; } +``` + +#### Property Value + + [IFlowResultFormatter](Flowthru.Results.IFlowResultFormatter.md)? + +#### Remarks + +Defaults to ConsoleResultFormatter if not specified. + +### SliceStrategy + +Optional slicing strategy to apply when executing pipelines. + +```csharp +public FlowSliceStrategy? SliceStrategy { get; set; } +``` + +#### Property Value + + [FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md)? + +#### Remarks + +When provided, only nodes matching the slice strategy will be executed. +Used when slicing flags are provided without a specific pipeline name. + +### StopOnFirstError + +Whether to stop execution on the first node failure. + +```csharp +public bool StopOnFirstError { get; set; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +When true (default), pipeline execution stops immediately when a node fails. +When false, execution continues to independent nodes (Phase 2 feature). + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.NodeResult.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.NodeResult.md new file mode 100644 index 00000000..cf5f79fb --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.NodeResult.md @@ -0,0 +1,161 @@ +# Class NodeResult + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Represents the execution result of a single pipeline node. + +```csharp +public class NodeResult +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[NodeResult](Flowthru.Flows.NodeResult.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### Exception + +Exception that occurred during node execution, if any. + +```csharp +public Exception? Exception { get; init; } +``` + +#### Property Value + + [Exception](https://learn.microsoft.com/dotnet/api/system.exception)? + +#### Remarks + +Null if Success is true. Contains the exception that caused +the node to fail if Success is false. + +### ExecutionTime + +Execution time for this specific node. + +```csharp +public TimeSpan ExecutionTime { get; init; } +``` + +#### Property Value + + [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +### InputCount + +Number of input items processed by this node. + +```csharp +public int InputCount { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +For multi-input nodes, this represents the total count across +all input catalog entries. + +### NodeName + +The name of the node that was executed. + +```csharp +public required string NodeName { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### OutputCount + +Number of output items produced by this node. + +```csharp +public int OutputCount { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +For multi-output nodes, this represents the total count across +all output catalog entries. + +### Success + +Indicates whether the node executed successfully. + +```csharp +public bool Success { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### CreateFailure\(string, TimeSpan, Exception, int\) + +Creates a failed node result. + +```csharp +public static NodeResult CreateFailure(string nodeName, TimeSpan executionTime, Exception exception, int inputCount = 0) +``` + +#### Parameters + +`nodeName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +`executionTime` [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +`exception` [Exception](https://learn.microsoft.com/dotnet/api/system.exception) + +`inputCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Returns + + [NodeResult](Flowthru.Flows.NodeResult.md) + +### CreateSuccess\(string, TimeSpan, int, int\) + +Creates a successful node result. + +```csharp +public static NodeResult CreateSuccess(string nodeName, TimeSpan executionTime, int inputCount, int outputCount) +``` + +#### Parameters + +`nodeName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +`executionTime` [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +`inputCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +`outputCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Returns + + [NodeResult](Flowthru.Flows.NodeResult.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Pipeline.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Pipeline.md new file mode 100644 index 00000000..21c60194 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Pipeline.md @@ -0,0 +1,418 @@ +# Class Pipeline + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Represents a complete data pipeline with nodes, dependencies, and execution order. + +```csharp +public class Pipeline +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Pipeline](Flowthru.Flows.Pipeline.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+A pipeline is a directed acyclic graph (DAG) of transformation nodes. +Each node reads data from catalog entries, performs transformations, +and writes results back to catalog entries. +

+

+Execution Model: +

+
  • Nodes are organized into layers via topological sort
  • Nodes in layer 0 have no dependencies (read external data only)
  • Nodes in layer N depend only on nodes in layers 0..N-1
  • Sequential execution: Execute all nodes in layer order
  • Parallel execution (Phase 2): Execute nodes within same layer concurrently
+

+Single Producer Rule: Each catalog entry can be written by at most +one node. This ensures deterministic execution order and prevents race conditions. +

+ +## Properties + +### Description + +Optional description of what this pipeline does. + +```csharp +public string? Description { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### IsBuilt + +Indicates whether the pipeline has been built (dependencies analyzed and layers assigned). + +```csharp +public bool IsBuilt { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### Logger + +Optional logger for pipeline execution. + +```csharp +public ILogger? Logger { get; set; } +``` + +#### Property Value + + [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger)? + +### Name + +Pipeline name for identification and logging. + +```csharp +public string? Name { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +Set by PipelineRegistry during pipeline registration. + +### Nodes + +All nodes in this pipeline, in the order they were added. + +```csharp +public IReadOnlyList Nodes { get; } +``` + +#### Property Value + + [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[PipelineNode](Flowthru.Flows.PipelineNode.md)\> + +#### Remarks + +Exposed as public to enable validation hooks (Phase 4) to inspect nodes. +The collection is read-only - nodes can only be added via FlowBuilder. + +### ServiceProvider + +Optional service provider for dependency injection into nodes. + +```csharp +public IServiceProvider? ServiceProvider { get; set; } +``` + +#### Property Value + + [IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider)? + +#### Remarks + +Set by the service layer before pipeline execution to enable nodes +to resolve services (e.g., database connections, external APIs). + +### ValidationHooks + +Validation hooks that run during pre-flight checks. + +```csharp +public List ValidationHooks { get; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[IPipelineValidationHook](Flowthru.Flows.Validation.IPipelineValidationHook.md)\> + +#### Remarks + +

+Extensions can register hooks to validate their own node types during pre-flight. +Hooks are invoked after DAG analysis but before external input inspection. +

+

+Hook execution order: +

+
  1. Pipeline.Build() - DAG construction and layer assignment
  2. ValidationHooks.ValidateAsync() - Extension-specific validation
  3. Pipeline.ValidateExternalInputsAsync() - External input inspection
+

+Example (Python extension): +

+
pipeline.ValidationHooks.Add(new PythonNodeValidator(executor, runtime));
+ +### ValidationOptions + +Validation options for this pipeline. + +```csharp +public ValidationOptions ValidationOptions { get; } +``` + +#### Property Value + + [ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md) + +#### Remarks + +Configures how external data sources (Layer 0 inputs) are validated +before pipeline execution begins. + +## Methods + +### Build\(FlowSliceStrategy?\) + +Builds the pipeline by analyzing dependencies and assigning execution layers. +Must be called before executing the pipeline. + +```csharp +public void Build(FlowSliceStrategy? sliceStrategy = null) +``` + +#### Parameters + +`sliceStrategy` [FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md)? + +Optional slicing strategy to filter nodes before execution + +#### Remarks + +

+Slicing: If a slicing strategy is provided, only nodes matching +the strategy will be included in the execution. The slice always forms a valid +sub-DAG with all required dependencies. +

+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if: +- Multiple nodes write to the same catalog entry (single producer rule) +- Circular dependency is detected +- Slice strategy references non-existent nodes or catalog entries + +### ExecuteAsync\(CancellationToken\) + +Executes the pipeline sequentially, layer by layer. + +```csharp +public Task ExecuteAsync(CancellationToken cancellationToken) +``` + +#### Parameters + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token to signal graceful shutdown + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task) + +Task representing the pipeline execution + +#### Remarks + +

+This method executes nodes in topological order: +1. Execute all nodes in layer 0 sequentially +2. Execute all nodes in layer 1 sequentially +3. Continue until all layers are complete +

+

+Note: This method throws exceptions on failure. For result-based +execution with error handling, use RunAsync() instead. +

+

+In Phase 2, this will be replaced with a parallel executor that can run +nodes within the same layer concurrently. +

+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if pipeline has not been built + +### ExportDag\(\) + +Exports DAG metadata for this pipeline. + +```csharp +public DagMetadata ExportDag() +``` + +#### Returns + + [DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +Complete DAG metadata including nodes, catalog entries, and edges + +#### Remarks + +

+This method extracts structural metadata from the built pipeline, creating +a complete representation of the DAG (Directed Acyclic Graph) that can be +serialized to JSON for visualization in Flowthru.Viz. +

+

+Prerequisites: Pipeline must be built before calling this method. +Call Build() first if IsBuilt is false. +

+

+Usage: +

+
var pipeline = DataProcessingPipeline.Create(catalog);
+pipeline.Build();
+
+var dag = pipeline.ExportDag();
+var json = dag.ToJson();
+File.WriteAllText("dag.json", json);
+

+This method is non-destructive and idempotent - it can be called multiple +times without affecting the pipeline state. +

+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if pipeline has not been built + +### Merge\(Dictionary\) + +Merges multiple pipelines into a single pipeline by combining all their nodes. + +```csharp +public static Pipeline Merge(Dictionary pipelines) +``` + +#### Parameters + +`pipelines` [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [Pipeline](Flowthru.Flows.Pipeline.md)\> + +Dictionary of pipeline names to pipeline instances + +#### Returns + + [Pipeline](Flowthru.Flows.Pipeline.md) + +A new pipeline containing all nodes from all input pipelines + +#### Remarks + +

+This method creates a new pipeline by combining all nodes from the input pipelines. +Node names are prefixed with their source pipeline name (e.g., "data_processing.PreprocessCompanies") +to ensure uniqueness and maintain traceability in logs. +

+

+The existing DependencyAnalyzer will automatically resolve cross-pipeline dependencies +based on catalog entries. The single producer rule is enforced - if multiple pipelines +attempt to write to the same catalog entry, Build() will throw an InvalidOperationException. +

+ +### RunAsync\(CancellationToken\) + +/// Builds and executes the pipeline, returning comprehensive execution results. + +```csharp +public Task RunAsync(CancellationToken cancellationToken) +``` + +#### Parameters + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token to signal graceful shutdown + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[FlowResult](Flowthru.Flows.FlowResult.md)\> + +FlowResult containing execution status, timing, and node results + +#### Remarks + +

+This is the primary high-level API for executing pipelines. It automatically +calls Build() if the pipeline hasn't been built yet, then executes and tracks results. +

+ +### ValidateExternalInputsAsync\(CancellationToken\) + +Validates all external inputs before pipeline execution. + +```csharp +public Task ValidateExternalInputsAsync(CancellationToken cancellationToken = default) +``` + +#### Parameters + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token for validation I/O operations + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +ValidationResult containing any errors found + +#### Remarks + +

+This method inspects catalog entries that are consumed by the pipeline but not +produced by any node in the execution set. These are pre-existing external data +sources (files, databases, APIs) that must exist and be valid before the pipeline +can execute. +

+

+Slicing Support: In sliced pipelines, catalog entries that were +produced by nodes outside the slice are correctly identified as external inputs +and validated. This prevents runtime failures from missing intermediate data. +

+

+Inspection Levels: +

+
  • None: Skip inspection entirely
  • Shallow: Validate file exists, check headers/schema, deserialize sample rows
  • Deep: Validate all rows in the dataset (expensive!)
+

+Default Behavior: +

+
  • If explicitly configured via WithValidation() → use that level
  • If entry has PreferredInspectionLevel set → use that level
  • Otherwise → Shallow (all storage adapters support inspection)
+

+Important: Only external inputs are inspected. Intermediate pipeline +outputs produced within the execution set are never inspected, as they don't exist yet. +

+

+Usage: +

+
pipeline.Build();
+var validationResult = await pipeline.ValidateExternalInputsAsync();
+if (!validationResult.IsValid) {
+  // Handle validation errors before execution
+  validationResult.ThrowIfInvalid();
+}
+await pipeline.RunAsync();
+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if pipeline has not been built + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineBuilder.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineBuilder.md new file mode 100644 index 00000000..c8b7a13d --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineBuilder.md @@ -0,0 +1,9677 @@ +# Class FlowBuilder + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Fluent builder for constructing type-safe data pipelines with function-based nodes. + +```csharp +public class FlowBuilder +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Function-Based Design (v0.5.0): +Nodes are pure transformation functions with compile-time type safety. +Both synchronous and asynchronous functions are supported: +- Sync: Func<TInput, TOutput> +- Async: Func<TInput, Task<TOutput>> +- Multi-input: Func<(TIn1, TIn2, ...), TOutput> or Task<TOutput> +- Multi-output: Func<TInput, (TOut1, TOut2, ...)> or Task<(TOut1, TOut2, ...)> +

+

+Use synchronous functions for pure data transformations. Use asynchronous functions +only when your node performs I/O operations (external APIs, databases, etc.). +

+

+The compiler infers all types from function signatures and validates catalog entry +types at pipeline construction time, catching type mismatches before execution. +

+

+Usage Patterns: +

+
var pipeline = FlowBuilder.CreateFlow(builder =>
+{
+    // Simple synchronous node
+    builder.AddStep(
+        name: "Preprocess",
+        transform: PreprocessNode.Create(),
+        input: catalog.RawData,
+        output: catalog.ProcessedData
+    );
+
+    // Multi-input node: tuple → single output
+    builder.AddStep(
+        name: "TrainModel",
+        transform: TrainModelNode.Create(),
+        input: (catalog.XTrain, catalog.YTrain),
+        output: catalog.Model
+    );
+
+    // Multi-output node: single input → tuple
+    builder.AddStep(
+        name: "SplitData",
+        transform: SplitDataNode.Create(),
+        input: catalog.Data,
+        output: (catalog.XTrain, catalog.XTest, catalog.YTrain, catalog.YTest)
+    );
+
+    // Asynchronous node (only when needed for I/O)
+    builder.AddStep(
+        name: "FetchExternalData",
+        transform: ExternalDataNode.Create(),
+        input: catalog.Config,
+        output: catalog.ExternalData
+    );
+});
+
+pipeline.Build();
+await pipeline.ExecuteAsync();
+ +## Methods + +### AddStep\(string, Func\>, IItem, IItem, string\) + +Adds a node with single input and single output (asynchronous transformation). +All types are inferred from the transformation function signature. + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function from input to output + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TInput` + +Input type (inferred from transform) + +`TOutput` + +Output type (inferred from transform) + +### AddStep\(string, Func\>, IItem, IItem, string\) + +Adds a node with single input and single output (asynchronous transformation with cancellation support). +All types are inferred from the transformation function signature. + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-3)\> + +Asynchronous transformation function from input to output with cancellation token + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TInput` + +Input type (inferred from transform) + +`TOutput` + +Output type (inferred from transform) + +### AddStep\(string, Func, IItem, IItem, string\) + +Adds a node with single input and single output (synchronous transformation). +All types are inferred from the transformation function signature. + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function from input to output + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TInput` + +Input type (inferred from transform) + +`TOutput` + +Output type (inferred from transform) + +### AddStep\(string, IReadOnlyList\>, IItem, Func, TOut\>, string\) + +Adds a homogeneous fan-in node: N catalog entries of the same element type collapse +into a single node whose transform receives all N loaded collections as a typed list. + +```csharp +public FlowBuilder AddStep(string label, IReadOnlyList> inputs, IItem output, Func, TOut> node, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`inputs` [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[IItem](Flowthru.Data.IItem\-1.md)\> + +Variable-length list of same-typed input entries + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry to store the merged result + +`node` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1), TOut\> + +Transform function receiving all N loaded values as a typed read-only list + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional human-readable description + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn` + +Element type of each input catalog entry + +`TOut` + +Output type produced by the transform + +#### Remarks + +Use this when the number of inputs is not known at compile time — for example, +aggregating per-partition catalogs constructed in a loop. The function receives +IReadOnlyList<TIn> where each element corresponds to one input entry +in declaration order. An empty inputs list is allowed but produces an empty list argument. + +### AddStep\(string, Func\>, IItem, \(IItem, IItem\), string\) + +Adds a node with 1 input and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func, IItem, \(IItem, IItem\), string\) + +Adds a node with 1 input and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func\>, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func> transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)\> + +Asynchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 1 input and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func transform, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Synchronous transformation function + +`input` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task\>, \(IItem, IItem\), IItem, string\) + +Adds a node with 2 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task> transform, (IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2\), TOut1\>, \(IItem, IItem\), IItem, string\) + +Adds a node with 2 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), TOut1> transform, (IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 2 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2)>> transform, (IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2\)\>, \(IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 2 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2)> transform, (IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3)> transform, (IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 2 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task\>, \(IItem, IItem, IItem\), IItem, string\) + +Adds a node with 3 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task> transform, (IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), TOut1\>, \(IItem, IItem, IItem\), IItem, string\) + +Adds a node with 3 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), TOut1> transform, (IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 3 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 3 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2)> transform, (IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 3 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task\>, \(IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a node with 4 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task> transform, (IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), TOut1\>, \(IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a node with 4 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), TOut1> transform, (IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 4 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 4 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 4 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task\>, \(IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a node with 5 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task> transform, (IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), TOut1\>, \(IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a node with 5 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), TOut1> transform, (IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 5 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 5 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 5 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a node with 6 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), TOut1\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a node with 6 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), TOut1> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 6 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 6 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 6 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a node with 7 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), TOut1\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a node with 7 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), TOut1> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 7 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 7 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 7 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a node with 8 inputs and 1 output (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), TOut1\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, string\) + +Adds a node with 8 inputs and 1 output (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), TOut1> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), TOut1\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` [IItem](Flowthru.Data.IItem\-1.md) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 8 inputs and 2 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), string\) + +Adds a node with 8 inputs and 2 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 3 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 3 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3, TOut4\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 4 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3, TOut4)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 4 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3, TOut4)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 5 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3, TOut4, TOut5)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 5 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3, TOut4, TOut5)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 6 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 6 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 7 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 7 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), Task<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 8 outputs (asynchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), Task<(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)>> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<\(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>\> + +Asynchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddStep\(string, Func<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\>, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), string\) + +Adds a node with 8 inputs and 8 outputs (synchronous transformation). + +```csharp +public FlowBuilder AddStep(string label, Func<(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8), (TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8)> transform, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, string description = "") +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`transform` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<\(TIn1, TIn2, TIn3, TIn4, TIn5, TIn6, TIn7, TIn8\), \(TOut1, TOut2, TOut3, TOut4, TOut5, TOut6, TOut7, TOut8\)\> + +Synchronous transformation function + +`input` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \([IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md), [IItem](Flowthru.Data.IItem\-1.md)\) + +Catalog entry or tuple of catalog entries to store output data + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional description of the node's purpose + +#### Returns + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### CreateFlow\(Action\) + +Creates and configures a new pipeline using the builder pattern. + +```csharp +public static Pipeline CreateFlow(Action configure) +``` + +#### Parameters + +`configure` [Action](https://learn.microsoft.com/dotnet/api/system.action\-1)<[FlowBuilder](Flowthru.Flows.FlowBuilder.md)\> + +Action to configure the pipeline by adding nodes + +#### Returns + + [Pipeline](Flowthru.Flows.Pipeline.md) + +Configured but not yet built pipeline + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineNode.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineNode.md new file mode 100644 index 00000000..30c419fd --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineNode.md @@ -0,0 +1,223 @@ +# Class PipelineNode + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Represents a node within a pipeline, wrapping the transformation function with metadata +about its inputs, outputs, and dependencies. + +```csharp +public class PipelineNode +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PipelineNode](Flowthru.Flows.PipelineNode.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+PipelineNode serves as the internal representation of a node during pipeline +construction and execution. It tracks: +- The transformation function (Func<TInput, Task<TOutput>>) +- Input catalog entries (what data it reads) +- Output catalog entries (what data it writes) +- Dependencies (other nodes that must run first) +

+

+Single Producer Rule: Each catalog entry can be written by at most +one node in a pipeline. This constraint ensures deterministic dependency resolution +and enables simple DAG construction via topological sort. +

+

+Function-Based Architecture (v0.5.0): Nodes are now pure transformation +functions instead of class instances. This enables compile-time type safety through +generic type inference at the pipeline construction site. +

+

+Visibility (Phase 4): +Made public to enable validation hooks to inspect node properties. +This is necessary for extensions (e.g., Python) to validate their own node types. +

+ +## Constructors + +### PipelineNode\(string, string?, Delegate, IReadOnlyList, IReadOnlyList\) + +Creates a new pipeline node with a transformation function. + +```csharp +public PipelineNode(string label, string? description, Delegate node, IReadOnlyList inputs, IReadOnlyList outputs) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +`node` [Delegate](https://learn.microsoft.com/dotnet/api/system.delegate) + +The transformation function (Func<TInput, Task<TOutput>>) + +`inputs` [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[IItem](Flowthru.Data.IItem.md)\> + +Catalog entries this node reads + +`outputs` [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[IItem](Flowthru.Data.IItem.md)\> + +Catalog entries this node writes + +## Properties + +### Dependencies + +Other pipeline nodes that must execute before this node. +Populated during dependency analysis by checking which nodes produce our inputs. + +```csharp +public List Dependencies { get; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[PipelineNode](Flowthru.Flows.PipelineNode.md)\> + +#### Remarks + +This forms the edges of the execution DAG: +- If node A produces output X, and node B consumes input X, then B depends on A. +- Topological sort uses these dependencies to determine execution order. + +### Description + +String description of the node's purpose. + +```csharp +public string Description { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Inputs + +Catalog entries that this node reads as input. +These may be produced by other nodes (dependencies) or be external prerequisites. + +```csharp +public IReadOnlyList Inputs { get; } +``` + +#### Property Value + + [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[IItem](Flowthru.Data.IItem.md)\> + +### Label + +Unique identifier for this node within the pipeline. +Typically the node type name or user-provided name. + +```csharp +public string Label { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Layer + +Execution layer determined by topological sort. +Nodes in layer 0 have no dependencies. Nodes in layer N depend on nodes in layers 0..N-1. + +```csharp +public int Layer { get; set; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### Outputs + +Catalog entries that this node writes as output. +Per the single producer rule, each entry here must be unique across all nodes. + +```csharp +public IReadOnlyList Outputs { get; } +``` + +#### Property Value + + [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[IItem](Flowthru.Data.IItem.md)\> + +### TransformFunction + +The transformation function that performs the node's work. +Type-erased to Delegate since we need to store different function signatures together. + +```csharp +public Delegate TransformFunction { get; } +``` + +#### Property Value + + [Delegate](https://learn.microsoft.com/dotnet/api/system.delegate) + +#### Remarks + +

+At execution time, this delegate will be invoked via DynamicInvoke with the +appropriate input parameter(s). The function signature can be either synchronous +or asynchronous: +- Sync single: Func<TInput, TOutput> +- Async single: Func<TInput, Task<TOutput>> +- Sync multi-input: Func<(TIn1, TIn2, ...), TOutput> +- Async multi-input: Func<(TIn1, TIn2, ...), Task<TOutput>> +- Sync multi-output: Func<TInput, (TOut1, TOut2, ...)> +- Async multi-output: Func<TInput, Task<(TOut1, TOut2, ...)>> +

+

+Optional Cancellation Support: Nodes can opt-in to cancellation awareness +by accepting a CancellationToken as the last parameter: +- Func<TInput, CancellationToken, Task<TOutput>> +- Func<(TIn1, TIn2), CancellationToken, Task<TOutput>> +

+

+When a node accepts a CancellationToken, the pipeline will pass the runtime token during +execution, allowing the node to cancel long-running operations cooperatively. Nodes that +do not accept a CancellationToken will only be cancelled between node executions. +

+

+The execution engine detects whether the result is a Task and awaits it if needed. +

+ +## Methods + +### ToString\(\) + +Returns a string representation for debugging. + +```csharp +public override string ToString() +``` + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineResult.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineResult.md new file mode 100644 index 00000000..aa107d5e --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineResult.md @@ -0,0 +1,214 @@ +# Class FlowResult + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Represents the result of a pipeline execution. + +```csharp +public class FlowResult +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowResult](Flowthru.Flows.FlowResult.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This class provides comprehensive execution information including success status, +timing, individual node results, and error details. +

+

+Usage Pattern: +

+
var result = await pipeline.RunAsync();
+
+if (result.Success)
+{
+    Console.WriteLine($"Pipeline completed in {result.ExecutionTime.TotalSeconds:F2}s");
+    foreach (var nodeResult in result.StepResults.Values)
+    {
+        Console.WriteLine($"  {nodeResult.NodeName}: {nodeResult.ExecutionTime.TotalSeconds:F2}s");
+    }
+}
+else
+{
+    Console.WriteLine($"Pipeline failed: {result.Exception?.Message}");
+}
+ +## Properties + +### Exception + +Exception that caused pipeline failure, if any. + +```csharp +public Exception? Exception { get; init; } +``` + +#### Property Value + + [Exception](https://learn.microsoft.com/dotnet/api/system.exception)? + +#### Remarks + +Null if Success is true. Contains the first exception that caused +pipeline execution to halt if Success is false. + +### ExecutionTime + +Total execution time for the entire pipeline. + +```csharp +public TimeSpan ExecutionTime { get; init; } +``` + +#### Property Value + + [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +### IsDryRun + +Indicates whether this was a dry run (pre-flight checks only). + +```csharp +public bool IsDryRun { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### StepResults + +Results for individual nodes, keyed by node name. + +```csharp +public Dictionary StepResults { get; init; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [NodeResult](Flowthru.Flows.NodeResult.md)\> + +#### Remarks + +Dictionary keys are the node names as specified in the pipeline definition. +Values contain execution details for each node. + +### FlowName + +The name of the pipeline that was executed. + +```csharp +public string? FlowName { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### Success + +Indicates whether the pipeline executed successfully. + +```csharp +public bool Success { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### CreateDryRunSuccess\(TimeSpan, int, int, int, string?\) + +Creates a successful dry run result. + +```csharp +public static FlowResult CreateDryRunSuccess(TimeSpan preFlightDuration, int nodeCount, int layerCount, int validatedInputCount, string? pipelineName = null) +``` + +#### Parameters + +`preFlightDuration` [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +Time spent on pre-flight checks + +`nodeCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Total number of nodes in the pipeline + +`layerCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of execution layers + +`validatedInputCount` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of external inputs validated + +`pipelineName` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Name of the pipeline + +#### Returns + + [FlowResult](Flowthru.Flows.FlowResult.md) + +A successful dry run result + +### CreateFailure\(TimeSpan, Exception, Dictionary?, string?\) + +Creates a failed pipeline result. + +```csharp +public static FlowResult CreateFailure(TimeSpan executionTime, Exception exception, Dictionary? nodeResults = null, string? pipelineName = null) +``` + +#### Parameters + +`executionTime` [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +`exception` [Exception](https://learn.microsoft.com/dotnet/api/system.exception) + +`nodeResults` [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [NodeResult](Flowthru.Flows.NodeResult.md)\>? + +`pipelineName` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Returns + + [FlowResult](Flowthru.Flows.FlowResult.md) + +### CreateSuccess\(TimeSpan, Dictionary, string?\) + +Creates a successful pipeline result. + +```csharp +public static FlowResult CreateSuccess(TimeSpan executionTime, Dictionary nodeResults, string? pipelineName = null) +``` + +#### Parameters + +`executionTime` [TimeSpan](https://learn.microsoft.com/dotnet/api/system.timespan) + +`nodeResults` [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [NodeResult](Flowthru.Flows.NodeResult.md)\> + +`pipelineName` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Returns + + [FlowResult](Flowthru.Flows.FlowResult.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineSliceStrategy.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineSliceStrategy.md new file mode 100644 index 00000000..d3bf1fa0 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.PipelineSliceStrategy.md @@ -0,0 +1,179 @@ +# Class FlowSliceStrategy + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Defines a strategy for slicing a pipeline to execute a subset of nodes. + +```csharp +public sealed class FlowSliceStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Pipeline slicing allows executing only specific portions of a DAG while maintaining +execution validity. All slicing operations preserve the runnability guarantee: +the resulting sub-DAG must be executable without missing dependencies. +

+

+Slicing Strategies: +

+
  • Pipelines: Filter to nodes from specific named pipelines (in merged DAGs)
  • FromNodes: Include specified nodes and all downstream dependents
  • ToNodes: Include specified nodes and all upstream dependencies (run "up to" these nodes)
  • FromData: Include nodes consuming specified catalog entries and all downstream dependents
  • ToData: Include nodes producing specified catalog entries and all upstream dependencies
  • OnlyNodes: Explicit allowlist of nodes plus minimal required dependencies
+

+Composition: Multiple strategies compose via intersection (additive filtering). +For example, --from-nodes A --to-data B produces nodes in the downstream dependency +tree of A that are also required to produce data B. +

+

+Runnability Guarantee: Slicing operations are additive only. Subtractive +operations (--from-nodes A --except B) would break the runnability guarantee and +are not supported. +

+ +## Properties + +### FromData + +Start from nodes that consume these catalog entry labels, including all downstream dependents. + +```csharp +public IReadOnlySet? FromData { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +Finds all nodes that read the specified catalog entries, then expands downstream. +Useful for impact analysis - "what breaks if I change this data?" + +### FromNodes + +Start from these nodes, including all downstream dependents. + +```csharp +public IReadOnlySet? FromNodes { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +Expands to include all nodes that depend on these nodes (transitively). +Useful for impact analysis - "what breaks if I change this node?" + +### IsSliced + +Whether any slicing is configured. + +```csharp +public bool IsSliced { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### OnlyNodes + +Explicit allowlist of node names (dependencies auto-included). + +```csharp +public IReadOnlySet? OnlyNodes { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +Specifies exactly which nodes to execute, then automatically includes any +required dependencies to maintain DAG validity. + +### Pipelines + +Filter to nodes from these named pipelines (applies to merged pipelines). + +```csharp +public IReadOnlySet? Pipelines { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +In merged pipelines, nodes are prefixed with their pipeline name (e.g., "DataScience.TrainModel"). +This filter includes only nodes from the specified pipelines. +Pipeline names are case-insensitive. + +### ToData + +End at nodes that produce these catalog entry labels, including all upstream dependencies. + +```csharp +public IReadOnlySet? ToData { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +Finds the nodes that write the specified catalog entries, then expands upstream. +Useful for targeted execution - "run everything needed to produce this data". + +### ToNodes + +End at these nodes, including all upstream dependencies needed to produce them. + +```csharp +public IReadOnlySet? ToNodes { get; init; } +``` + +#### Property Value + + [IReadOnlySet](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlyset\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>? + +#### Remarks + +Expands to include all transitive dependencies needed to run these nodes. +Equivalent to "run everything up to and including these nodes". +Useful for testing specific outputs without running the entire pipeline. + +## Methods + +### All\(\) + +No filtering - execute entire pipeline. + +```csharp +public static FlowSliceStrategy All() +``` + +#### Returns + + [FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Validation.IPipelineValidationHook.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Validation.IPipelineValidationHook.md new file mode 100644 index 00000000..ec1b114d --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Validation.IPipelineValidationHook.md @@ -0,0 +1,86 @@ +# Interface IPipelineValidationHook + +Namespace: [Flowthru.Flows.Validation](Flowthru.Flows.Validation.md) +Assembly: Flowthru.Core.dll + +Validation hook that runs during pipeline pre-flight checks. + +```csharp +public interface IPipelineValidationHook +``` + +## Remarks + +

+Validation hooks provide an extensibility point for extensions to contribute +their own validation logic during the pre-flight phase. Hooks are invoked after +DAG analysis but before external input inspection, allowing extensions to +validate their own node types. +

+

+Example use cases: +

  • Python extension validates @node decorators match C# types
  • Custom extensions validate node-specific configuration
  • Third-party plugins validate external dependencies
+

+

+Hook execution order: +

+
  1. Pipeline.Build() - DAG construction and layer assignment
  2. ValidationHooks.ValidateAsync() - Extension-specific validation
  3. Pipeline.ValidateExternalInputsAsync() - External input inspection
+

+Error handling: +Hooks should return ValidationResult with errors, never throw exceptions. +Multiple hooks may run, and all errors are aggregated into a single result. +

+ +## Methods + +### ValidateAsync\(Pipeline, CancellationToken\) + +Validates pipeline nodes during pre-flight checks. + +```csharp +Task ValidateAsync(Pipeline pipeline, CancellationToken cancellationToken) +``` + +#### Parameters + +`pipeline` [Pipeline](Flowthru.Flows.Pipeline.md) + +The pipeline being validated + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token for async operations + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Validation result containing any errors found + +#### Remarks + +

+Implementations should: +

  • Never throw exceptions (return errors in ValidationResult)
  • Be idempotent (safe to call multiple times)
  • Be reasonably fast (executed during pre-flight, blocks pipeline start)
  • Only validate nodes they understand (ignore other node types)
+

+

+Example implementation (Python extension): +

+
public async Task<ValidationResult> ValidateAsync(
+  Pipeline pipeline,
+  CancellationToken cancellationToken)
+{
+  var result = ValidationResult.Success();
+
+  foreach (var node in pipeline.Nodes)
+  {
+    if (IsPythonNode(node))
+    {
+      var nodeResult = await ValidatePythonNode(node, cancellationToken);
+      result.Merge(nodeResult);
+    }
+  }
+
+  return result;
+}
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Validation.ValidationOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Validation.ValidationOptions.md new file mode 100644 index 00000000..8f42ef0c --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Validation.ValidationOptions.md @@ -0,0 +1,106 @@ +# Class ValidationOptions + +Namespace: [Flowthru.Flows.Validation](Flowthru.Flows.Validation.md) +Assembly: Flowthru.Core.dll + +Configuration for pipeline validation behavior. + +```csharp +public class ValidationOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+ValidationOptions provides pipeline-level overrides for validation configuration. +The primary mechanism for validation configuration is catalog-level via the + property and the fluent +.WithInspectionLevel() API. +

+

+Default Behavior (if not configured): +

+
  • Catalog entry has PreferredInspectionLevel set → use that level
  • Layer 0 inputs → Shallow inspection (all storage adapters support inspection)
  • All intermediate outputs (Layer 1+) → None (never inspected)
+

+Catalog-Level Configuration (Recommended): +

+
public ICatalogDataset<Company> Companies =>
+  GetOrCreateDataset(() => new CsvCatalogDataset<Company>("companies", "data/companies.csv")
+    .WithInspectionLevel(InspectionLevel.Deep));
+

+Pipeline-Level Override (Advanced): +

+
builder
+  .RegisterPipeline<MyCatalog>("data_processing", MyPipeline.Create)
+  .WithValidation(validation => {
+    // Override catalog-level setting for this specific pipeline
+    validation.Inspect(catalog.Companies, InspectionLevel.Shallow); // Temporarily use shallow
+  });
+

+Design Rationale: +

+

+Validation configuration is primarily a property of the data source itself, not the pipeline +consuming it. Critical external datasets should always be deeply validated, regardless of +which pipeline uses them. Pipeline-level overrides exist for rare cases where different +validation is needed temporarily (e.g., performance testing, debugging). +

+ +## Methods + +### Default\(\) + +Creates a new ValidationOptions instance with default settings. + +```csharp +public static ValidationOptions Default() +``` + +#### Returns + + [ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md) + +### Inspect\(IItem, InspectionLevel\) + +Specifies the inspection level for a specific catalog entry. + +```csharp +public ValidationOptions Inspect(IItem catalogEntry, InspectionLevel level) +``` + +#### Parameters + +`catalogEntry` [IItem](Flowthru.Data.IItem.md) + +The catalog entry to configure + +`level` [InspectionLevel](Flowthru.Data.Validation.InspectionLevel.md) + +The inspection level to use for this entry + +#### Returns + + [ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md) + +This ValidationOptions instance for fluent chaining + +#### Remarks + +This configuration only applies to Layer 0 inputs (external data). +Intermediate outputs are never inspected regardless of this setting. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Validation.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Validation.md new file mode 100644 index 00000000..e05e83d2 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.Validation.md @@ -0,0 +1,14 @@ +# Namespace Flowthru.Flows.Validation + +### Classes + + [ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md) + +Configuration for pipeline validation behavior. + +### Interfaces + + [IPipelineValidationHook](Flowthru.Flows.Validation.IPipelineValidationHook.md) + +Validation hook that runs during pipeline pre-flight checks. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.ValidationDepth.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.ValidationDepth.md new file mode 100644 index 00000000..d00acbf5 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.ValidationDepth.md @@ -0,0 +1,27 @@ +# Enum ValidationDepth + +Namespace: [Flowthru.Flows](Flowthru.Flows.md) +Assembly: Flowthru.Core.dll + +Controls how deeply a dry run validates the pipeline before stopping. + +```csharp +public enum ValidationDepth +``` + +## Fields + +`Full = 1` + +Structure validation plus external data presence checks (default dry-run behaviour). + + + +`StructureOnly = 0` + +Validates graph structure only: no cycles, all node type contracts satisfied, +all catalog entry dependencies wired, and all validation hooks run. +No data source access. + + + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.md new file mode 100644 index 00000000..e7df9cdc --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Pipelines.md @@ -0,0 +1,50 @@ +# Namespace Flowthru.Flows + +### Namespaces + + [Flowthru.Flows.Validation](Flowthru.Flows.Validation.md) + +### Classes + + [ExecutionOptions](Flowthru.Flows.ExecutionOptions.md) + +Configuration options for pipeline execution. + + [NodeResult](Flowthru.Flows.NodeResult.md) + +Represents the execution result of a single pipeline node. + + [Pipeline](Flowthru.Flows.Pipeline.md) + +Represents a complete data pipeline with nodes, dependencies, and execution order. + + [FlowBuilder](Flowthru.Flows.FlowBuilder.md) + +Fluent builder for constructing type-safe data pipelines with function-based nodes. + + [PipelineNode](Flowthru.Flows.PipelineNode.md) + +Represents a node within a pipeline, wrapping the transformation function with metadata +about its inputs, outputs, and dependencies. + + [FlowResult](Flowthru.Flows.FlowResult.md) + +Represents the result of a pipeline execution. + + [FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md) + +Defines a strategy for slicing a pipeline to execute a subset of nodes. + +### Structs + + [DryRunOption](Flowthru.Flows.DryRunOption.md) + +Represents a dry-run configuration. Can be assigned from a +or a value. + +### Enums + + [ValidationDepth](Flowthru.Flows.ValidationDepth.md) + +Controls how deeply a dry run validates the pipeline before stopping. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Registry.IFlowRegistrar-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Registry.IFlowRegistrar-1.md new file mode 100644 index 00000000..9a2dad17 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Registry.IFlowRegistrar-1.md @@ -0,0 +1,151 @@ +# Interface IFlowRegistrar + +Namespace: [Flowthru.Registry](Flowthru.Registry.md) +Assembly: Flowthru.Core.dll + +Fluent interface for registering flows in a type-safe manner. + +```csharp +public interface IFlowRegistrar where TCatalog : CatalogAbstract +``` + +#### Type Parameters + +`TCatalog` + +The catalog type that flows will use + +## Remarks + +

+This interface provides compile-time type safety by tying flow factories +to a specific catalog type. The registrar validates that all registered +flows accept the correct catalog type. +

+

+Usage: +

protected override void RegisterFlows(IFlowRegistrar<MyCatalog> registrar)
+{
+    // Flow without parameters
+    registrar.Register("processing", ProcessingFlow.Create);
+
+    // Flow with parameters
+    registrar.Register("training", TrainFlow.Create, new TrainOptions());
+
+    // Add metadata
+    registrar.WithDescription("processing", "Cleans and transforms raw data");
+}
+

+ +## Methods + +### Register\(string, Func\) + +Registers a flow with a parameterless factory function. + +```csharp +IFlowRegistrar Register(string name, Func flowFactory) +``` + +#### Parameters + +`name` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique flow name + +`flowFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Factory function that creates the flow from catalog + +#### Returns + + [IFlowRegistrar](Flowthru.Registry.IFlowRegistrar\-1.md) + +This registrar for method chaining + +#### Remarks + +Use this overload when the flow doesn't require parameters. + +### Register\(string, Func, TParams\) + +Registers a flow with a parameterized factory function. + +```csharp +IFlowRegistrar Register(string name, Func flowFactory, TParams parameters) +``` + +#### Parameters + +`name` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique flow name + +`flowFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-3) + +Factory function that creates the flow from catalog and parameters + +`parameters` TParams + +Parameter instance to pass to the flow + +#### Returns + + [IFlowRegistrar](Flowthru.Registry.IFlowRegistrar\-1.md) + +This registrar for method chaining + +#### Type Parameters + +`TParams` + +The type of parameters the flow requires + +#### Remarks + +

+Use this overload when the flow requires configuration parameters. +Parameters are strongly typed and checked at compile time. +

+

+The factory signature must match: Func<TCatalog, TParams, Flow> +

+ +### WithDescription\(string\) + +Adds a description to the most recently registered flow. + +```csharp +IFlowRegistrar WithDescription(string description) +``` + +#### Parameters + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Human-readable description of what the flow does + +#### Returns + + [IFlowRegistrar](Flowthru.Registry.IFlowRegistrar\-1.md) + +This registrar for method chaining + +#### Remarks + +Use this overload when fluently chaining after Register(). + +### WithValidation\(Action\) + +```csharp +IFlowRegistrar WithValidation(Action configure) +``` + +#### Parameters + +`configure` [Action](https://learn.microsoft.com/dotnet/api/system.action\-1)<[ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md)\> + +#### Returns + + [IFlowRegistrar](Flowthru.Registry.IFlowRegistrar\-1.md) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Registry.IPipelineRegistrar-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Registry.IPipelineRegistrar-1.md new file mode 100644 index 00000000..05f962ad --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Registry.IPipelineRegistrar-1.md @@ -0,0 +1,172 @@ +# Interface IFlowRegistrar + +Namespace: [Flowthru.Registry](Flowthru.Registry.md) +Assembly: Flowthru.Core.dll + +Fluent interface for registering pipelines in a type-safe manner. + +```csharp +public interface IFlowRegistrar where TCatalog : DataCatalogBase +``` + +#### Type Parameters + +`TCatalog` + +The catalog type that pipelines will use + +## Remarks + +

+This interface provides compile-time type safety by tying pipeline factories +to a specific catalog type. The registrar validates that all registered +pipelines accept the correct catalog type. +

+

+Usage: +

protected override void RegisterPipelines(IFlowRegistrar<MyCatalog> registrar)
+{
+    // Pipeline without parameters
+    registrar.Register("processing", ProcessingPipeline.Create);
+
+    // Pipeline with parameters
+    registrar.Register("training", TrainPipeline.Create, new TrainOptions());
+
+    // Add metadata
+    registrar.WithDescription("processing", "Cleans and transforms raw data");
+}
+

+ +## Methods + +### Register\(string, Func\) + +Registers a pipeline with a parameterless factory function. + +```csharp +IFlowRegistrar Register(string name, Func pipelineFactory) +``` + +#### Parameters + +`name` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique pipeline name + +`pipelineFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +Factory function that creates the pipeline from catalog + +#### Returns + + [IFlowRegistrar](Flowthru.Registry.IFlowRegistrar\-1.md) + +This registrar for method chaining + +#### Remarks + +Use this overload when the pipeline doesn't require parameters. + +### Register\(string, Func, TParams\) + +Registers a pipeline with a parameterized factory function. + +```csharp +IFlowRegistrar Register(string name, Func pipelineFactory, TParams parameters) +``` + +#### Parameters + +`name` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique pipeline name + +`pipelineFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-3) + +Factory function that creates the pipeline from catalog and parameters + +`parameters` TParams + +Parameter instance to pass to the pipeline + +#### Returns + + [IFlowRegistrar](Flowthru.Registry.IFlowRegistrar\-1.md) + +This registrar for method chaining + +#### Type Parameters + +`TParams` + +The type of parameters the pipeline requires + +#### Remarks + +

+Use this overload when the pipeline requires configuration parameters. +Parameters are strongly typed and checked at compile time. +

+

+The factory signature must match: Func<TCatalog, TParams, Pipeline> +

+ +### WithDescription\(string\) + +Adds a description to the most recently registered pipeline. + +```csharp +IFlowRegistrar WithDescription(string description) +``` + +#### Parameters + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Human-readable description of what the pipeline does + +#### Returns + + [IFlowRegistrar](Flowthru.Registry.IFlowRegistrar\-1.md) + +This registrar for method chaining + +#### Remarks + +Use this overload when fluently chaining after Register(). + +### WithValidation\(Action\) + +Configures validation options for the most recently registered pipeline. + +```csharp +IFlowRegistrar WithValidation(Action configure) +``` + +#### Parameters + +`configure` [Action](https://learn.microsoft.com/dotnet/api/system.action\-1)<[ValidationOptions](Flowthru.Flows.Validation.ValidationOptions.md)\> + +Action to configure validation behavior + +#### Returns + + [IFlowRegistrar](Flowthru.Registry.IFlowRegistrar\-1.md) + +This registrar for method chaining + +#### Remarks + +

+Use this to opt into deep inspection for critical external data sources +or to explicitly disable inspection for specific inputs. +

+

+Example: +

+
registrar.Register("data_processing", ProcessingPipeline.Create)
+  .WithValidation(validation => {
+    validation.Inspect(catalog.Companies, InspectionLevel.Deep);
+    validation.Inspect(catalog.Shuttles, InspectionLevel.Deep);
+  });
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Registry.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Registry.md new file mode 100644 index 00000000..fc10c535 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Registry.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Registry + +### Interfaces + + [IFlowRegistrar](Flowthru.Registry.IFlowRegistrar\-1.md) + +Fluent interface for registering flows in a type-safe manner. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Results.ConsoleResultFormatter.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Results.ConsoleResultFormatter.md new file mode 100644 index 00000000..50bfdd17 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Results.ConsoleResultFormatter.md @@ -0,0 +1,55 @@ +# Class ConsoleResultFormatter + +Namespace: [Flowthru.Results](Flowthru.Results.md) +Assembly: Flowthru.Core.dll + +Formats pipeline results as human-readable console output. + +```csharp +public class ConsoleResultFormatter : IFlowResultFormatter +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ConsoleResultFormatter](Flowthru.Results.ConsoleResultFormatter.md) + +#### Implements + +[IFlowResultFormatter](Flowthru.Results.IFlowResultFormatter.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +This is the default formatter used by the CLI. +Produces colorful, detailed output suitable for interactive terminal sessions. + +## Methods + +### Format\(FlowResult, ILogger\) + +Formats and outputs the flow result. + +```csharp +public void Format(FlowResult result, ILogger logger) +``` + +#### Parameters + +`result` [FlowResult](Flowthru.Flows.FlowResult.md) + +The flow execution result + +`logger` [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger) + +The logger to write output to + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Results.IFlowResultFormatter.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Results.IFlowResultFormatter.md new file mode 100644 index 00000000..182763f3 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Results.IFlowResultFormatter.md @@ -0,0 +1,45 @@ +# Interface IFlowResultFormatter + +Namespace: [Flowthru.Results](Flowthru.Results.md) +Assembly: Flowthru.Core.dll + +Interface for formatting flow execution results. + +```csharp +public interface IFlowResultFormatter +``` + +## Remarks + +

+Result formatters transform a FlowResult into human-readable or +machine-readable output via logging. +

+

+Built-in formatters: +- - Human-readable console output (default) +

+

+Future formatters: JSON, Markdown, compact CI/CD format. +

+ +## Methods + +### Format\(FlowResult, ILogger\) + +Formats and outputs the flow result. + +```csharp +void Format(FlowResult result, ILogger logger) +``` + +#### Parameters + +`result` [FlowResult](Flowthru.Flows.FlowResult.md) + +The flow execution result + +`logger` [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger) + +The logger to write output to + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Results.IPipelineResultFormatter.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Results.IPipelineResultFormatter.md new file mode 100644 index 00000000..a635a6bf --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Results.IPipelineResultFormatter.md @@ -0,0 +1,45 @@ +# Interface IFlowResultFormatter + +Namespace: [Flowthru.Results](Flowthru.Results.md) +Assembly: Flowthru.Core.dll + +Interface for formatting pipeline execution results. + +```csharp +public interface IFlowResultFormatter +``` + +## Remarks + +

+Result formatters transform a FlowResult into human-readable or +machine-readable output via logging. +

+

+Built-in formatters: +- - Human-readable console output (default) +

+

+Future formatters: JSON, Markdown, compact CI/CD format. +

+ +## Methods + +### Format\(FlowResult, ILogger\) + +Formats and outputs the pipeline result. + +```csharp +void Format(FlowResult result, ILogger logger) +``` + +#### Parameters + +`result` [FlowResult](Flowthru.Flows.FlowResult.md) + +The pipeline execution result + +`logger` [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger) + +The logger to write output to + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Results.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Results.md new file mode 100644 index 00000000..d3723701 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Results.md @@ -0,0 +1,14 @@ +# Namespace Flowthru.Results + +### Classes + + [ConsoleResultFormatter](Flowthru.Results.ConsoleResultFormatter.md) + +Formats pipeline results as human-readable console output. + +### Interfaces + + [IFlowResultFormatter](Flowthru.Results.IFlowResultFormatter.md) + +Interface for formatting flow execution results. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Services.FlowthruServiceBuilder.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.FlowthruServiceBuilder.md new file mode 100644 index 00000000..035fa431 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.FlowthruServiceBuilder.md @@ -0,0 +1,385 @@ +# Class FlowthruServiceBuilder + +Namespace: [Flowthru.Services](Flowthru.Services.md) +Assembly: Flowthru.Core.dll + +Fluent builder for configuring Flowthru service registration. + +```csharp +public sealed class FlowthruServiceBuilder +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This builder configures the service layer without CLI coupling. +Use it to register catalogs, flows, and optional features. +

+

+Basic Usage: +

services.AddFlowthru(flowthru =>
+{
+    flowthru.RegisterCatalog(_ => new MyCatalog(dataPath));
+    flowthru.RegisterFlow("my_flow", MyFlow.Create);
+});
+

+ +## Methods + +### ConfigureMetadata\(Action\) + +Configures metadata export. + +```csharp +public FlowthruServiceBuilder ConfigureMetadata(Action configure) +``` + +#### Parameters + +`configure` [Action](https://learn.microsoft.com/dotnet/api/system.action\-1)<[FlowthruMetadataBuilder](Flowthru.Meta.FlowthruMetadataBuilder.md)\> + +Action to configure the metadata builder + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +#### Remarks + +

+Metadata export is optional. If not configured, flows will execute +without generating DAG diagrams or metadata files. +

+

+Example: +

flowthru.ConfigureMetadata(meta =>
+{
+    meta.WithOutputDirectory("metadata")
+        .AddProvider<JsonMetadataProvider, JsonMetadataProviderBuilder>()
+        .AddProvider<MermaidMetadataProvider, MermaidMetadataProviderBuilder>();
+});
+

+ +### RegisterCatalog\(\) + +Registers a catalog type with constructor injection. + +```csharp +public FlowthruServiceBuilder RegisterCatalog() where TCatalog : CatalogAbstract +``` + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TCatalog` + +The catalog type + +#### Remarks + +The catalog will be resolved from the DI container, allowing constructor +parameter injection (e.g., IConfiguration, IOptions). + +### RegisterCatalog\(CatalogAbstract\) + +Registers a catalog instance directly. + +```csharp +public FlowthruServiceBuilder RegisterCatalog(CatalogAbstract catalog) +``` + +#### Parameters + +`catalog` [CatalogAbstract](Flowthru.Data.CatalogAbstract.md) + +The catalog instance + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +#### Remarks + +Use this when the catalog doesn't require dependency injection. + +### RegisterCatalog\(Func\) + +Registers a catalog factory that receives the service provider. + +```csharp +public FlowthruServiceBuilder RegisterCatalog(Func catalogFactory) where TCatalog : CatalogAbstract +``` + +#### Parameters + +`catalogFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider), TCatalog\> + +Factory function to create the catalog + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TCatalog` + +The concrete catalog type + +#### Remarks + +Use this when the catalog needs to resolve services during construction, +or when construction requires parameters unavailable at the call site. + +### RegisterCatalogs\(IEnumerable\) + +Registers a collection of pre-built catalog instances produced by iterative or dynamic +construction — the fan-out pattern where N identical-shaped catalogs differ only by +their construction parameters (e.g., one catalog per US state). + +```csharp +public FlowthruServiceBuilder RegisterCatalogs(IEnumerable catalogs) +``` + +#### Parameters + +`catalogs` [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)<[CatalogAbstract](Flowthru.Data.CatalogAbstract.md)\> + +The catalog instances to register + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +#### Remarks + +All registered catalogs will receive DI service injection and appear in +. Use with to +wire per-catalog flows in a loop. + +### RegisterCatalogs\(Func\>\) + +Registers catalogs produced by a factory that receives the service provider — +useful when catalog construction itself requires DI resolution. + +```csharp +public FlowthruServiceBuilder RegisterCatalogs(Func> catalogsFactory) +``` + +#### Parameters + +`catalogsFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider), [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)<[CatalogAbstract](Flowthru.Data.CatalogAbstract.md)\>\> + +Factory that returns the catalog collection + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +### RegisterFlow\(string, Delegate, string?\) + +Registers a flow by inspecting the delegate's parameter types at runtime. +Parameters that extend are resolved from DI as catalogs. +All other parameters are resolved from DI as services. + +```csharp +public FlowthruServiceBuilder RegisterFlow(string label, Delegate flow, string? configurationSection = null) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique flow name + +`flow` [Delegate](https://learn.microsoft.com/dotnet/api/system.delegate) + +A method group or delegate whose parameters are catalogs, services, or config objects + +`configurationSection` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Optional configuration section path. When provided, the last non-catalog, non-service parameter +is bound from configuration instead of DI. + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +### RegisterFlows\(Func\>\) + +Escape-hatch for registering flows via a full-access service provider factory. + +```csharp +public FlowthruServiceBuilder RegisterFlows(Func> flowFactory) +``` + +#### Parameters + +`flowFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider), [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [Flow](Flowthru.Flows.Flow.md)\>\> + +Factory function that receives the service provider and returns the flow dictionary + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +#### Remarks + +Prefer for standard flow registration. +Use this only when you need full service provider access during flow construction. + +### UseConfiguration\(Action?\) + +Enables configuration loading from JSON and YAML files. + +```csharp +public FlowthruServiceBuilder UseConfiguration(Action? configure = null) +``` + +#### Parameters + +`configure` [Action](https://learn.microsoft.com/dotnet/api/system.action\-1)<[FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md)\>? + +Optional action to configure how configuration files are loaded + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +#### Remarks + +By default, configuration is loaded from appsettings.json and environment-specific overrides. + +### UseStorageStrategy\(\) + +Registers a storage entry factory (for environment-specific entries). + +```csharp +public FlowthruServiceBuilder UseStorageStrategy() where TStrategy : class, IStorageEntryFactory +``` + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +#### Type Parameters + +`TStrategy` + +The storage strategy type + +#### Remarks + +

+Storage strategies enable environment-specific catalog entries: +

+
if (env.IsDevelopment())
+{
+    flowthru.UseStorageStrategy<CsvStorageEntryFactory>();
+}
+else if (env.IsProduction())
+{
+    flowthru.UseStorageStrategy<DatabaseStorageEntryFactory>();
+}
+else if (env.IsTest())
+{
+    flowthru.UseStorageStrategy<MemoryStorageEntryFactory>();
+}
+ +### UseStorageStrategy\(IStorageEntryFactory\) + +Registers a storage entry factory instance. + +```csharp +public FlowthruServiceBuilder UseStorageStrategy(IStorageEntryFactory strategy) +``` + +#### Parameters + +`strategy` [IStorageEntryFactory](Flowthru.Data.Storage.Strategies.IStorageEntryFactory.md) + +The storage strategy instance + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +### UseStorageStrategy\(Func\) + +Registers a storage entry factory using a factory function. + +```csharp +public FlowthruServiceBuilder UseStorageStrategy(Func strategyFactory) +``` + +#### Parameters + +`strategyFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider), [IStorageEntryFactory](Flowthru.Data.Storage.Strategies.IStorageEntryFactory.md)\> + +Factory function to create the strategy + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + +### WithDescription\(string\) + +Adds a description to the most recently registered flow. + +```csharp +public FlowthruServiceBuilder WithDescription(string description) +``` + +#### Parameters + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Human-readable description of what the flow does + +#### Returns + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +This builder for method chaining + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Services.IFlowthruService.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.IFlowthruService.md new file mode 100644 index 00000000..ce32182a --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.IFlowthruService.md @@ -0,0 +1,248 @@ +# Interface IFlowthruService + +Namespace: [Flowthru.Services](Flowthru.Services.md) +Assembly: Flowthru.Core.dll + +Core service for executing Flowthru flows programmatically. + +```csharp +public interface IFlowthruService +``` + +## Remarks + +

+This service is DI-injectable and CLI-agnostic, enabling use in: +- Console applications (via shallow CLI wrapper) +- ASP.NET Core applications (controller/background service injection) +- Azure Functions (function injection) +- Unit tests (with mocked dependencies) +

+

+Usage Example: +

public class DataProcessingService
+{
+    private readonly IFlowthruService _flowthru;
+
+    public DataProcessingService(IFlowthruService flowthru)
+    {
+        _flowthru = flowthru;
+    }
+
+    public async Task ProcessData()
+    {
+        // Execute with optional slicing
+        var options = new ExecutionOptions
+        {
+            DryRun = false,
+            SliceStrategy = new FlowSliceStrategy
+            {
+                Flows = new HashSet<string> { "data_processing" }
+            }
+        };
+
+        var result = await _flowthru.ExecuteFlowAsync(options);
+
+        if (result.Success)
+        {
+            Console.WriteLine($"Processed {result.StepResults.Count} flow");
+        }
+    }
+}
+

+ +## Properties + +### Catalogs + +Gets all registered catalog instances. + +```csharp +IReadOnlyList Catalogs { get; } +``` + +#### Property Value + + [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[CatalogAbstract](Flowthru.Data.CatalogAbstract.md)\> + +### FlowNames + +Gets the names of all registered flows. + +```csharp +IReadOnlyCollection FlowNames { get; } +``` + +#### Property Value + + [IReadOnlyCollection](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlycollection\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +## Methods + +### ExecuteFlowAsync\(ExecutionOptions?, bool, string?, CancellationToken\) + +Executes all registered flows, optionally sliced by criteria. + +```csharp +Task ExecuteFlowAsync(ExecutionOptions? options = null, bool exportMetadata = true, string? metadataOutputDirectory = null, CancellationToken cancellationToken = default) +``` + +#### Parameters + +`options` [ExecutionOptions](Flowthru.Flows.ExecutionOptions.md)? + +Execution options with optional slice strategy + +`exportMetadata` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +Whether to export DAG metadata + +`metadataOutputDirectory` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Override for metadata output directory + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[FlowResult](Flowthru.Flows.FlowResult.md)\> + +Execution result with timing, step results, and status + +#### Remarks + +This method always merges all registered flows into a single DAG, +then applies optional slicing criteria from the execution options. +This enables cross-flow queries (e.g., --to-data across all flows). +To execute only specific flows, use SliceStrategy.Flows. + +The method performs: +1. Flow merging into unified DAG +2. Service injection +3. DAG building and slice application +4. Metadata export (if requested) +5. External input validation +6. Flow execution (unless dry run) +7. Result formatting + +### GetDagMetadata\(string?, FlowSliceStrategy?\) + +Gets the full DAG metadata for flow introspection. + +```csharp +DagMetadata GetDagMetadata(string? flowName = null, FlowSliceStrategy? sliceStrategy = null) +``` + +#### Parameters + +`flowName` [string](https://learn.microsoft.com/dotnet/api/system.string)? + +Optional flow name to inspect a single flow. +When null, all registered flows are merged into a unified DAG. + +`sliceStrategy` [FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md)? + +Optional slice strategy to filter the DAG (e.g., from-node). +When provided, the returned metadata includes slice overlay information +(SlicedNodeIds and SlicedItemKeys) identifying which nodes +and data are in the active execution subset. + +#### Returns + + [DagMetadata](Flowthru.Meta.Models.DagMetadata.md) + +Full DAG metadata including steps, catalog entries, edges, schemas, +and producer-consumer relationships. + +#### Remarks + +This method does not execute the flow. It returns structural metadata +useful for visualization, impact analysis, data lineage, and debugging. + +Examples: +
// Inspect all flow merged
+var dag = flowthru.GetDagMetadata();
+
+// Inspect a single flow
+var dag = flowthru.GetDagMetadata("DataProcessing");
+
+// Inspect downstream of a specific flow node
+var dag = flowthru.GetDagMetadata(sliceStrategy: new FlowSliceStrategy
+{
+    FromNodes = new HashSet<string> { "PreprocessCompanies" }
+});
+ +#### Exceptions + + [KeyNotFoundException](https://learn.microsoft.com/dotnet/api/system.collections.generic.keynotfoundexception) + +Thrown if flowName is specified but not found. + +### GetFlowMetadata\(string\) + +Gets metadata about a Flow's structure. + +```csharp +FlowMetadata GetFlowMetadata(string flowName) +``` + +#### Parameters + +`flowName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Flow name + +#### Returns + + [FlowMetadata](Flowthru.Services.Models.FlowMetadata.md) + +Flow metadata + +#### Remarks + +Returns structural information without executing the flow. +The flow must be built for accurate layer and input information. + +#### Exceptions + + [KeyNotFoundException](https://learn.microsoft.com/dotnet/api/system.collections.generic.keynotfoundexception) + +Thrown if flow name not found + +### ValidateFlowAsync\(string, CancellationToken\) + +Validates all external inputs (Layer 0) for a flow. + +```csharp +Task ValidateFlowAsync(string flowName, CancellationToken cancellationToken = default) +``` + +#### Parameters + +`flowName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Flow name + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\> + +Validation result + +#### Remarks + +Checks accessibility of external data sources without executing the flow. +Useful for pre-flight validation in CI/CD or scheduled jobs. + +#### Exceptions + + [KeyNotFoundException](https://learn.microsoft.com/dotnet/api/system.collections.generic.keynotfoundexception) + +Thrown if flow name not found + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.FlowMetadata.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.FlowMetadata.md new file mode 100644 index 00000000..eafe80fb --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.FlowMetadata.md @@ -0,0 +1,108 @@ +# Class FlowMetadata + +Namespace: [Flowthru.Services.Models](Flowthru.Services.Models.md) +Assembly: Flowthru.Core.dll + +Metadata about a flow's structure and configuration. + +```csharp +public sealed record FlowMetadata : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowMetadata](Flowthru.Services.Models.FlowMetadata.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Provides read-only information about a flow without executing it. +Useful for discovery, validation, and UI generation. + +## Properties + +### Description + +Optional description of the flow's purpose. + +```csharp +public string? Description { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### ExternalInputs + +Labels of external data sources (Layer 0 inputs). + +```csharp +public required IReadOnlyList ExternalInputs { get; init; } +``` + +#### Property Value + + [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +### IsBuilt + +Whether the flow has been built (DAG analyzed). + +```csharp +public required bool IsBuilt { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### LayerCount + +Number of execution layers in the flow's DAG. + +```csharp +public required int LayerCount { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### Name + +The flow's registered name. + +```csharp +public required string Name { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### StepCount + +Total number of steps in the flow. + +```csharp +public required int StepCount { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.PipelineExecutionRequest.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.PipelineExecutionRequest.md new file mode 100644 index 00000000..6e8284f1 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.PipelineExecutionRequest.md @@ -0,0 +1,109 @@ +# Class PipelineExecutionRequest + +Namespace: [Flowthru.Services.Models](Flowthru.Services.Models.md) +Assembly: Flowthru.Core.dll + +Request model for pipeline execution. + +```csharp +public record PipelineExecutionRequest : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PipelineExecutionRequest](Flowthru.Services.Models.PipelineExecutionRequest.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Encapsulates all configuration needed to execute a pipeline programmatically, +separate from CLI argument parsing. + +## Properties + +### ExportMetadata + +Whether to export DAG metadata. + +```csharp +public bool ExportMetadata { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +#### Remarks + +Defaults to true. Only applies if a metadata builder is configured. + +### MetadataOutputDirectory + +Output directory for metadata (if null, uses default from metadata builder). + +```csharp +public string? MetadataOutputDirectory { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### Options + +Execution options (dry run, parallel execution, etc.). + +```csharp +public ExecutionOptions? Options { get; init; } +``` + +#### Property Value + + [ExecutionOptions](Flowthru.Flows.ExecutionOptions.md)? + +#### Remarks + +If null, uses default execution options. + +### FlowName + +Name of the pipeline to execute. + +```csharp +public required string FlowName { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### SliceStrategy + +Optional slicing strategy to execute a subset of the pipeline. + +```csharp +public FlowSliceStrategy? SliceStrategy { get; init; } +``` + +#### Property Value + + [FlowSliceStrategy](Flowthru.Flows.FlowSliceStrategy.md)? + +#### Remarks + +If null or IsSliced=false, the entire pipeline executes. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.PipelineMetadata.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.PipelineMetadata.md new file mode 100644 index 00000000..e608e10e --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.PipelineMetadata.md @@ -0,0 +1,108 @@ +# Class PipelineMetadata + +Namespace: [Flowthru.Services.Models](Flowthru.Services.Models.md) +Assembly: Flowthru.Core.dll + +Metadata about a pipeline's structure and configuration. + +```csharp +public sealed record PipelineMetadata : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PipelineMetadata](Flowthru.Services.Models.PipelineMetadata.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Provides read-only information about a pipeline without executing it. +Useful for discovery, validation, and UI generation. + +## Properties + +### Description + +Optional description of the pipeline's purpose. + +```csharp +public string? Description { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### ExternalInputs + +Labels of external data sources (Layer 0 inputs). + +```csharp +public required IReadOnlyList ExternalInputs { get; init; } +``` + +#### Property Value + + [IReadOnlyList](https://learn.microsoft.com/dotnet/api/system.collections.generic.ireadonlylist\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +### IsBuilt + +Whether the pipeline has been built (DAG analyzed). + +```csharp +public required bool IsBuilt { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### LayerCount + +Number of execution layers in the pipeline's DAG. + +```csharp +public required int LayerCount { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### Name + +The pipeline's registered name. + +```csharp +public required string Name { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### NodeCount + +Total number of nodes in the pipeline. + +```csharp +public required int NodeCount { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.md new file mode 100644 index 00000000..34276b54 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.Models.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Services.Models + +### Classes + + [FlowMetadata](Flowthru.Services.Models.FlowMetadata.md) + +Metadata about a flow's structure and configuration. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Services.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.md new file mode 100644 index 00000000..7fb5f6d2 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Services.md @@ -0,0 +1,18 @@ +# Namespace Flowthru.Services + +### Namespaces + + [Flowthru.Services.Models](Flowthru.Services.Models.md) + +### Classes + + [FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md) + +Fluent builder for configuring Flowthru service registration. + +### Interfaces + + [IFlowthruService](Flowthru.Services.IFlowthruService.md) + +Core service for executing Flowthru flows programmatically. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.Factory.StepFactory.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.Factory.StepFactory.md new file mode 100644 index 00000000..d14f71ba --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.Factory.StepFactory.md @@ -0,0 +1,74 @@ +# Class StepFactory + +Namespace: [Flowthru.Steps.Factory](Flowthru.Steps.Factory.md) +Assembly: Flowthru.Core.dll + +Factory for creating step instances using TypeActivator. + +```csharp +public static class StepFactory +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[StepFactory](Flowthru.Steps.Factory.StepFactory.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Design Pattern: Factory Pattern - provides a centralized location for +step instantiation logic. +

+

+This is a thin wrapper around TypeActivator, providing a domain-specific API for +creating steps. Could be extended in the future with: +- Step validation logic +- Pre/post-creation hooks +- Step decoration/wrapping +

+ +## Methods + +### Create\(\) + +Creates a new instance of the specified step type. + +```csharp +public static TStep Create() where TStep : new() +``` + +#### Returns + + TStep + +A new step instance + +#### Type Parameters + +`TStep` + +The step type to instantiate + +#### Remarks + +

+Requirements: +- TStep must inherit from StepBase<TInput, TOutput> +- TStep must have a parameterless constructor +

+

+These requirements are enforced at compile-time via generic constraints in +FlowBuilder.AddStep methods. +

+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.Factory.TypeActivator.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.Factory.TypeActivator.md new file mode 100644 index 00000000..29f73898 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.Factory.TypeActivator.md @@ -0,0 +1,114 @@ +# Class TypeActivator + +Namespace: [Flowthru.Steps.Factory](Flowthru.Steps.Factory.md) +Assembly: Flowthru.Core.dll + +Factory for creating instances of types using compiled expression trees for performance. + +```csharp +public static class TypeActivator +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[TypeActivator](Flowthru.Steps.Factory.TypeActivator.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Design Pattern: Factory Pattern with caching - creates instances of types +using reflection on first call, then caches compiled expression trees for subsequent calls. +

+

+Performance: +- First call uses Expression.Compile() which has overhead +- Subsequent calls use cached delegate which is nearly as fast as `new T()` +- Significantly faster than Activator.CreateInstance<T>() for repeated calls +

+

+Inspiration: ChainSharp uses similar pattern for node instantiation. +

+

+Thread Safety: This class is thread-safe. Multiple threads can safely +call Create<T>() concurrently. +

+ +## Properties + +### CacheCount + +Gets the number of cached factory functions. + +```csharp +public static int CacheCount { get; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +## Methods + +### ClearCache\(\) + +Clears the factory cache. + +```csharp +public static void ClearCache() +``` + +#### Remarks + +Useful for testing or memory management in long-running applications +that dynamically load/unload types. + +### Create\(\) + +Creates an instance of type T using a cached factory. + +```csharp +public static T Create() where T : new() +``` + +#### Returns + + T + +A new instance of type T + +#### Type Parameters + +`T` + +The type to instantiate (must have parameterless constructor) + +#### Remarks + +

+Compile-Time Safety: The `new()` constraint ensures that T has a +parameterless constructor. This is enforced at compile-time by the C# compiler. +

+

+Caching Strategy: +- First call: Compiles an expression tree and caches the resulting delegate +- Subsequent calls: Reuses the cached delegate +- One cache entry per type T +

+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if type T does not have a parameterless constructor + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.Factory.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.Factory.md new file mode 100644 index 00000000..0ebab3e3 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.Factory.md @@ -0,0 +1,12 @@ +# Namespace Flowthru.Steps.Factory + +### Classes + + [StepFactory](Flowthru.Steps.Factory.StepFactory.md) + +Factory for creating step instances using TypeActivator. + + [TypeActivator](Flowthru.Steps.Factory.TypeActivator.md) + +Factory for creating instances of types using compiled expression trees for performance. + diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.NoData.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.NoData.md new file mode 100644 index 00000000..3236968f --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.NoData.md @@ -0,0 +1,182 @@ +# Class NoData + +Namespace: [Flowthru.Steps](Flowthru.Steps.md) +Assembly: Flowthru.Core.dll + +Marker type representing "no meaningful data" for nodes with side-effects or data generation. +Used as input/output type in NodeBase when a node doesn't consume or produce meaningful data. + +```csharp +public sealed class NoData +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[NoData](Flowthru.Steps.NoData.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Design Rationale: NoData provides a type-safe way to represent nodes that: +- Generate data without inputs (e.g., synthetic data generation, seeding) +- Perform side-effects without outputs (e.g., validation, logging, diagnostics) +

+

+This pattern is inspired by functional programming's "Unit" type but uses more intuitive +naming for .NET developers unfamiliar with functional terminology. +

+

+Usage Examples: +

+
// Node with no inputs (data generation)
+public class GenerateDataNode : NodeBase<NoData, OutputSchema>
+{
+    protected override Task<IEnumerable<OutputSchema>> Transform(IEnumerable<NoData> input)
+    {
+        // Generate data from scratch...
+        return Task.FromResult(generatedData);
+    }
+}
+
+// Node with no outputs (side-effects only)
+public class ValidateNode : NodeBase<InputSchema, NoData>
+{
+    protected override Task<IEnumerable<NoData>> Transform(IEnumerable<InputSchema> input)
+    {
+        // Perform validation, logging, etc...
+        return Task.FromResult(Enumerable.Repeat(NoData.Value, 1));
+    }
+}
+

+Pipeline Registration: Use NoData type directly - it automatically converts +to a unique NullCatalogDataset instance: +

+
// Simple syntax with automatic unique key generation
+pipeline.AddStep<ValidationNode>(
+    input: catalog.InputData,
+    output: NoData.Output  // or just: NoData.Discard
+);
+
+pipeline.AddStep<GenerateDataNode>(
+    input: NoData.Input,  // or just: NoData.None
+    output: catalog.GeneratedData
+);
+ +## Fields + +### Value + +Singleton instance of NoData. +Use this value when returning NoData from node transformations. + +```csharp +public static readonly NoData Value +``` + +#### Field Value + + [NoData](Flowthru.Steps.NoData.md) + +## Properties + +### Discard + +Creates a unique null catalog entry for use as a node output (side-effect-only nodes). +Semantic alias for Output - use whichever reads better in context. + +```csharp +public static IItem Discard { get; } +``` + +#### Property Value + + [IItem](Flowthru.Data.IItem\-1.md)<[NoData](Flowthru.Steps.NoData.md)\> + +### Input + +Creates a unique null catalog entry for use as a node input (no-input nodes). +Each call generates a new instance with a unique key to avoid DAG conflicts. + +```csharp +public static IItem Input { get; } +``` + +#### Property Value + + [IItem](Flowthru.Data.IItem\-1.md)<[NoData](Flowthru.Steps.NoData.md)\> + +#### Remarks + +Alias for readability in pipeline declarations where nodes don't consume external inputs. + +### None + +Creates a unique null catalog entry for use as a node input (no-input nodes). +Semantic alias for Input - use whichever reads better in context. + +```csharp +public static IItem None { get; } +``` + +#### Property Value + + [IItem](Flowthru.Data.IItem\-1.md)<[NoData](Flowthru.Steps.NoData.md)\> + +### Output + +Creates a unique null catalog entry for use as a node output (side-effect-only nodes). +Each call generates a new instance with a unique key to avoid DAG conflicts. + +```csharp +public static IItem Output { get; } +``` + +#### Property Value + + [IItem](Flowthru.Data.IItem\-1.md)<[NoData](Flowthru.Steps.NoData.md)\> + +#### Remarks + +Alias for readability in pipeline declarations where nodes produce no meaningful output. + +## Methods + +### Result\(\) + +Returns the standard NoData result for side-effect-only nodes. +Use this at the end of Transform() methods that return NoData. + +```csharp +public static Task> Result() +``` + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)<[NoData](Flowthru.Steps.NoData.md)\>\> + +Singleton collection containing NoData.Value + +#### Remarks + +

+This helper eliminates the verbose Task.FromResult(Enumerable.Repeat(NoData.Value, 1)) +boilerplate. Simply return NoData.Result(). +

+ +
// Instead of:
+return Task.FromResult(Enumerable.Repeat(NoData.Value, 1));
+
+// Use:
+return NoData.Result();
+
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.NoParams.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.NoParams.md new file mode 100644 index 00000000..d8335245 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.NoParams.md @@ -0,0 +1,49 @@ +# Class NoParams + +Namespace: [Flowthru.Steps](Flowthru.Steps.md) +Assembly: Flowthru.Core.dll + +Marker type for nodes that don't require parameters. +Used as the default TParameters type in NodeBase<TInput, TOutput, TParameters>. + +```csharp +public sealed class NoParams +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[NoParams](Flowthru.Steps.NoParams.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This is a simple empty class that serves as a shorthand for users and the library +when no parameters are needed for a node. Nodes that don't need configuration can +omit the third type parameter by using the two-parameter NodeBase<TInput, TOutput> +convenience base class. +

+

+Usage Examples: +

+
// Explicit NoParams (rarely needed)
+public class MyNode : NodeBase<Input, Output, NoParams> { }
+
+// Recommended: Use two-parameter base class
+public class MyNode : NodeBase<Input, Output> { }
+
+// With parameters
+public class ConfigurableNode : NodeBase<Input, Output, MyParameters>
+{
+    // Parameters property is automatically available
+}
+ diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.md new file mode 100644 index 00000000..55745091 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Steps.md @@ -0,0 +1,18 @@ +# Namespace Flowthru.Steps + +### Namespaces + + [Flowthru.Steps.Factory](Flowthru.Steps.Factory.md) + +### Classes + + [NoData](Flowthru.Steps.NoData.md) + +Marker type representing "no meaningful data" for nodes with side-effects or data generation. +Used as input/output type in NodeBase when a node doesn't consume or produce meaningful data. + + [NoParams](Flowthru.Steps.NoParams.md) + +Marker type for nodes that don't require parameters. +Used as the default TParameters type in NodeBase<TInput, TOutput, TParameters>. + diff --git a/docs/reference/src/core/Flowthru.Core/Microsoft.Extensions.DependencyInjection.FlowthruServiceCollectionExtensions.md b/docs/reference/src/core/Flowthru.Core/Microsoft.Extensions.DependencyInjection.FlowthruServiceCollectionExtensions.md new file mode 100644 index 00000000..d46a63ca --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Microsoft.Extensions.DependencyInjection.FlowthruServiceCollectionExtensions.md @@ -0,0 +1,94 @@ +# Class FlowthruServiceCollectionExtensions + +Namespace: [Microsoft.Extensions.DependencyInjection](Microsoft.Extensions.DependencyInjection.md) +Assembly: Flowthru.Core.dll + +Extension methods for registering Flowthru services with the DI container. + +```csharp +public static class FlowthruServiceCollectionExtensions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowthruServiceCollectionExtensions](Microsoft.Extensions.DependencyInjection.FlowthruServiceCollectionExtensions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### AddFlowthru\(IServiceCollection, Action\) + +Registers Flowthru service with the DI container. + +```csharp +public static IServiceCollection AddFlowthru(this IServiceCollection services, Action configure) +``` + +#### Parameters + +`services` [IServiceCollection](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dependencyinjection.iservicecollection) + +The service collection + +`configure` [Action](https://learn.microsoft.com/dotnet/api/system.action\-1)<[FlowthruServiceBuilder](Flowthru.Services.FlowthruServiceBuilder.md)\> + +Action to configure the Flowthru service + +#### Returns + + [IServiceCollection](https://learn.microsoft.com/dotnet/api/microsoft.extensions.dependencyinjection.iservicecollection) + +The service collection for method chaining + +#### Remarks + +

+This extension method provides a clean API for registering Flowthru +in any .NET application with dependency injection. +

+

+Example Usage: +

// In Program.cs or Startup.cs
+services.AddFlowthru(flowthru =>
+{
+    flowthru.RegisterCatalog<MyCatalog>();
+    flowthru.RegisterPipelines(catalog => new Dictionary<string, Pipeline>
+    {
+        ["my_pipeline"] = MyPipeline.Create((MyCatalog)catalog)
+    });
+    flowthru.ConfigureMetadata(meta =>
+    {
+        meta.WithOutputDirectory("metadata")
+            .AddProvider<MermaidMetadataProvider, MermaidMetadataProviderBuilder>();
+    });
+});
+
+// Then inject IFlowthruService anywhere
+public class MyController
+{
+    private readonly IFlowthruService _flowthru;
+
+    public MyController(IFlowthruService flowthru)
+    {
+        _flowthru = flowthru;
+    }
+
+    public async Task<IActionResult> RunPipeline(string name)
+    {
+        var request = new PipelineExecutionRequest { FlowName = name };
+        var result = await _flowthru.ExecutePipelineAsync(request);
+        return Ok(result);
+    }
+}
+

+ diff --git a/docs/reference/src/core/Flowthru.Core/Microsoft.Extensions.DependencyInjection.md b/docs/reference/src/core/Flowthru.Core/Microsoft.Extensions.DependencyInjection.md new file mode 100644 index 00000000..ec963843 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/Microsoft.Extensions.DependencyInjection.md @@ -0,0 +1,8 @@ +# Namespace Microsoft.Extensions.DependencyInjection + +### Classes + + [FlowthruServiceCollectionExtensions](Microsoft.Extensions.DependencyInjection.FlowthruServiceCollectionExtensions.md) + +Extension methods for registering Flowthru services with the DI container. + diff --git a/docs/reference/src/core/Flowthru.Core/toc.yml b/docs/reference/src/core/Flowthru.Core/toc.yml new file mode 100644 index 00000000..d18e7360 --- /dev/null +++ b/docs/reference/src/core/Flowthru.Core/toc.yml @@ -0,0 +1,318 @@ +### YamlMime:TableOfContent +- name: Flowthru.Abstractions + href: Flowthru.Abstractions.md + items: + - name: Classes + - name: FlowthruSchemaAttribute + href: Flowthru.Abstractions.FlowthruSchemaAttribute.md + - name: SerializedEnumAttribute + href: Flowthru.Abstractions.SerializedEnumAttribute.md + - name: SerializedLabelAttribute + href: Flowthru.Abstractions.SerializedLabelAttribute.md + - name: Interfaces + - name: IBinarySerializable + href: Flowthru.Abstractions.IBinarySerializable.md + - name: IFlatSchema + href: Flowthru.Abstractions.IFlatSchema.md + - name: IFlatSerializable + href: Flowthru.Abstractions.IFlatSerializable.md + - name: INestedSchema + href: Flowthru.Abstractions.INestedSchema.md + - name: INestedSerializable + href: Flowthru.Abstractions.INestedSerializable.md + - name: IScalar + href: Flowthru.Abstractions.IScalar.md + - name: IStructuredSerializable + href: Flowthru.Abstractions.IStructuredSerializable.md + - name: ITextSerializable + href: Flowthru.Abstractions.ITextSerializable.md +- name: Flowthru.Cli + href: Flowthru.Cli.md + items: + - name: Classes + - name: FlowthruCli + href: Flowthru.Cli.FlowthruCli.md +- name: Flowthru.Configuration + href: Flowthru.Configuration.md + items: + - name: Classes + - name: CatalogOptions + href: Flowthru.Configuration.CatalogOptions.md + - name: ConfigurationExtensions + href: Flowthru.Configuration.ConfigurationExtensions.md + - name: FlowOptions + href: Flowthru.Configuration.FlowOptions.md + - name: FlowValidationOptions + href: Flowthru.Configuration.FlowValidationOptions.md + - name: FlowthruConfigurationOptions + href: Flowthru.Configuration.FlowthruConfigurationOptions.md + - name: FlowthruOptions + href: Flowthru.Configuration.FlowthruOptions.md + - name: JsonMetadataOptions + href: Flowthru.Configuration.JsonMetadataOptions.md + - name: LoggingOptions + href: Flowthru.Configuration.LoggingOptions.md + - name: MermaidMetadataOptions + href: Flowthru.Configuration.MermaidMetadataOptions.md + - name: MetadataOptions + href: Flowthru.Configuration.MetadataOptions.md + - name: TimestampConfiguration + href: Flowthru.Configuration.TimestampConfiguration.md + - name: Interfaces + - name: ICatalogFactory + href: Flowthru.Configuration.ICatalogFactory.md +- name: Flowthru.Data + href: Flowthru.Data.md + items: + - name: Classes + - name: CatalogAbstract + href: Flowthru.Data.CatalogAbstract.md + - name: EnumerableItems + href: Flowthru.Data.EnumerableItems.md + - name: Item + href: Flowthru.Data.Item-1.md + - name: Items + href: Flowthru.Data.Items.md + - name: Items.Single + href: Flowthru.Data.Items.Single.md + - name: Interfaces + - name: IItem + href: Flowthru.Data.IItem.md + - name: IItem + href: Flowthru.Data.IItem-1.md +- name: Flowthru.Data.Capabilities + href: Flowthru.Data.Capabilities.md + items: + - name: Classes + - name: StorageTraits + href: Flowthru.Data.Capabilities.StorageTraits.md +- name: Flowthru.Data.Storage + href: Flowthru.Data.Storage.md + items: + - name: Classes + - name: BinaryFileStorageAdapter + href: Flowthru.Data.Storage.BinaryFileStorageAdapter.md + - name: ComposedStorageAdapter + href: Flowthru.Data.Storage.ComposedStorageAdapter-2.md + - name: MemoryStorageAdapter + href: Flowthru.Data.Storage.MemoryStorageAdapter-1.md + - name: NullStorageAdapter + href: Flowthru.Data.Storage.NullStorageAdapter-1.md + - name: PropertyMappingConfiguration + href: Flowthru.Data.Storage.PropertyMappingConfiguration.md + - name: SchemaActivator + href: Flowthru.Data.Storage.SchemaActivator.md + - name: SingletonJsonStorageAdapter + href: Flowthru.Data.Storage.SingletonJsonStorageAdapter-1.md + - name: TextFileStorageAdapter + href: Flowthru.Data.Storage.TextFileStorageAdapter.md + - name: Interfaces + - name: IContainerAdapter + href: Flowthru.Data.Storage.IContainerAdapter-2.md + - name: IFormatSerializer + href: Flowthru.Data.Storage.IFormatSerializer-1.md + - name: IStorageAdapter + href: Flowthru.Data.Storage.IStorageAdapter-1.md + - name: IStorageMedium + href: Flowthru.Data.Storage.IStorageMedium.md + - name: Enums + - name: PropertyMappingStrategy + href: Flowthru.Data.Storage.PropertyMappingStrategy.md +- name: Flowthru.Data.Storage.Container + href: Flowthru.Data.Storage.Container.md + items: + - name: Classes + - name: EnumerableContainerAdapter + href: Flowthru.Data.Storage.Container.EnumerableContainerAdapter-1.md +- name: Flowthru.Data.Storage.Format + href: Flowthru.Data.Storage.Format.md + items: + - name: Classes + - name: JsonFormatSerializer + href: Flowthru.Data.Storage.Format.JsonFormatSerializer-1.md + - name: PropertyMappingHelper + href: Flowthru.Data.Storage.Format.PropertyMappingHelper.md +- name: Flowthru.Data.Storage.Medium + href: Flowthru.Data.Storage.Medium.md + items: + - name: Classes + - name: FileStorageMedium + href: Flowthru.Data.Storage.Medium.FileStorageMedium.md + - name: MemoryStorageMedium + href: Flowthru.Data.Storage.Medium.MemoryStorageMedium.md +- name: Flowthru.Data.Storage.Strategies + href: Flowthru.Data.Storage.Strategies.md + items: + - name: Classes + - name: DatabaseStorageEntryFactory + href: Flowthru.Data.Storage.Strategies.DatabaseStorageEntryFactory.md + - name: MemoryStorageEntryFactory + href: Flowthru.Data.Storage.Strategies.MemoryStorageEntryFactory.md + - name: StorageOptions + href: Flowthru.Data.Storage.Strategies.StorageOptions.md + - name: Interfaces + - name: IStorageEntryFactory + href: Flowthru.Data.Storage.Strategies.IStorageEntryFactory.md +- name: Flowthru.Data.Validation + href: Flowthru.Data.Validation.md + items: + - name: Classes + - name: ValidationError + href: Flowthru.Data.Validation.ValidationError.md + - name: ValidationException + href: Flowthru.Data.Validation.ValidationException.md + - name: ValidationResult + href: Flowthru.Data.Validation.ValidationResult.md + - name: Enums + - name: InspectionLevel + href: Flowthru.Data.Validation.InspectionLevel.md + - name: ValidationErrorType + href: Flowthru.Data.Validation.ValidationErrorType.md +- name: Flowthru.Effects + href: Flowthru.Effects.md + items: + - name: Classes + - name: FlowIO + href: Flowthru.Effects.FlowIO.md + - name: Structs + - name: FlowIO + href: Flowthru.Effects.FlowIO-1.md + - name: FlowUnit + href: Flowthru.Effects.FlowUnit.md +- name: Flowthru.Flows + href: Flowthru.Flows.md + items: + - name: Classes + - name: ExecutionOptions + href: Flowthru.Flows.ExecutionOptions.md + - name: Flow + href: Flowthru.Flows.Flow.md + - name: FlowBuilder + href: Flowthru.Flows.FlowBuilder.md + - name: FlowResult + href: Flowthru.Flows.FlowResult.md + - name: FlowSliceStrategy + href: Flowthru.Flows.FlowSliceStrategy.md + - name: FlowStep + href: Flowthru.Flows.FlowStep.md + - name: StepResult + href: Flowthru.Flows.StepResult.md + - name: Structs + - name: DryRunOption + href: Flowthru.Flows.DryRunOption.md + - name: Enums + - name: ValidationDepth + href: Flowthru.Flows.ValidationDepth.md +- name: Flowthru.Flows.Validation + href: Flowthru.Flows.Validation.md + items: + - name: Classes + - name: ValidationOptions + href: Flowthru.Flows.Validation.ValidationOptions.md + - name: Interfaces + - name: IFlowValidationHook + href: Flowthru.Flows.Validation.IFlowValidationHook.md +- name: Flowthru.Meta + href: Flowthru.Meta.md + items: + - name: Classes + - name: FlowthruMetadataBuilder + href: Flowthru.Meta.FlowthruMetadataBuilder.md + - name: FlowthruMetadataConfiguration + href: Flowthru.Meta.FlowthruMetadataConfiguration.md + - name: JsonMetadataProviderBuilder + href: Flowthru.Meta.JsonMetadataProviderBuilder.md + - name: MermaidMetadataExtensions + href: Flowthru.Meta.MermaidMetadataExtensions.md + - name: MermaidMetadataProviderBuilder + href: Flowthru.Meta.MermaidMetadataProviderBuilder.md + - name: MetadataJsonExtensions + href: Flowthru.Meta.MetadataJsonExtensions.md + - name: MetadataProviderBuilderAttribute + href: Flowthru.Meta.MetadataProviderBuilderAttribute.md + - name: TimestampConfiguration + href: Flowthru.Meta.TimestampConfiguration.md +- name: Flowthru.Meta.Models + href: Flowthru.Meta.Models.md + items: + - name: Classes + - name: DagMetadata + href: Flowthru.Meta.Models.DagMetadata.md + - name: DagSliceMetadata + href: Flowthru.Meta.Models.DagSliceMetadata.md + - name: EdgeMetadata + href: Flowthru.Meta.Models.EdgeMetadata.md + - name: ItemMetadata + href: Flowthru.Meta.Models.ItemMetadata.md + - name: SchemaField + href: Flowthru.Meta.Models.SchemaField.md + - name: SchemaMetadata + href: Flowthru.Meta.Models.SchemaMetadata.md + - name: StepMetadata + href: Flowthru.Meta.Models.StepMetadata.md +- name: Flowthru.Meta.Providers + href: Flowthru.Meta.Providers.md + items: + - name: Classes + - name: JsonMetadataProvider + href: Flowthru.Meta.Providers.JsonMetadataProvider.md + - name: MermaidMetadataProvider + href: Flowthru.Meta.Providers.MermaidMetadataProvider.md + - name: Interfaces + - name: IMetadataProvider + href: Flowthru.Meta.Providers.IMetadataProvider.md + - name: Enums + - name: MermaidMetadataProvider.MermaidFlowchartDirection + href: Flowthru.Meta.Providers.MermaidMetadataProvider.MermaidFlowchartDirection.md +- name: Flowthru.Registry + href: Flowthru.Registry.md + items: + - name: Interfaces + - name: IFlowRegistrar + href: Flowthru.Registry.IFlowRegistrar-1.md +- name: Flowthru.Results + href: Flowthru.Results.md + items: + - name: Classes + - name: ConsoleResultFormatter + href: Flowthru.Results.ConsoleResultFormatter.md + - name: Interfaces + - name: IFlowResultFormatter + href: Flowthru.Results.IFlowResultFormatter.md +- name: Flowthru.Services + href: Flowthru.Services.md + items: + - name: Classes + - name: FlowthruServiceBuilder + href: Flowthru.Services.FlowthruServiceBuilder.md + - name: Interfaces + - name: IFlowthruService + href: Flowthru.Services.IFlowthruService.md +- name: Flowthru.Services.Models + href: Flowthru.Services.Models.md + items: + - name: Classes + - name: FlowMetadata + href: Flowthru.Services.Models.FlowMetadata.md +- name: Flowthru.Steps + href: Flowthru.Steps.md + items: + - name: Classes + - name: NoData + href: Flowthru.Steps.NoData.md + - name: NoParams + href: Flowthru.Steps.NoParams.md +- name: Flowthru.Steps.Factory + href: Flowthru.Steps.Factory.md + items: + - name: Classes + - name: StepFactory + href: Flowthru.Steps.Factory.StepFactory.md + - name: TypeActivator + href: Flowthru.Steps.Factory.TypeActivator.md +- name: Microsoft.Extensions.DependencyInjection + href: Microsoft.Extensions.DependencyInjection.md + items: + - name: Classes + - name: FlowthruServiceCollectionExtensions + href: Microsoft.Extensions.DependencyInjection.FlowthruServiceCollectionExtensions.md diff --git a/docs/reference/src/core/Flowthru/toc.yml b/docs/reference/src/core/Flowthru/toc.yml new file mode 100644 index 00000000..20c32afa --- /dev/null +++ b/docs/reference/src/core/Flowthru/toc.yml @@ -0,0 +1,2 @@ +### YamlMime:TableOfContent +[] diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.CsvCatalogEntryExtensions.md b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.CsvCatalogEntryExtensions.md new file mode 100644 index 00000000..de53868b --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.CsvCatalogEntryExtensions.md @@ -0,0 +1,114 @@ +# Class CsvItemExtensions + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Extensions.Csv.dll + +Extension methods that add CSV support to . + +```csharp +public static class CsvItemExtensions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[CsvItemExtensions](Flowthru.Data.CsvItemExtensions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### Csv\(EnumerableItems, string, string\) + +Creates a CSV file catalog entry with IEnumerable container. + +```csharp +public static Item> Csv(this EnumerableItems _, string label, string filePath) where TRow : notnull, IFlatSchema, ITextSerializable +``` + +#### Parameters + +`_` EnumerableItems + +The enumerable catalog entries factory (from ) + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to CSV file + +#### Returns + + Item<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Catalog entry with file + CSV + IEnumerable composition + +#### Type Parameters + +`TRow` + +Row schema type (must be flat and text-serializable) + +#### Remarks + +

+Requirements: +

+
  • TRow must implement IFlatSchema (no nested objects)
  • TRow must implement ITextSerializable
+

+Storage Traits: +

+
  • CanStream: true (CSV supports row-by-row streaming)
  • All other traits use filesystem baseline defaults
+ +### CsvDirectory\(EnumerableItems, string, string\) + +Creates a catalog entry that reads all CSV files in a directory and +concatenates them into a single . + +```csharp +public static Item> CsvDirectory(this EnumerableItems _, string label, string directoryPath) where TRow : notnull, IFlatSchema, ITextSerializable +``` + +#### Parameters + +`_` EnumerableItems + +The enumerable catalog entries factory (from ) + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`directoryPath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to the directory containing the CSV files + +#### Returns + + Item<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Read-only catalog entry that concatenates every *.csv in the directory + +#### Type Parameters + +`TRow` + +Row schema type (must be flat and text-serializable) + +#### Remarks + +Files are read in lexicographic order. All files must share the same schema. +This entry is read-only — attempting to save will fail with +. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.DirectoryCsvStorageAdapter-1.md b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.DirectoryCsvStorageAdapter-1.md new file mode 100644 index 00000000..4ed7535f --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.DirectoryCsvStorageAdapter-1.md @@ -0,0 +1,263 @@ +# Class DirectoryCsvStorageAdapter + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Extensions.Csv.dll + +Storage adapter that reads all CSV files in a directory and concatenates +them into a single . + +```csharp +public sealed class DirectoryCsvStorageAdapter : IStorageAdapter> where TRow : notnull, IFlatSchema, ITextSerializable +``` + +#### Type Parameters + +`TRow` + +Row schema type (must be flat and text-serializable) + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[DirectoryCsvStorageAdapter](Flowthru.Data.Storage.DirectoryCsvStorageAdapter\-1.md) + +#### Implements + +IStorageAdapter\> + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This adapter is read-only. It enumerates every *.csv file in the +given directory in lexicographic order, deserialises each with a shared +, and returns all rows concatenated. +

+

+All files must share the same schema (identical column headers). Files from +mixed schemas will cause deserialization errors at load time. +

+

+Typical use case: a raw ingest layer where data is delivered as one file per +day, one file per region, etc. +

+ +## Constructors + +### DirectoryCsvStorageAdapter\(string\) + +Creates a new directory CSV adapter. + +```csharp +public DirectoryCsvStorageAdapter(string directoryPath) +``` + +#### Parameters + +`directoryPath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to the directory containing CSV files. + +#### Exceptions + + [ArgumentException](https://learn.microsoft.com/dotnet/api/system.argumentexception) + +Thrown if directoryPath is null or whitespace. + +## Properties + +### Traits + +Structural constraints and capabilities of this storage implementation. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + StorageTraits + +#### Remarks + +

+Adapter authors must declare what their storage can and cannot do. +These are intrinsic properties of the storage medium, not runtime state. +

+

+Pipeline validation uses these traits to fail fast when a pipeline attempts +invalid operations (e.g., writing to a read-only source). +

+ +## Methods + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + FlowIO<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+Delegates to the underlying medium's Exists check. +Used to determine if a catalog entry is a seed (Layer 0 input). +

+ +### InspectDeep\(\) + +Performs deep validation by examining the entire dataset. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + FlowIO + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that all data is available, accessible, and valid. +

+

+Additional Checks Beyond Shallow: +

+
  • Validate ALL rows can be deserialized (not just sample)
  • Check data quality constraints across entire dataset
  • Detect corruption or inconsistencies throughout data
+

+Implementation Guidelines: +

+
  • File adapters: Read and validate entire file
  • Memory adapters: Validate all stored data
  • Database adapters: Full table scan with validation
  • Null adapters: Always return success (no data required)
+

+Performance: Potentially expensive - only use when data integrity is critical. +

+ +### InspectShallow\(int\) + +Performs shallow validation by checking data availability and sampling a subset of data. + +```csharp +public FlowIO InspectShallow(int sampleSize) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + FlowIO + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that data is available and accessible. +

+

+Typical Checks: +

+
  • Data source exists (file, table, etc.)
  • Data source is accessible (permissions, connectivity)
  • Sample rows can be read and deserialized successfully
  • Schema matches expected structure
+

+Implementation Guidelines: +

+
  • File adapters: Check file exists, read and validate sample rows
  • Memory adapters: Check if data has been initialized
  • Database adapters: Check table exists, query sample rows
  • Null adapters: Always return success (no data required)
+

+Performance: Should be fast (~10-100ms) - suitable for pre-flight validation. +

+ +### Load\(\) + +Loads data from storage. + +```csharp +public FlowIO> Load() +``` + +#### Returns + + FlowIO<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Effect that produces data on success + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. medium.ReadStream()           → Stream
+2. format.DeserializeRows()      → IAsyncEnumerable<TRow>
+3. container.FromRows()          → TContainer
+

+Error Handling: +

+

+Errors from any layer are propagated: +- Medium errors (file not found, access denied) +- Format errors (parse failures, schema mismatches) +- Container errors (memory allocation, type conversion) +

+ +### Save\(IEnumerable\) + +Saves data to storage. + +```csharp +public FlowIO Save(IEnumerable data) +``` + +#### Parameters + +`data` [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1) + +The data to save + +#### Returns + + FlowIO + +Effect that completes on successful save + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. container.ToRows()            → IAsyncEnumerable<TRow>
+2. format.SerializeRows()        → Stream
+3. medium.WriteStream()          → FlowUnit
+

+Atomicity: +

+

+Implementations should strive for atomic saves to avoid partial writes on failure. +

+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Format.CsvFormatSerializer-1.md b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Format.CsvFormatSerializer-1.md new file mode 100644 index 00000000..dfb17aa2 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Format.CsvFormatSerializer-1.md @@ -0,0 +1,320 @@ +# Class CsvFormatSerializer + +Namespace: [Flowthru.Data.Storage.Format](Flowthru.Data.Storage.Format.md) +Assembly: Flowthru.Extensions.Csv.dll + +Format serializer for CSV (Comma-Separated Values) files. + +```csharp +public sealed class CsvFormatSerializer : IFormatSerializer where TRow : notnull, IFlatSchema, ITextSerializable +``` + +#### Type Parameters + +`TRow` + +The row schema type + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[CsvFormatSerializer](Flowthru.Data.Storage.Format.CsvFormatSerializer\-1.md) + +#### Implements + +IFormatSerializer + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
public record CompanySchema(
+    int Id,
+    string Name,
+    float Rating
+) : IFlatSchema, ITextSerializable;
+
+var serializer = new CsvFormatSerializer<CompanySchema>();
+
+// Deserialize
+using var readStream = File.OpenRead("companies.csv");
+await foreach (var row in serializer.DeserializeRows(readStream))
+{
+    Console.WriteLine($"Company: {row.Name}, Rating: {row.Rating}");
+}
+
+// Serialize
+var companies = new[] {
+    new CompanySchema(1, "Acme Corp", 4.5f),
+    new CompanySchema(2, "Tech Inc", 4.8f)
+};
+
+using var writeStream = File.Create("output.csv");
+await serializer.SerializeRows(writeStream, companies.ToAsyncEnumerable());
+ +## Remarks + +

+Type Constraints: +

+

+TRow must implement both: +

+
  • - No nested structures (primitives only)
  • - Can be serialized to text
+

+These constraints are enforced at compile-time, preventing invalid usage: +

+
// ✅ Compiles - flat schema with text serialization
+var csv = new CsvFormatSerializer<CompanySchema>();
+
+// ❌ Compile error - nested schema not allowed
+var csv = new CsvFormatSerializer<OrderWithItems>(); // OrderWithItems : INestedSchema
+

+Configuration: +

+

+Uses CsvHelper library with default configuration: +- HasHeaderRecord = true +- InvariantCulture +- Comma delimiter +

+

+Custom configuration can be provided via constructor. +

+

+Streaming Behavior: +

+

+Both deserialization and serialization use streaming: +- Rows are yielded/consumed lazily +- Low memory footprint for large files +- Backpressure support via IAsyncEnumerable +

+ +## Constructors + +### CsvFormatSerializer\(\) + +Creates a new CSV format serializer with default configuration. + +```csharp +public CsvFormatSerializer() +``` + +### CsvFormatSerializer\(CsvConfiguration\) + +Creates a new CSV format serializer with custom configuration. + +```csharp +public CsvFormatSerializer(CsvConfiguration configuration) +``` + +#### Parameters + +`configuration` CsvConfiguration + +CsvHelper configuration + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +Thrown if configuration is null + +## Properties + +### Configuration + +Gets the CSV configuration for this serializer. + +```csharp +public CsvConfiguration Configuration { get; } +``` + +#### Property Value + + CsvConfiguration + +### Traits + +Structural capabilities of this format serializer. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + StorageTraits + +#### Remarks + +

+Format traits focus on HOW data is serialized and whether it supports streaming. +For composed adapters, these traits are merged with medium and container traits. +

+

+Most formats should declare CanStream = true if they can deserialize row-by-row +without buffering the entire stream (e.g., CSV, JSONL). Formats that require full +parsing before yielding rows (e.g., JSON arrays) should set CanStream = false. +

+ +## Methods + +### DeserializeRows\(Stream\) + +Deserializes a stream of bytes into a stream of rows. + +```csharp +public IAsyncEnumerable DeserializeRows(Stream stream) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +The stream containing serialized data + +#### Returns + + [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +Async enumerable of deserialized rows + +#### Remarks + +

+Streaming Behavior: +

+

+Rows should be yielded as they are deserialized (lazy evaluation). +This allows processing large datasets without loading everything into memory. +

+

+Error Handling: +

+

+Deserialization errors should throw exceptions: +- Format exceptions (malformed CSV, invalid JSON) +- Schema mismatches (missing columns, type conversion failures) +- I/O errors during stream reading +

+

+The caller should handle these exceptions appropriately. +

+ +### GetPropertyMappingConfiguration\(\) + +Configures how this serializer handles property-to-field name mapping for the schema. + +```csharp +public PropertyMappingConfiguration GetPropertyMappingConfiguration() +``` + +#### Returns + + PropertyMappingConfiguration + +Property mapping configuration describing the mapping strategy + +#### Examples + +
// CSV serializer using SerializedLabel
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.FromSerializedLabel<TRow>();
+
+// ML.NET serializer using native attributes
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.FromNativeAttributes("LoadColumnAttribute");
+
+// Parquet with library-controlled mapping
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.LibraryControlled();
+ +#### Remarks + +

+Contractual Obligation: Every format serializer MUST implement this method +to explicitly declare how it handles property name mapping. +

+

+Implementation Strategies: +

+
  • +SerializedLabel Support: Use +to respect [SerializedLabel] attributes. Return . +
  • +Native Attribute Mapping: Use format-specific attributes (e.g., ML.NET's [LoadColumn]). +Return with the attribute type name. +
  • +Library-Controlled: When the underlying library handles mapping internally +(e.g., Parquet.NET with no programmatic API). Return PropertyMappingConfiguration.LibraryControlled(). +
  • +Adapter Pattern: Bridge between SerializedLabel and native attributes. +Return . +
+

+Design Intent: This contract makes property mapping an explicit, discoverable +capability rather than an implicit behavior, enabling: +

+
  • Runtime introspection of mapping capabilities
  • Better error messages when schemas don't match storage
  • Documentation generation for serializer capabilities
  • Testing framework validation of mapping correctness
+ +### SerializeRows\(Stream, IAsyncEnumerable\) + +Serializes a stream of rows into a stream of bytes. + +```csharp +public Task SerializeRows(Stream stream, IAsyncEnumerable rows) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +The stream to write serialized data to + +`rows` [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +The rows to serialize + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task) + +Task that completes when serialization finishes + +#### Remarks + +

+Streaming Behavior: +

+

+Rows should be written as they are enumerated (lazy evaluation). +This allows handling large datasets efficiently. +

+

+Format-Specific Headers: +

+

+Implementations should handle format-specific initialization: +- CSV: Write header row with column names +- JSON: Write opening bracket for array +- Parquet: Write schema metadata +

+

+Error Handling: +

+

+Serialization errors should throw exceptions: +- Type conversion failures +- I/O errors during stream writing +- Invalid data values for format constraints +

+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Format.md b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Format.md new file mode 100644 index 00000000..dc4c870a --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Format.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Data.Storage.Format + +### Classes + + [CsvFormatSerializer](Flowthru.Data.Storage.Format.CsvFormatSerializer\-1.md) + +Format serializer for CSV (Comma-Separated Values) files. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Strategies.CsvStorageEntryFactory.md b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Strategies.CsvStorageEntryFactory.md new file mode 100644 index 00000000..55677218 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Strategies.CsvStorageEntryFactory.md @@ -0,0 +1,160 @@ +# Class CsvStorageEntryFactory + +Namespace: [Flowthru.Data.Storage.Strategies](Flowthru.Data.Storage.Strategies.md) +Assembly: Flowthru.Extensions.Csv.dll + +CSV file-based storage strategy for local development. + +```csharp +public sealed class CsvStorageEntryFactory : IStorageEntryFactory +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[CsvStorageEntryFactory](Flowthru.Data.Storage.Strategies.CsvStorageEntryFactory.md) + +#### Implements + +IStorageEntryFactory + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Uses CSV files for data storage, enabling: +

+
  • Easy inspection with text editors or spreadsheet tools
  • Version control-friendly (human-readable diffs)
  • No external dependencies (no database required)
+

+Path Resolution: +

+
// With explicit path
+factory.CreateEnumerable<Company>("Companies",
+    StorageOptions.WithPath("_01_Raw/data.csv"))
+// → {BasePath}/_01_Raw/data.csv
+
+// With default path (label-based)
+factory.CreateEnumerable<Company>("Companies")
+// → {BasePath}/Companies.csv
+ +## Constructors + +### CsvStorageEntryFactory\(IConfiguration\) + +Initializes a new CSV storage factory. + +```csharp +public CsvStorageEntryFactory(IConfiguration configuration) +``` + +#### Parameters + +`configuration` [IConfiguration](https://learn.microsoft.com/dotnet/api/microsoft.extensions.configuration.iconfiguration) + +Configuration containing optional DataPath setting + +### CsvStorageEntryFactory\(string\) + +Initializes a new CSV storage factory with explicit base path. + +```csharp +public CsvStorageEntryFactory(string basePath) +``` + +#### Parameters + +`basePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Base directory for all CSV files + +## Methods + +### CreateEnumerable\(string, StorageOptions?\) + +Creates a catalog entry for an enumerable dataset. + +```csharp +public IItem> CreateEnumerable(string label, StorageOptions? options = null) where T : notnull, IFlatSchema, ITextSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Catalog label for the entry + +`options` StorageOptions? + +Optional storage options + +#### Returns + + IItem<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Configured catalog entry + +#### Type Parameters + +`T` + +Schema type (must implement IFlatSchema and ITextSerializable) + +#### Remarks + +

+Type constraints ensure compatibility with CSV and Parquet serialization. +Memory storage also works since it has no serialization requirements. +

+

+If options.Path is null, the label is used to derive a default path +(e.g., "Companies" → "Companies.csv" or "dbo.Companies"). +

+ +### CreateSingle\(string, StorageOptions?\) + +Creates a catalog entry for a singleton object. + +```csharp +public IItem CreateSingle(string label, StorageOptions? options = null) where T : IStructuredSerializable +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Catalog label for the entry + +`options` StorageOptions? + +Optional storage options + +#### Returns + + IItem + +Configured catalog entry + +#### Type Parameters + +`T` + +Object type (must implement IStructuredSerializable) + +#### Remarks + +

+Type constraint ensures compatibility with JSON serialization. +Memory storage also works since it has no serialization requirements. +

+

+Typically uses structured formats (JSON, MessagePack) for singletons. +

+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Strategies.md b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Strategies.md new file mode 100644 index 00000000..b00cd4ee --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.Strategies.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Data.Storage.Strategies + +### Classes + + [CsvStorageEntryFactory](Flowthru.Data.Storage.Strategies.CsvStorageEntryFactory.md) + +CSV file-based storage strategy for local development. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.md b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.md new file mode 100644 index 00000000..269f0a6f --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.Storage.md @@ -0,0 +1,15 @@ +# Namespace Flowthru.Data.Storage + +### Namespaces + + [Flowthru.Data.Storage.Format](Flowthru.Data.Storage.Format.md) + + [Flowthru.Data.Storage.Strategies](Flowthru.Data.Storage.Strategies.md) + +### Classes + + [DirectoryCsvStorageAdapter](Flowthru.Data.Storage.DirectoryCsvStorageAdapter\-1.md) + +Storage adapter that reads all CSV files in a directory and concatenates +them into a single . + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.md b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.md new file mode 100644 index 00000000..9ee4827b --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Csv/Flowthru.Data.md @@ -0,0 +1,12 @@ +# Namespace Flowthru.Data + +### Namespaces + + [Flowthru.Data.Storage](Flowthru.Data.Storage.md) + +### Classes + + [CsvItemExtensions](Flowthru.Data.CsvItemExtensions.md) + +Extension methods that add CSV support to . + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Csv/toc.yml b/docs/reference/src/extensions/Flowthru.Extensions.Csv/toc.yml new file mode 100644 index 00000000..a2d55e2e --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Csv/toc.yml @@ -0,0 +1,25 @@ +### YamlMime:TableOfContent +- name: Flowthru.Data + href: Flowthru.Data.md + items: + - name: Classes + - name: CsvItemExtensions + href: Flowthru.Data.CsvItemExtensions.md +- name: Flowthru.Data.Storage + href: Flowthru.Data.Storage.md + items: + - name: Classes + - name: DirectoryCsvStorageAdapter + href: Flowthru.Data.Storage.DirectoryCsvStorageAdapter-1.md +- name: Flowthru.Data.Storage.Format + href: Flowthru.Data.Storage.Format.md + items: + - name: Classes + - name: CsvFormatSerializer + href: Flowthru.Data.Storage.Format.CsvFormatSerializer-1.md +- name: Flowthru.Data.Storage.Strategies + href: Flowthru.Data.Storage.Strategies.md + items: + - name: Classes + - name: CsvStorageEntryFactory + href: Flowthru.Data.Storage.Strategies.CsvStorageEntryFactory.md diff --git a/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Data.Storage.EFCoreSingleStorageAdapter-1.md b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Data.Storage.EFCoreSingleStorageAdapter-1.md new file mode 100644 index 00000000..222b90c6 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Data.Storage.EFCoreSingleStorageAdapter-1.md @@ -0,0 +1,315 @@ +# Class EFCoreSingleStorageAdapter + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Extensions.EFCore.dll + +Storage adapter for single Entity Framework Core entities. +Stores exactly one row in a database table. + +```csharp +public sealed class EFCoreSingleStorageAdapter : IStorageAdapter where T : class +``` + +#### Type Parameters + +`T` + +Entity type (must be a class type configured as EF entity) + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EFCoreSingleStorageAdapter](Flowthru.Data.Storage.EFCoreSingleStorageAdapter\-1.md) + +#### Implements + +IStorageAdapter + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Save Semantics: Replace - removes all existing rows and inserts the new entity. +Ensures table contains exactly one row after save. +

+

+Load Semantics: Returns the single row from the table. +Throws if table contains zero or more than one row. +

+

+Exists Semantics: Returns true if table has exactly one row. +

+ +## Constructors + +### EFCoreSingleStorageAdapter\(DbContext, bool, bool, Func, IQueryable\>?, Func?\) + +Creates an adapter with an injected DbContext instance. + +```csharp +public EFCoreSingleStorageAdapter(DbContext context, bool ownsContext, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func? saveFunc = null) +``` + +#### Parameters + +`context` [DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext) + +DbContext to use for operations + +`ownsContext` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +If true, adapter disposes context after operations + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext), T, [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task)\>? + +#### Remarks + +To create a read-only catalog entry, use .Constrain(traits => traits with { CanWrite = false }) +on the catalog entry after construction. + +### EFCoreSingleStorageAdapter\(Func, bool, Func, IQueryable\>?, Func?\) + +Creates an adapter with a DbContext factory. + +```csharp +public EFCoreSingleStorageAdapter(Func contextFactory, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func? saveFunc = null) +``` + +#### Parameters + +`contextFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-1)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext)\> + +Factory to create DbContext instances + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext), T, [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task)\>? + +#### Remarks + +To create a read-only catalog entry, use .Constrain(traits => traits with { CanWrite = false }) +on the catalog entry after construction. + +## Properties + +### Traits + +Structural constraints and capabilities of this storage implementation. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + StorageTraits + +#### Remarks + +

+Adapter authors must declare what their storage can and cannot do. +These are intrinsic properties of the storage medium, not runtime state. +

+

+Pipeline validation uses these traits to fail fast when a pipeline attempts +invalid operations (e.g., writing to a read-only source). +

+ +## Methods + +### DefaultSave\(DbContext, T, CancellationToken\) + +Gets a DbContext from either the injected instance or factory. + +```csharp +public static Task DefaultSave(DbContext context, T data, CancellationToken ct) +``` + +#### Parameters + +`context` [DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext) + +`data` T + +`ct` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task) + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + FlowIO<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+Delegates to the underlying medium's Exists check. +Used to determine if a catalog entry is a seed (Layer 0 input). +

+ +### InspectDeep\(\) + +Performs deep validation by examining the entire dataset. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + FlowIO + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that all data is available, accessible, and valid. +

+

+Additional Checks Beyond Shallow: +

+
  • Validate ALL rows can be deserialized (not just sample)
  • Check data quality constraints across entire dataset
  • Detect corruption or inconsistencies throughout data
+

+Implementation Guidelines: +

+
  • File adapters: Read and validate entire file
  • Memory adapters: Validate all stored data
  • Database adapters: Full table scan with validation
  • Null adapters: Always return success (no data required)
+

+Performance: Potentially expensive - only use when data integrity is critical. +

+ +### InspectShallow\(int\) + +Performs shallow validation by checking data availability and sampling a subset of data. + +```csharp +public FlowIO InspectShallow(int sampleSize) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + FlowIO + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that data is available and accessible. +

+

+Typical Checks: +

+
  • Data source exists (file, table, etc.)
  • Data source is accessible (permissions, connectivity)
  • Sample rows can be read and deserialized successfully
  • Schema matches expected structure
+

+Implementation Guidelines: +

+
  • File adapters: Check file exists, read and validate sample rows
  • Memory adapters: Check if data has been initialized
  • Database adapters: Check table exists, query sample rows
  • Null adapters: Always return success (no data required)
+

+Performance: Should be fast (~10-100ms) - suitable for pre-flight validation. +

+ +### Load\(\) + +Loads data from storage. + +```csharp +public FlowIO Load() +``` + +#### Returns + + FlowIO + +Effect that produces data on success + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. medium.ReadStream()           → Stream
+2. format.DeserializeRows()      → IAsyncEnumerable<TRow>
+3. container.FromRows()          → TContainer
+

+Error Handling: +

+

+Errors from any layer are propagated: +- Medium errors (file not found, access denied) +- Format errors (parse failures, schema mismatches) +- Container errors (memory allocation, type conversion) +

+ +### Save\(T\) + +Saves data to storage. + +```csharp +public FlowIO Save(T data) +``` + +#### Parameters + +`data` T + +The data to save + +#### Returns + + FlowIO + +Effect that completes on successful save + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. container.ToRows()            → IAsyncEnumerable<TRow>
+2. format.SerializeRows()        → Stream
+3. medium.WriteStream()          → FlowUnit
+

+Atomicity: +

+

+Implementations should strive for atomic saves to avoid partial writes on failure. +

+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Data.Storage.EFCoreStorageAdapter-1.md b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Data.Storage.EFCoreStorageAdapter-1.md new file mode 100644 index 00000000..6aa28fd8 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Data.Storage.EFCoreStorageAdapter-1.md @@ -0,0 +1,298 @@ +# Class EFCoreStorageAdapter + +Namespace: [Flowthru.Data.Storage](Flowthru.Data.Storage.md) +Assembly: Flowthru.Extensions.EFCore.dll + +```csharp +public sealed class EFCoreStorageAdapter : IStorageAdapter> where T : class +``` + +#### Type Parameters + +`T` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EFCoreStorageAdapter](Flowthru.Data.Storage.EFCoreStorageAdapter\-1.md) + +#### Implements + +IStorageAdapter\> + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Constructors + +### EFCoreStorageAdapter\(DbContext, bool, Func, IQueryable\>?, Func, CancellationToken, Task\>?\) + +Creates an adapter with an injected DbContext. + +```csharp +public EFCoreStorageAdapter(DbContext context, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func, CancellationToken, Task>? saveFunc = null) +``` + +#### Parameters + +`context` [DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext) + +DbContext instance (caller owns lifecycle) + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +If true, empty tables are considered valid during validation + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext), [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1), [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task)\>? + +#### Remarks + +To create a read-only catalog entry, use .Constrain(traits => traits with { CanWrite = false }) +on the catalog entry after construction. + +### EFCoreStorageAdapter\(Func, bool, Func, IQueryable\>?, Func, CancellationToken, Task\>?\) + +Creates an adapter with a DbContext factory. + +```csharp +public EFCoreStorageAdapter(Func contextFactory, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func, CancellationToken, Task>? saveFunc = null) +``` + +#### Parameters + +`contextFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-1)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext)\> + +Factory function to create DbContext instances + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +If true, empty tables are considered valid during validation + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext), [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1), [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task)\>? + +#### Remarks + +To create a read-only catalog entry, use .Constrain(traits => traits with { CanWrite = false }) +on the catalog entry after construction. + +## Properties + +### Traits + +Structural constraints and capabilities of this storage implementation. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + StorageTraits + +#### Remarks + +

+Adapter authors must declare what their storage can and cannot do. +These are intrinsic properties of the storage medium, not runtime state. +

+

+Pipeline validation uses these traits to fail fast when a pipeline attempts +invalid operations (e.g., writing to a read-only source). +

+ +## Methods + +### DefaultSave\(DbContext, IEnumerable, CancellationToken\) + +Default save strategy: replaces all rows with the new data. +Reference this explicitly when composing with a custom save delegate +(e.g., "use default load but custom save"). + +```csharp +public static Task DefaultSave(DbContext context, IEnumerable data, CancellationToken ct) +``` + +#### Parameters + +`context` [DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext) + +`data` [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1) + +`ct` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task) + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + FlowIO<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+Delegates to the underlying medium's Exists check. +Used to determine if a catalog entry is a seed (Layer 0 input). +

+ +### InspectDeep\(\) + +Performs deep validation by examining the entire dataset. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + FlowIO + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that all data is available, accessible, and valid. +

+

+Additional Checks Beyond Shallow: +

+
  • Validate ALL rows can be deserialized (not just sample)
  • Check data quality constraints across entire dataset
  • Detect corruption or inconsistencies throughout data
+

+Implementation Guidelines: +

+
  • File adapters: Read and validate entire file
  • Memory adapters: Validate all stored data
  • Database adapters: Full table scan with validation
  • Null adapters: Always return success (no data required)
+

+Performance: Potentially expensive - only use when data integrity is critical. +

+ +### InspectShallow\(int\) + +Performs shallow validation by checking data availability and sampling a subset of data. + +```csharp +public FlowIO InspectShallow(int sampleSize) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + FlowIO + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that data is available and accessible. +

+

+Typical Checks: +

+
  • Data source exists (file, table, etc.)
  • Data source is accessible (permissions, connectivity)
  • Sample rows can be read and deserialized successfully
  • Schema matches expected structure
+

+Implementation Guidelines: +

+
  • File adapters: Check file exists, read and validate sample rows
  • Memory adapters: Check if data has been initialized
  • Database adapters: Check table exists, query sample rows
  • Null adapters: Always return success (no data required)
+

+Performance: Should be fast (~10-100ms) - suitable for pre-flight validation. +

+ +### Load\(\) + +Loads data from storage. + +```csharp +public FlowIO> Load() +``` + +#### Returns + + FlowIO<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Effect that produces data on success + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. medium.ReadStream()           → Stream
+2. format.DeserializeRows()      → IAsyncEnumerable<TRow>
+3. container.FromRows()          → TContainer
+

+Error Handling: +

+

+Errors from any layer are propagated: +- Medium errors (file not found, access denied) +- Format errors (parse failures, schema mismatches) +- Container errors (memory allocation, type conversion) +

+ +### Save\(IEnumerable\) + +Saves data to storage. + +```csharp +public FlowIO Save(IEnumerable data) +``` + +#### Parameters + +`data` [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1) + +The data to save + +#### Returns + + FlowIO + +Effect that completes on successful save + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. container.ToRows()            → IAsyncEnumerable<TRow>
+2. format.SerializeRows()        → Stream
+3. medium.WriteStream()          → FlowUnit
+

+Atomicity: +

+

+Implementations should strive for atomic saves to avoid partial writes on failure. +

+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Data.Storage.md b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Data.Storage.md new file mode 100644 index 00000000..5f263a5d --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Data.Storage.md @@ -0,0 +1,11 @@ +# Namespace Flowthru.Data.Storage + +### Classes + + [EFCoreSingleStorageAdapter](Flowthru.Data.Storage.EFCoreSingleStorageAdapter\-1.md) + +Storage adapter for single Entity Framework Core entities. +Stores exactly one row in a database table. + + [EFCoreStorageAdapter](Flowthru.Data.Storage.EFCoreStorageAdapter\-1.md) + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.EFCoreCatalogEntries.Enumerable.md b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.EFCoreCatalogEntries.Enumerable.md new file mode 100644 index 00000000..4407c94c --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.EFCoreCatalogEntries.Enumerable.md @@ -0,0 +1,264 @@ +# Class EFCoreItems.Enumerable + +Namespace: [Flowthru.Extensions.EFCore.Data](Flowthru.Extensions.EFCore.Data.md) +Assembly: Flowthru.Extensions.EFCore.dll + +```csharp +public static class EFCoreItems.Enumerable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EFCoreItems.Enumerable](Flowthru.Extensions.EFCore.Data.EFCoreItems.Enumerable.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### EFCore\(string, DbContext, bool, Func, IQueryable\>?, Func, CancellationToken, Task\>?\) + +Creates an Entity Framework Core catalog entry for database-backed collections. + +```csharp +public static Item> EFCore(string label, DbContext context, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func, CancellationToken, Task>? saveFunc = null) where T : class +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`context` [DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext) + +DbContext instance (caller owns lifecycle) + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +If true, empty tables pass validation (default: false) + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext), [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1), [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task)\>? + +#### Returns + + Item<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Catalog entry for EFCore database storage + +#### Type Parameters + +`T` + +Entity type (must be a class configured in DbContext) + +#### Examples + +
// In catalog
+public static partial class DataCatalog
+{
+  public static IItem<IEnumerable<Company>> Companies(DbContext db) =>
+    Items.Enumerable.EFCore<Company>("companies", db);
+}
+
+// In pipeline
+var pipeline = new FlowBuilder("CompanyPipeline")
+  .AddStep("load_companies", catalog => new LoadCompaniesNode(
+    outputs: catalog.Companies(db)
+  ))
+  .Build();
+ +#### Remarks + +

+Use Case: Read/write entities from relational databases using EF Core +

+

+DbContext Lifecycle: Caller provides DbContext and manages its lifecycle. +Use this overload when DbContext comes from DI container or is shared across operations. +

+

+Read-Only Entries: +To create a read-only catalog entry, apply a constraint: +.Constrain(traits => traits with { CanWrite = false }) +

+

+Empty Data Validation: +By default (allowEmptyData: false), empty tables fail pre-flight validation. +Set allowEmptyData: true for tables that may legitimately be empty. +

+ +### EFCore\(string, Func, bool, Func, IQueryable\>?, Func, CancellationToken, Task\>?\) + +Creates an Entity Framework Core catalog entry with a DbContext factory. + +```csharp +public static Item> EFCore(string label, Func contextFactory, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func, CancellationToken, Task>? saveFunc = null) where T : class +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`contextFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-1)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext)\> + +Factory function to create DbContext instances per operation + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +If true, empty tables pass validation (default: false) + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext), [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1), [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task)\>? + +#### Returns + + Item<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Catalog entry for EFCore database storage + +#### Type Parameters + +`T` + +Entity type (must be a class configured in DbContext) + +#### Examples + +
// In catalog
+public static partial class DataCatalog
+{
+  private static AppDbContext CreateDbContext() =>
+    new AppDbContext(new DbContextOptionsBuilder<AppDbContext>()
+      .UseSqlServer(connectionString)
+      .Options);
+
+  public static IItem<IEnumerable<Company>> Companies() =>
+    Items.Enumerable.EFCore<Company>("companies", CreateDbContext);
+}
+ +#### Remarks + +

+Use Case: When DbContext should be created fresh for each Load/Save operation +

+

+DbContext Lifecycle: Adapter creates DbContext via factory and disposes it +after each operation. Use this overload for scoped DbContext patterns. +

+

+Read-Only Entries: +To create a read-only catalog entry, apply a constraint: +.Constrain(traits => traits with { CanWrite = false }) +

+ +### EFCore\(string, Func, bool, Func, IQueryable\>?, Func, CancellationToken, Task\>?\) + +Creates an EFCore catalog entry with a typed DbContext factory. +The concrete TContext flows through to the save delegate, +eliminating any downcast inside the delegate body. + +```csharp +public static Item> EFCore(string label, Func contextFactory, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func, CancellationToken, Task>? saveFunc = null) where T : class where TContext : DbContext +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`contextFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-1) + +Typed factory; called per Load/Save operation + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +If true, empty tables pass validation (default: false) + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +Optional query transformation applied before materialization (e.g. Include, Where, OrderBy) + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4), [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task)\>? + +Optional save delegate receiving the concrete TContext. + Defaults to RemoveRange + AddRange when null. + +#### Returns + + Item<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Catalog entry for EFCore database storage + +#### Type Parameters + +`T` + +Entity type + +`TContext` + +Concrete DbContext type + +### EFCore\(string, IDbContextFactory, bool, Func, IQueryable\>?, Func, CancellationToken, Task\>?\) + +Creates an EFCore catalog entry using — +the idiomatic EFCore pattern for per-operation context isolation and concurrent node safety. + +```csharp +public static Item> EFCore(string label, IDbContextFactory contextFactory, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func, CancellationToken, Task>? saveFunc = null) where T : class where TContext : DbContext +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`contextFactory` [IDbContextFactory](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.idbcontextfactory\-1) + +EFCore context factory; a fresh context is created per Load/Save operation + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +If true, empty tables pass validation (default: false) + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +Optional query transformation applied before materialization + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4), [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task)\>? + +Optional save delegate receiving the concrete TContext. + Defaults to RemoveRange + AddRange when null. + +#### Returns + + Item<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Catalog entry for EFCore database storage + +#### Type Parameters + +`T` + +Entity type + +`TContext` + +Concrete DbContext type + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.EFCoreCatalogEntries.Single.md b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.EFCoreCatalogEntries.Single.md new file mode 100644 index 00000000..8f936d33 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.EFCoreCatalogEntries.Single.md @@ -0,0 +1,252 @@ +# Class EFCoreItems.Single + +Namespace: [Flowthru.Extensions.EFCore.Data](Flowthru.Extensions.EFCore.Data.md) +Assembly: Flowthru.Extensions.EFCore.dll + +```csharp +public static class EFCoreItems.Single +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EFCoreItems.Single](Flowthru.Extensions.EFCore.Data.EFCoreItems.Single.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### EFCore\(string, DbContext, bool, Func, IQueryable\>?, Func?\) + +Creates an Entity Framework Core catalog entry for single database-backed entities. + +```csharp +public static Item EFCore(string label, DbContext context, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func? saveFunc = null) where T : class +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`context` [DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext) + +DbContext instance (caller owns lifecycle) + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext), T, [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task)\>? + +#### Returns + + Item + +Catalog entry for EFCore single entity storage + +#### Type Parameters + +`T` + +Entity type (must be a class configured in DbContext) + +#### Examples + +
// In catalog
+public IItem<ModelMetrics> Metrics(DbContext db) =>
+  Items.Single.EFCore<ModelMetrics>("metrics", db);
+
+// In pipeline
+var pipeline = new FlowBuilder("MetricsPipeline")
+  .AddStep("save_metrics", catalog => new SaveMetricsNode(
+    outputs: catalog.Metrics(db)
+  ))
+  .Build();
+ +#### Remarks + +

+Use Case: Store single entities (models, metrics, configs) in database +

+

+Implementation: Stores entity in a table, expects exactly one row on Load. +Save replaces the single row (clear table, insert new row). +

+

+DbContext Lifecycle: Caller provides DbContext and manages its lifecycle. +Use this overload when DbContext comes from DI container or is shared across operations. +

+

+Read-Only Entries: +To create a read-only catalog entry, apply a constraint: +.Constrain(traits => traits with { CanWrite = false }) +

+ +### EFCore\(string, Func, bool, Func, IQueryable\>?, Func?\) + +Creates an Entity Framework Core catalog entry for single database-backed entities using a factory. + +```csharp +public static Item EFCore(string label, Func contextFactory, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func? saveFunc = null) where T : class +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`contextFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-1)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext)\> + +Factory that creates DbContext instances (adapter owns lifecycle) + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4)<[DbContext](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.dbcontext), T, [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken), [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task)\>? + +#### Returns + + Item + +Catalog entry for EFCore single entity storage + +#### Type Parameters + +`T` + +Entity type (must be a class configured in DbContext) + +#### Examples + +
// In catalog with factory
+private readonly IServiceProvider _serviceProvider;
+
+public IItem<ModelMetrics> Metrics =>
+  Items.Single.EFCore<ModelMetrics>(
+    "metrics",
+    () => _serviceProvider.GetRequiredService<MyDbContext>()
+  );
+ +#### Remarks + +

+Use Case: Store single entities when you want adapter to manage DbContext lifecycle +

+

+DbContext Lifecycle: Adapter creates and disposes DbContext per operation. +Use this overload when operations should be isolated or when DbContext is expensive to keep alive. +

+

+Read-Only Entries: +To create a read-only catalog entry, apply a constraint: +.Constrain(traits => traits with { CanWrite = false }) +

+ +### EFCore\(string, Func, bool, Func, IQueryable\>?, Func?\) + +Creates a single-entity EFCore catalog entry with a typed DbContext factory. +The concrete TContext flows through to the save delegate, +eliminating any downcast inside the delegate body. + +```csharp +public static Item EFCore(string label, Func contextFactory, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func? saveFunc = null) where T : class where TContext : DbContext +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`contextFactory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-1) + +Typed factory; called per Load/Save operation + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +If true, an empty table passes validation (default: false) + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +Optional query transformation applied before SingleAsync + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4)? + +Optional save delegate receiving the concrete TContext. + Defaults to clear-and-insert when null. + +#### Returns + + Item + +Catalog entry for EFCore single entity storage + +#### Type Parameters + +`T` + +Entity type + +`TContext` + +Concrete DbContext type + +### EFCore\(string, IDbContextFactory, bool, Func, IQueryable\>?, Func?\) + +Creates a single-entity EFCore catalog entry using . + +```csharp +public static Item EFCore(string label, IDbContextFactory contextFactory, bool allowEmptyData = false, Func, IQueryable>? queryCustomizer = null, Func? saveFunc = null) where T : class where TContext : DbContext +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`contextFactory` [IDbContextFactory](https://learn.microsoft.com/dotnet/api/microsoft.entityframeworkcore.idbcontextfactory\-1) + +EFCore context factory; a fresh context is created per Load/Save operation + +`allowEmptyData` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +If true, an empty table passes validation (default: false) + +`queryCustomizer` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1), [IQueryable](https://learn.microsoft.com/dotnet/api/system.linq.iqueryable\-1)\>? + +Optional query transformation applied before SingleAsync + +`saveFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-4)? + +Optional save delegate receiving the concrete TContext. + Defaults to clear-and-insert when null. + +#### Returns + + Item + +Catalog entry for EFCore single entity storage + +#### Type Parameters + +`T` + +Entity type + +`TContext` + +Concrete DbContext type + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.EFCoreCatalogEntries.md b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.EFCoreCatalogEntries.md new file mode 100644 index 00000000..0190dc48 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.EFCoreCatalogEntries.md @@ -0,0 +1,24 @@ +# Class EFCoreItems + +Namespace: [Flowthru.Extensions.EFCore.Data](Flowthru.Extensions.EFCore.Data.md) +Assembly: Flowthru.Extensions.EFCore.dll + +```csharp +public static class EFCoreItems +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EFCoreItems](Flowthru.Extensions.EFCore.Data.EFCoreItems.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.md b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.md new file mode 100644 index 00000000..e16e5a3c --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Data.md @@ -0,0 +1,10 @@ +# Namespace Flowthru.Extensions.EFCore.Data + +### Classes + + [EFCoreItems](Flowthru.Extensions.EFCore.Data.EFCoreItems.md) + + [EFCoreItems.Enumerable](Flowthru.Extensions.EFCore.Data.EFCoreItems.Enumerable.md) + + [EFCoreItems.Single](Flowthru.Extensions.EFCore.Data.EFCoreItems.Single.md) + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Tests.CompilationTest.md b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Tests.CompilationTest.md new file mode 100644 index 00000000..5fac47d1 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Tests.CompilationTest.md @@ -0,0 +1,52 @@ +# Class CompilationTest + +Namespace: [Flowthru.Extensions.EFCore.Tests](Flowthru.Extensions.EFCore.Tests.md) +Assembly: Flowthru.Extensions.EFCore.dll + +Minimal compilation test to verify extension pattern works. + +```csharp +public class CompilationTest +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[CompilationTest](Flowthru.Extensions.EFCore.Tests.CompilationTest.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### DbContextFactoryOverloadsWork\(\) + +```csharp +public void DbContextFactoryOverloadsWork() +``` + +### PartialClassExtensionWorks\(\) + +```csharp +public void PartialClassExtensionWorks() +``` + +### SingleTypedContextOverloadsWork\(\) + +```csharp +public void SingleTypedContextOverloadsWork() +``` + +### TypedContextFactoryOverloadsWork\(\) + +```csharp +public void TypedContextFactoryOverloadsWork() +``` + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Tests.md b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Tests.md new file mode 100644 index 00000000..d6273dc9 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/Flowthru.Extensions.EFCore.Tests.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Extensions.EFCore.Tests + +### Classes + + [CompilationTest](Flowthru.Extensions.EFCore.Tests.CompilationTest.md) + +Minimal compilation test to verify extension pattern works. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.EFCore/toc.yml b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/toc.yml new file mode 100644 index 00000000..82336d47 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.EFCore/toc.yml @@ -0,0 +1,25 @@ +### YamlMime:TableOfContent +- name: Flowthru.Data.Storage + href: Flowthru.Data.Storage.md + items: + - name: Classes + - name: EFCoreSingleStorageAdapter + href: Flowthru.Data.Storage.EFCoreSingleStorageAdapter-1.md + - name: EFCoreStorageAdapter + href: Flowthru.Data.Storage.EFCoreStorageAdapter-1.md +- name: Flowthru.Extensions.EFCore.Data + href: Flowthru.Extensions.EFCore.Data.md + items: + - name: Classes + - name: EFCoreItems + href: Flowthru.Extensions.EFCore.Data.EFCoreItems.md + - name: EFCoreItems.Enumerable + href: Flowthru.Extensions.EFCore.Data.EFCoreItems.Enumerable.md + - name: EFCoreItems.Single + href: Flowthru.Extensions.EFCore.Data.EFCoreItems.Single.md +- name: Flowthru.Extensions.EFCore.Tests + href: Flowthru.Extensions.EFCore.Tests.md + items: + - name: Classes + - name: CompilationTest + href: Flowthru.Extensions.EFCore.Tests.CompilationTest.md diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.ExcelCatalogEntryExtensions.md b/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.ExcelCatalogEntryExtensions.md new file mode 100644 index 00000000..91616a06 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.ExcelCatalogEntryExtensions.md @@ -0,0 +1,81 @@ +# Class ExcelItemExtensions + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Extensions.Excel.dll + +Extension methods that add Excel support to . + +```csharp +public static class ExcelItemExtensions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ExcelItemExtensions](Flowthru.Data.ExcelItemExtensions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### Excel\(EnumerableItems, string, string, string\) + +Creates a read-only Excel file catalog entry with IEnumerable container. + +```csharp +public static Item> Excel(this EnumerableItems _, string label, string filePath, string sheetName) where TRow : notnull, IFlatSchema, ITextSerializable +``` + +#### Parameters + +`_` EnumerableItems + +The enumerable catalog entries factory (from ) + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to Excel file (.xlsx) + +`sheetName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Name of the sheet to read + +#### Returns + + Item<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Catalog entry with read-only Excel support + +#### Type Parameters + +`TRow` + +Row schema type (must be flat and text-serializable) + +#### Remarks + +

+Requirements: +

+
  • TRow must implement IFlatSchema (Excel is tabular)
  • TRow must implement ITextSerializable
+

+Limitations: Read-only support via ExcelDataReader. +Writing Excel files is not supported. +

+

+Storage Traits: +

+
  • CanWrite: false (Excel adapter is read-only via ExcelDataReader)
+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.Storage.Format.ExcelFormatSerializer-1.md b/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.Storage.Format.ExcelFormatSerializer-1.md new file mode 100644 index 00000000..4d44fb9c --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.Storage.Format.ExcelFormatSerializer-1.md @@ -0,0 +1,235 @@ +# Class ExcelFormatSerializer + +Namespace: [Flowthru.Data.Storage.Format](Flowthru.Data.Storage.Format.md) +Assembly: Flowthru.Extensions.Excel.dll + +Serializes flat schemas to/from Excel (.xlsx) files using ExcelDataReader. + +```csharp +public sealed class ExcelFormatSerializer : IFormatSerializer where TRow : notnull, IFlatSchema, ITextSerializable +``` + +#### Type Parameters + +`TRow` + +Row type (must be flat and text-serializable) + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ExcelFormatSerializer](Flowthru.Data.Storage.Format.ExcelFormatSerializer\-1.md) + +#### Implements + +IFormatSerializer + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Read-Only: ExcelDataReader only supports reading Excel files. +Calling SerializeRows will throw NotSupportedException. +

+

+Sheet Selection: Reads from specified sheet name. +

+ +## Constructors + +### ExcelFormatSerializer\(string\) + +```csharp +public ExcelFormatSerializer(string sheetName) +``` + +#### Parameters + +`sheetName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +## Properties + +### Traits + +Structural capabilities of this format serializer. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + StorageTraits + +#### Remarks + +

+Format traits focus on HOW data is serialized and whether it supports streaming. +For composed adapters, these traits are merged with medium and container traits. +

+

+Most formats should declare CanStream = true if they can deserialize row-by-row +without buffering the entire stream (e.g., CSV, JSONL). Formats that require full +parsing before yielding rows (e.g., JSON arrays) should set CanStream = false. +

+ +## Methods + +### DeserializeRows\(Stream\) + +Deserializes a stream of bytes into a stream of rows. + +```csharp +public IAsyncEnumerable DeserializeRows(Stream stream) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +The stream containing serialized data + +#### Returns + + [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +Async enumerable of deserialized rows + +#### Remarks + +

+Streaming Behavior: +

+

+Rows should be yielded as they are deserialized (lazy evaluation). +This allows processing large datasets without loading everything into memory. +

+

+Error Handling: +

+

+Deserialization errors should throw exceptions: +- Format exceptions (malformed CSV, invalid JSON) +- Schema mismatches (missing columns, type conversion failures) +- I/O errors during stream reading +

+

+The caller should handle these exceptions appropriately. +

+ +### GetPropertyMappingConfiguration\(\) + +Configures how this serializer handles property-to-field name mapping for the schema. + +```csharp +public PropertyMappingConfiguration GetPropertyMappingConfiguration() +``` + +#### Returns + + PropertyMappingConfiguration + +Property mapping configuration describing the mapping strategy + +#### Examples + +
// CSV serializer using SerializedLabel
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.FromSerializedLabel<TRow>();
+
+// ML.NET serializer using native attributes
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.FromNativeAttributes("LoadColumnAttribute");
+
+// Parquet with library-controlled mapping
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.LibraryControlled();
+ +#### Remarks + +

+Contractual Obligation: Every format serializer MUST implement this method +to explicitly declare how it handles property name mapping. +

+

+Implementation Strategies: +

+
  • +SerializedLabel Support: Use +to respect [SerializedLabel] attributes. Return . +
  • +Native Attribute Mapping: Use format-specific attributes (e.g., ML.NET's [LoadColumn]). +Return with the attribute type name. +
  • +Library-Controlled: When the underlying library handles mapping internally +(e.g., Parquet.NET with no programmatic API). Return PropertyMappingConfiguration.LibraryControlled(). +
  • +Adapter Pattern: Bridge between SerializedLabel and native attributes. +Return . +
+

+Design Intent: This contract makes property mapping an explicit, discoverable +capability rather than an implicit behavior, enabling: +

+
  • Runtime introspection of mapping capabilities
  • Better error messages when schemas don't match storage
  • Documentation generation for serializer capabilities
  • Testing framework validation of mapping correctness
+ +### SerializeRows\(Stream, IAsyncEnumerable\) + +Serializes a stream of rows into a stream of bytes. + +```csharp +public Task SerializeRows(Stream stream, IAsyncEnumerable rows) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +The stream to write serialized data to + +`rows` [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +The rows to serialize + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task) + +Task that completes when serialization finishes + +#### Remarks + +

+Streaming Behavior: +

+

+Rows should be written as they are enumerated (lazy evaluation). +This allows handling large datasets efficiently. +

+

+Format-Specific Headers: +

+

+Implementations should handle format-specific initialization: +- CSV: Write header row with column names +- JSON: Write opening bracket for array +- Parquet: Write schema metadata +

+

+Error Handling: +

+

+Serialization errors should throw exceptions: +- Type conversion failures +- I/O errors during stream writing +- Invalid data values for format constraints +

+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.Storage.Format.md b/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.Storage.Format.md new file mode 100644 index 00000000..370126dd --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.Storage.Format.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Data.Storage.Format + +### Classes + + [ExcelFormatSerializer](Flowthru.Data.Storage.Format.ExcelFormatSerializer\-1.md) + +Serializes flat schemas to/from Excel (.xlsx) files using ExcelDataReader. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.md b/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.md new file mode 100644 index 00000000..7591d031 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Excel/Flowthru.Data.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Data + +### Classes + + [ExcelItemExtensions](Flowthru.Data.ExcelItemExtensions.md) + +Extension methods that add Excel support to . + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Excel/toc.yml b/docs/reference/src/extensions/Flowthru.Extensions.Excel/toc.yml new file mode 100644 index 00000000..d0cf467a --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Excel/toc.yml @@ -0,0 +1,13 @@ +### YamlMime:TableOfContent +- name: Flowthru.Data + href: Flowthru.Data.md + items: + - name: Classes + - name: ExcelItemExtensions + href: Flowthru.Data.ExcelItemExtensions.md +- name: Flowthru.Data.Storage.Format + href: Flowthru.Data.Storage.Format.md + items: + - name: Classes + - name: ExcelFormatSerializer + href: Flowthru.Data.Storage.Format.ExcelFormatSerializer-1.md diff --git a/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.CatalogEntriesMLNet.md b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.CatalogEntriesMLNet.md new file mode 100644 index 00000000..c720c1aa --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.CatalogEntriesMLNet.md @@ -0,0 +1,81 @@ +# Class ItemsMLNet + +Namespace: [Flowthru.Extensions.MLNet](Flowthru.Extensions.MLNet.md) +Assembly: Flowthru.Extensions.MLNet.dll + +Factory methods for creating ML.NET-related catalog entries. + +```csharp +public static class ItemsMLNet +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ItemsMLNet](Flowthru.Extensions.MLNet.ItemsMLNet.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Discovery Pattern: Import this class alongside +for ML.NET-specific catalog entry factory methods. +

+

+Use Cases: +

+
  • Loading pre-trained ONNX models for inference
  • Working with ML.NET IDataView data structures
+

+Usage: +

+
using Flowthru.Data;
+using Flowthru.Extensions.MLNet;
+
+// Core entries
+var csvEntry = Items.Enumerable.Csv<MySchema>("data", "data.csv");
+
+// MLNet entries
+var modelEntry = ItemsMLNet.OnnxModel("model", "model.onnx");
+ +## Methods + +### OnnxModel\(string, string\) + +Creates a catalog entry for an ONNX model file. + +```csharp +public static IItem OnnxModel(string label, string filePath) +``` + +#### Parameters + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Human-readable label for the catalog entry + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to the .onnx model file + +#### Returns + + IItem<[byte](https://learn.microsoft.com/dotnet/api/system.byte)\[\]\> + +A catalog entry wrapping an ONNX model storage adapter + +#### Examples + +
var entry = ItemsMLNet.OnnxModel(
+    label: "BertModel",
+    filePath: "models/bert-base.onnx"
+);
+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Container.DataViewContainerAdapter-1.md b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Container.DataViewContainerAdapter-1.md new file mode 100644 index 00000000..d6179c00 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Container.DataViewContainerAdapter-1.md @@ -0,0 +1,186 @@ +# Class DataViewContainerAdapter + +Namespace: [Flowthru.Extensions.MLNet.Container](Flowthru.Extensions.MLNet.Container.md) +Assembly: Flowthru.Extensions.MLNet.dll + +Container adapter for ML.NET IDataView - columnar data representation. + +```csharp +public sealed class DataViewContainerAdapter : IContainerAdapter where T : class, new() +``` + +#### Type Parameters + +`T` + +The row schema type + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[DataViewContainerAdapter](Flowthru.Extensions.MLNet.Container.DataViewContainerAdapter\-1.md) + +#### Implements + +IContainerAdapter + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
public record FeatureRow(
+    float Feature1,
+    float Feature2,
+    int Label
+) : IFlatSchema, ITextSerializable;
+
+var mlContext = new MLContext();
+var adapter = new DataViewContainerAdapter<FeatureRow>(mlContext);
+
+// From rows to IDataView
+var dataView = await adapter.FromRows(rowStream);
+
+// Use with ML.NET
+var pipeline = mlContext.Transforms
+    .NormalizeMinMax("Feature1")
+    .Append(mlContext.Transforms.NormalizeMinMax("Feature2"));
+
+var model = pipeline.Fit(dataView);
+var transformedData = model.Transform(dataView);
+
+// Back to rows
+var transformedRows = adapter.ToRows(transformedData);
+ +## Remarks + +

+NEW CAPABILITY: This adapter enables native ML.NET integration with Flowthru! +

+

+Characteristics: +

+
  • Columnar storage: Optimized for ML.NET operations
  • Lazy evaluation: Data loaded on-demand during iteration
  • Type safety: Strongly-typed row schema
  • ML.NET native: Direct integration with ML.NET pipelines
+

+Use Cases: +

+
  • Machine learning pipelines using ML.NET
  • Data transformations (normalization, encoding, etc.)
  • Feature engineering workflows
  • Model training and evaluation
+

+Integration with ML.Next: +

+

+This adapter bridges Flowthru catalogs with ML.Next's type-safe wrappers, +enabling end-to-end compile-time safety for ML pipelines. +

+ +## Constructors + +### DataViewContainerAdapter\(MLContext\) + +Creates a new IDataView container adapter. + +```csharp +public DataViewContainerAdapter(MLContext mlContext) +``` + +#### Parameters + +`mlContext` [MLContext](https://learn.microsoft.com/dotnet/api/microsoft.ml.mlcontext) + +The ML.NET context for data operations + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +Thrown if mlContext is null + +## Properties + +### MLContext + +Gets the ML.NET context used by this adapter. + +```csharp +public MLContext MLContext { get; } +``` + +#### Property Value + + [MLContext](https://learn.microsoft.com/dotnet/api/microsoft.ml.mlcontext) + +## Methods + +### FromRows\(IAsyncEnumerable\) + +Materializes an async stream of rows into an in-memory container. + +```csharp +public Task FromRows(IAsyncEnumerable rows) +``` + +#### Parameters + +`rows` [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +Async stream of rows from format deserialization + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1)<[IDataView](https://learn.microsoft.com/dotnet/api/microsoft.ml.idataview)\> + +Task producing the populated container + +#### Remarks + +

+Materialization Strategy: +

+

+Different containers have different materialization approaches: +

+
  • IEnumerable: Eager - load all rows into List
  • Seq: Lazy - wrap async enumerable
  • IDataView: Columnar - convert to columnar format
+

+Memory Considerations: +

+

+Be aware that eager containers (IEnumerable) will load all data into memory. +For large datasets, prefer streaming containers (Seq, IDataView with lazy loading). +

+ +### ToRows\(IDataView\) + +Converts an in-memory container back to an async stream of rows. + +```csharp +public IAsyncEnumerable ToRows(IDataView container) +``` + +#### Parameters + +`container` [IDataView](https://learn.microsoft.com/dotnet/api/microsoft.ml.idataview) + +The container to stream from + +#### Returns + + [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +Async enumerable of rows for format serialization + +#### Remarks + +

+Streaming Strategy: +

+

+Rows should be yielded lazily if the container supports it: +

+
  • IEnumerable: Enumerate and yield
  • Seq: Already lazy - expose as async enumerable
  • IDataView: Enumerate rows from columnar format
+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Container.md b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Container.md new file mode 100644 index 00000000..9d8586f4 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Container.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Extensions.MLNet.Container + +### Classes + + [DataViewContainerAdapter](Flowthru.Extensions.MLNet.Container.DataViewContainerAdapter\-1.md) + +Container adapter for ML.NET IDataView - columnar data representation. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Storage.OnnxModelStorageAdapter.md b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Storage.OnnxModelStorageAdapter.md new file mode 100644 index 00000000..c0eec92b --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Storage.OnnxModelStorageAdapter.md @@ -0,0 +1,269 @@ +# Class OnnxModelStorageAdapter + +Namespace: [Flowthru.Extensions.MLNet.Storage](Flowthru.Extensions.MLNet.Storage.md) +Assembly: Flowthru.Extensions.MLNet.dll + +Storage adapter for ONNX model files. + +```csharp +public sealed class OnnxModelStorageAdapter : IStorageAdapter +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[OnnxModelStorageAdapter](Flowthru.Extensions.MLNet.Storage.OnnxModelStorageAdapter.md) + +#### Implements + +IStorageAdapter + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Examples + +
var adapter = new OnnxModelStorageAdapter("models/bert-base.onnx");
+var validation = await adapter.ShallowInspect().RunAsync();
+
+if (!validation.IsValid)
+{
+    Console.WriteLine($"ONNX model validation failed: {validation.ErrorMessage}");
+}
+ +## Remarks + +

+Use Case: Pre-trained ONNX models for ML.NET inference +

+

+Validation: Implements inspection for early pipeline validation +

+

+Storage Traits: +

+
  • CanWrite: false (ONNX models are read-only seed data)
  • Inspection: validates file before pipeline execution
+ +## Constructors + +### OnnxModelStorageAdapter\(string\) + +Creates a new ONNX model storage adapter. + +```csharp +public OnnxModelStorageAdapter(string filePath) +``` + +#### Parameters + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to .onnx model file + +## Properties + +### FilePath + +Gets the file path to the ONNX model. + +```csharp +public string FilePath { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### Traits + +Structural constraints and capabilities of this storage implementation. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + StorageTraits + +#### Remarks + +

+Adapter authors must declare what their storage can and cannot do. +These are intrinsic properties of the storage medium, not runtime state. +

+

+Pipeline validation uses these traits to fail fast when a pipeline attempts +invalid operations (e.g., writing to a read-only source). +

+ +## Methods + +### Exists\(\) + +Checks if data exists at this storage location. + +```csharp +public FlowIO Exists() +``` + +#### Returns + + FlowIO<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\> + +Effect that produces true if data exists, false otherwise + +#### Remarks + +

+Delegates to the underlying medium's Exists check. +Used to determine if a catalog entry is a seed (Layer 0 input). +

+ +### InspectDeep\(\) + +Performs deep validation by examining the entire dataset. + +```csharp +public FlowIO InspectDeep() +``` + +#### Returns + + FlowIO + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that all data is available, accessible, and valid. +

+

+Additional Checks Beyond Shallow: +

+
  • Validate ALL rows can be deserialized (not just sample)
  • Check data quality constraints across entire dataset
  • Detect corruption or inconsistencies throughout data
+

+Implementation Guidelines: +

+
  • File adapters: Read and validate entire file
  • Memory adapters: Validate all stored data
  • Database adapters: Full table scan with validation
  • Null adapters: Always return success (no data required)
+

+Performance: Potentially expensive - only use when data integrity is critical. +

+ +### InspectShallow\(int\) + +Performs shallow validation by checking data availability and sampling a subset of data. + +```csharp +public FlowIO InspectShallow(int sampleSize) +``` + +#### Parameters + +`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of rows/records to sample for validation + +#### Returns + + FlowIO + +Effect producing validation result + +#### Remarks + +

+Semantic Intent: Validate that data is available and accessible. +

+

+Typical Checks: +

+
  • Data source exists (file, table, etc.)
  • Data source is accessible (permissions, connectivity)
  • Sample rows can be read and deserialized successfully
  • Schema matches expected structure
+

+Implementation Guidelines: +

+
  • File adapters: Check file exists, read and validate sample rows
  • Memory adapters: Check if data has been initialized
  • Database adapters: Check table exists, query sample rows
  • Null adapters: Always return success (no data required)
+

+Performance: Should be fast (~10-100ms) - suitable for pre-flight validation. +

+ +### Load\(\) + +Loads data from storage. + +```csharp +public FlowIO Load() +``` + +#### Returns + + FlowIO<[byte](https://learn.microsoft.com/dotnet/api/system.byte)\[\]\> + +Effect that produces data on success + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. medium.ReadStream()           → Stream
+2. format.DeserializeRows()      → IAsyncEnumerable<TRow>
+3. container.FromRows()          → TContainer
+

+Error Handling: +

+

+Errors from any layer are propagated: +- Medium errors (file not found, access denied) +- Format errors (parse failures, schema mismatches) +- Container errors (memory allocation, type conversion) +

+ +### Save\(byte\[\]\) + +Saves data to storage. + +```csharp +public FlowIO Save(byte[] data) +``` + +#### Parameters + +`data` [byte](https://learn.microsoft.com/dotnet/api/system.byte)\[\] + +The data to save + +#### Returns + + FlowIO + +Effect that completes on successful save + +#### Remarks + +

+Execution Flow: +

+

+For composed adapters, this orchestrates: +

+
1. container.ToRows()            → IAsyncEnumerable<TRow>
+2. format.SerializeRows()        → Stream
+3. medium.WriteStream()          → FlowUnit
+

+Atomicity: +

+

+Implementations should strive for atomic saves to avoid partial writes on failure. +

+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Storage.md b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Storage.md new file mode 100644 index 00000000..714f88a3 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.Storage.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Extensions.MLNet.Storage + +### Classes + + [OnnxModelStorageAdapter](Flowthru.Extensions.MLNet.Storage.OnnxModelStorageAdapter.md) + +Storage adapter for ONNX model files. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.md b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.md new file mode 100644 index 00000000..3e8689ea --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/Flowthru.Extensions.MLNet.md @@ -0,0 +1,14 @@ +# Namespace Flowthru.Extensions.MLNet + +### Namespaces + + [Flowthru.Extensions.MLNet.Container](Flowthru.Extensions.MLNet.Container.md) + + [Flowthru.Extensions.MLNet.Storage](Flowthru.Extensions.MLNet.Storage.md) + +### Classes + + [ItemsMLNet](Flowthru.Extensions.MLNet.ItemsMLNet.md) + +Factory methods for creating ML.NET-related catalog entries. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.MLNet/toc.yml b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/toc.yml new file mode 100644 index 00000000..01a9389e --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.MLNet/toc.yml @@ -0,0 +1,19 @@ +### YamlMime:TableOfContent +- name: Flowthru.Extensions.MLNet + href: Flowthru.Extensions.MLNet.md + items: + - name: Classes + - name: ItemsMLNet + href: Flowthru.Extensions.MLNet.ItemsMLNet.md +- name: Flowthru.Extensions.MLNet.Container + href: Flowthru.Extensions.MLNet.Container.md + items: + - name: Classes + - name: DataViewContainerAdapter + href: Flowthru.Extensions.MLNet.Container.DataViewContainerAdapter-1.md +- name: Flowthru.Extensions.MLNet.Storage + href: Flowthru.Extensions.MLNet.Storage.md + items: + - name: Classes + - name: OnnxModelStorageAdapter + href: Flowthru.Extensions.MLNet.Storage.OnnxModelStorageAdapter.md diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.ParquetCatalogEntryExtensions.md b/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.ParquetCatalogEntryExtensions.md new file mode 100644 index 00000000..12870d08 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.ParquetCatalogEntryExtensions.md @@ -0,0 +1,72 @@ +# Class ParquetItemExtensions + +Namespace: [Flowthru.Data](Flowthru.Data.md) +Assembly: Flowthru.Extensions.Parquet.dll + +Extension methods that add Parquet support to . + +```csharp +public static class ParquetItemExtensions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ParquetItemExtensions](Flowthru.Data.ParquetItemExtensions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### Parquet\(EnumerableItems, string, string\) + +Creates a Parquet file catalog entry with IEnumerable container. + +```csharp +public static Item> Parquet(this EnumerableItems _, string label, string filePath) where TRow : notnull, IFlatSchema, IBinarySerializable +``` + +#### Parameters + +`_` EnumerableItems + +The enumerable catalog entries factory (from ) + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique catalog label for DAG resolution + +`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Path to Parquet file + +#### Returns + + Item<[IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1)\> + +Catalog entry with file + Parquet + IEnumerable composition + +#### Type Parameters + +`TRow` + +Row schema type (must be flat and binary-serializable) + +#### Remarks + +

+Requirements: +

+
  • TRow must implement IFlatSchema (Parquet is columnar)
  • TRow must implement IBinarySerializable
+

+Performance: Optimized for large datasets with columnar storage. +

+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.Storage.Format.ParquetFormatSerializer-1.md b/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.Storage.Format.ParquetFormatSerializer-1.md new file mode 100644 index 00000000..55c24f84 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.Storage.Format.ParquetFormatSerializer-1.md @@ -0,0 +1,240 @@ +# Class ParquetFormatSerializer + +Namespace: [Flowthru.Data.Storage.Format](Flowthru.Data.Storage.Format.md) +Assembly: Flowthru.Extensions.Parquet.dll + +Format serializer for Parquet (columnar storage) files using adapter pattern. + +```csharp +public sealed class ParquetFormatSerializer : IFormatSerializer where TRow : notnull, IFlatSchema, IBinarySerializable +``` + +#### Type Parameters + +`TRow` + +The Flowthru schema type + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ParquetFormatSerializer](Flowthru.Data.Storage.Format.ParquetFormatSerializer\-1.md) + +#### Implements + +IFormatSerializer + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Architecture: +

+

+Converts between Flowthru schemas (with required members) and Parquet-compatible DTOs: +

+
Serialize:   TRow (required members) → DTO (parameterless ctor) → Parquet
+Deserialize: Parquet → DTO (parameterless ctor) → TRow (required members)
+

+Features: +

+
  • SerializedLabel - Respects [SerializedLabel] attributes for property name mapping
  • Null Safety - Enforces non-nullable contracts during deserialization
  • Value Type Nullability - DTOs use nullable value types to match Parquet schema conventions
  • Enum Support - Automatically converts between Parquet's integer storage and enum types
+

+Current Limitations: +

+
  • SerializedEnum attributes are not used - enums stored/retrieved by underlying integer value
+ +## Constructors + +### ParquetFormatSerializer\(\) + +```csharp +public ParquetFormatSerializer() +``` + +## Properties + +### Traits + +Structural capabilities of this format serializer. + +```csharp +public StorageTraits Traits { get; } +``` + +#### Property Value + + StorageTraits + +#### Remarks + +

+Format traits focus on HOW data is serialized and whether it supports streaming. +For composed adapters, these traits are merged with medium and container traits. +

+

+Most formats should declare CanStream = true if they can deserialize row-by-row +without buffering the entire stream (e.g., CSV, JSONL). Formats that require full +parsing before yielding rows (e.g., JSON arrays) should set CanStream = false. +

+ +## Methods + +### DeserializeRows\(Stream\) + +Deserializes a stream of bytes into a stream of rows. + +```csharp +public IAsyncEnumerable DeserializeRows(Stream stream) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +The stream containing serialized data + +#### Returns + + [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +Async enumerable of deserialized rows + +#### Remarks + +

+Streaming Behavior: +

+

+Rows should be yielded as they are deserialized (lazy evaluation). +This allows processing large datasets without loading everything into memory. +

+

+Error Handling: +

+

+Deserialization errors should throw exceptions: +- Format exceptions (malformed CSV, invalid JSON) +- Schema mismatches (missing columns, type conversion failures) +- I/O errors during stream reading +

+

+The caller should handle these exceptions appropriately. +

+ +### GetPropertyMappingConfiguration\(\) + +Configures how this serializer handles property-to-field name mapping for the schema. + +```csharp +public PropertyMappingConfiguration GetPropertyMappingConfiguration() +``` + +#### Returns + + PropertyMappingConfiguration + +Property mapping configuration describing the mapping strategy + +#### Examples + +
// CSV serializer using SerializedLabel
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.FromSerializedLabel<TRow>();
+
+// ML.NET serializer using native attributes
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.FromNativeAttributes("LoadColumnAttribute");
+
+// Parquet with library-controlled mapping
+public PropertyMappingConfiguration GetPropertyMappingConfiguration()
+    => PropertyMappingConfiguration.LibraryControlled();
+ +#### Remarks + +

+Contractual Obligation: Every format serializer MUST implement this method +to explicitly declare how it handles property name mapping. +

+

+Implementation Strategies: +

+
  • +SerializedLabel Support: Use +to respect [SerializedLabel] attributes. Return . +
  • +Native Attribute Mapping: Use format-specific attributes (e.g., ML.NET's [LoadColumn]). +Return with the attribute type name. +
  • +Library-Controlled: When the underlying library handles mapping internally +(e.g., Parquet.NET with no programmatic API). Return PropertyMappingConfiguration.LibraryControlled(). +
  • +Adapter Pattern: Bridge between SerializedLabel and native attributes. +Return . +
+

+Design Intent: This contract makes property mapping an explicit, discoverable +capability rather than an implicit behavior, enabling: +

+
  • Runtime introspection of mapping capabilities
  • Better error messages when schemas don't match storage
  • Documentation generation for serializer capabilities
  • Testing framework validation of mapping correctness
+ +### SerializeRows\(Stream, IAsyncEnumerable\) + +Serializes a stream of rows into a stream of bytes. + +```csharp +public Task SerializeRows(Stream stream, IAsyncEnumerable rows) +``` + +#### Parameters + +`stream` [Stream](https://learn.microsoft.com/dotnet/api/system.io.stream) + +The stream to write serialized data to + +`rows` [IAsyncEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.iasyncenumerable\-1) + +The rows to serialize + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task) + +Task that completes when serialization finishes + +#### Remarks + +

+Streaming Behavior: +

+

+Rows should be written as they are enumerated (lazy evaluation). +This allows handling large datasets efficiently. +

+

+Format-Specific Headers: +

+

+Implementations should handle format-specific initialization: +- CSV: Write header row with column names +- JSON: Write opening bracket for array +- Parquet: Write schema metadata +

+

+Error Handling: +

+

+Serialization errors should throw exceptions: +- Type conversion failures +- I/O errors during stream writing +- Invalid data values for format constraints +

+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.Storage.Format.md b/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.Storage.Format.md new file mode 100644 index 00000000..51f0052a --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.Storage.Format.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Data.Storage.Format + +### Classes + + [ParquetFormatSerializer](Flowthru.Data.Storage.Format.ParquetFormatSerializer\-1.md) + +Format serializer for Parquet (columnar storage) files using adapter pattern. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.md b/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.md new file mode 100644 index 00000000..a06fc12a --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Parquet/Flowthru.Data.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Data + +### Classes + + [ParquetItemExtensions](Flowthru.Data.ParquetItemExtensions.md) + +Extension methods that add Parquet support to . + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Parquet/toc.yml b/docs/reference/src/extensions/Flowthru.Extensions.Parquet/toc.yml new file mode 100644 index 00000000..1d1a585e --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Parquet/toc.yml @@ -0,0 +1,13 @@ +### YamlMime:TableOfContent +- name: Flowthru.Data + href: Flowthru.Data.md + items: + - name: Classes + - name: ParquetItemExtensions + href: Flowthru.Data.ParquetItemExtensions.md +- name: Flowthru.Data.Storage.Format + href: Flowthru.Data.Storage.Format.md + items: + - name: Classes + - name: ParquetFormatSerializer + href: Flowthru.Data.Storage.Format.ParquetFormatSerializer-1.md diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Build.PythonSchemaExportTask.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Build.PythonSchemaExportTask.md new file mode 100644 index 00000000..4ff5ba42 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Build.PythonSchemaExportTask.md @@ -0,0 +1,100 @@ +# Class PythonSchemaExportTask + +Namespace: [Flowthru.Extensions.Python.Build](Flowthru.Extensions.Python.Build.md) +Assembly: Flowthru.Extensions.Python.dll + +MSBuild task that exports C# schema metadata to Python schema files. + +```csharp +public class PythonSchemaExportTask : Task, ITask +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[Task](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task) ← +[PythonSchemaExportTask](Flowthru.Extensions.Python.Build.PythonSchemaExportTask.md) + +#### Implements + +[ITask](https://learn.microsoft.com/dotnet/api/microsoft.build.framework.itask) + +#### Inherited Members + +[Task.Execute\(\)](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.execute), +[Task.BuildEngine](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.buildengine), +[Task.BuildEngine2](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.buildengine2), +[Task.BuildEngine3](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.buildengine3), +[Task.BuildEngine4](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.buildengine4), +[Task.BuildEngine5](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.buildengine5), +[Task.BuildEngine6](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.buildengine6), +[Task.BuildEngine7](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.buildengine7), +[Task.BuildEngine8](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.buildengine8), +[Task.BuildEngine9](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.buildengine9), +[Task.HostObject](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.hostobject), +[Task.Log](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.log), +[Task.TaskResources](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.taskresources), +[Task.HelpKeywordPrefix](https://learn.microsoft.com/dotnet/api/microsoft.build.utilities.task.helpkeywordprefix), +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Phase 5 implementation: +Reads the SchemaManifest generated by SchemaInterfaceGenerator and emits +corresponding Python dataclass files to _generated/flowthru_schemas/. +

+

+Invoked from: Flowthru.Extensions.Python.targets AfterBuild +

+ +## Properties + +### AssemblyPath + +Path to the compiled assembly containing the schema manifest. + +```csharp +[Required] +public string AssemblyPath { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### OutputDirectory + +Directory where Python schema files will be generated. + +```csharp +[Required] +public string OutputDirectory { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +## Methods + +### Execute\(\) + +Must be implemented by derived class. + +```csharp +public override bool Execute() +``` + +#### Returns + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +true, if successful + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Build.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Build.md new file mode 100644 index 00000000..ba8792f4 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Build.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Extensions.Python.Build + +### Classes + + [PythonSchemaExportTask](Flowthru.Extensions.Python.Build.PythonSchemaExportTask.md) + +MSBuild task that exports C# schema metadata to Python schema files. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.IPythonExecutor.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.IPythonExecutor.md new file mode 100644 index 00000000..e16b9b2a --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.IPythonExecutor.md @@ -0,0 +1,103 @@ +# Interface IPythonExecutor + +Namespace: [Flowthru.Extensions.Python.Execution](Flowthru.Extensions.Python.Execution.md) +Assembly: Flowthru.Extensions.Python.dll + +Abstraction for executing Python code. + +```csharp +public interface IPythonExecutor +``` + +## Remarks + +

+Decouples the execution strategy from Python step wiring. +Two implementations ship out of the box: +

  • — in-process via Python.NET (opt-in)
  • — isolated child process per service (default)
+

+

+Isolation contract: +Two FlowthruServices using do not share Python state. +Each executor spawns its own Python worker process with its own venv, sys.path, +sys.modules, and GIL — complete isolation at the cost of IPC marshalling overhead. +

+

+All implementations must handle: +

  • Module import and caching
  • Function resolution and invocation
  • Argument marshalling (C# ↔ Python) — scalar, tabular (Arrow IPC), and raw bytes
  • Error propagation (Python exceptions → )
+

+ +## Methods + +### Invoke\(string, string, TInput\) + +Invokes a Python function, marshalling input and output to/from C# types. + +```csharp +TOutput Invoke(string moduleName, string functionName, TInput input) +``` + +#### Parameters + +`moduleName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted module name (e.g., "Flows.DataScience.train_model"). +Must be resolvable via the executor's configured sys.path. + +`functionName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module. + +`input` TInput + +Typed input value. + +#### Returns + + TOutput + +Typed output value returned by the Python function. + +#### Type Parameters + +`TInput` + +C# input type. May be a scalar, IEnumerable<TSchema> (tabular), +byte[] (raw bytes), or a ValueTuple of any of those (multi-input). + +`TOutput` + +C# output type. Same range as TInput. + +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if the module cannot be imported, the function cannot be resolved, +or marshalling fails. + +### ValidateStep\(string, string\) + +Validates that a Python step exists and satisfies Flowthru's @step contract. + +```csharp +void ValidateStep(string moduleName, string functionName) +``` + +#### Parameters + +`moduleName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name. + +`functionName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module. + +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if the module is not importable, the function is missing, or the +@step decorator is absent. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.PythonNetExecutor.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.PythonNetExecutor.md new file mode 100644 index 00000000..7acc36d8 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.PythonNetExecutor.md @@ -0,0 +1,134 @@ +# Class PythonNetExecutor + +Namespace: [Flowthru.Extensions.Python.Execution](Flowthru.Extensions.Python.Execution.md) +Assembly: Flowthru.Extensions.Python.dll + +In-process Python executor using Python.NET. + +```csharp +public sealed class PythonNetExecutor : IPythonExecutor +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PythonNetExecutor](Flowthru.Extensions.Python.Execution.PythonNetExecutor.md) + +#### Implements + +[IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Executes Python functions within the same process via Python.NET's embedded runtime. +All marshalling (scalar, tabular Arrow IPC, bytes, multi-I/O tuples) is handled +internally — callers interact only with strongly-typed C# values. +

+

+Isolation caveat: +PythonEngine is process-global. Multiple PythonNetExecutor instances share +the same interpreter, sys.modules, and GIL. Use +when true per-service isolation is required. +

+

+Thread-safety: GIL acquisition serialises all Python execution. +

+ +## Constructors + +### PythonNetExecutor\(PythonRuntime, ILogger\) + +```csharp +public PythonNetExecutor(PythonRuntime runtime, ILogger logger) +``` + +#### Parameters + +`runtime` [PythonRuntime](Flowthru.Extensions.Python.Runtime.PythonRuntime.md) + +`logger` [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger\-1)<[PythonNetExecutor](Flowthru.Extensions.Python.Execution.PythonNetExecutor.md)\> + +## Methods + +### Invoke\(string, string, TInput\) + +Invokes a Python function, marshalling input and output to/from C# types. + +```csharp +public TOutput Invoke(string moduleName, string functionName, TInput input) +``` + +#### Parameters + +`moduleName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted module name (e.g., "Flows.DataScience.train_model"). +Must be resolvable via the executor's configured sys.path. + +`functionName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module. + +`input` TInput + +Typed input value. + +#### Returns + + TOutput + +Typed output value returned by the Python function. + +#### Type Parameters + +`TInput` + +C# input type. May be a scalar, IEnumerable<TSchema> (tabular), +byte[] (raw bytes), or a ValueTuple of any of those (multi-input). + +`TOutput` + +C# output type. Same range as TInput. + +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if the module cannot be imported, the function cannot be resolved, +or marshalling fails. + +### ValidateStep\(string, string\) + +Validates that a Python step exists and satisfies Flowthru's @step contract. + +```csharp +public void ValidateStep(string moduleName, string functionName) +``` + +#### Parameters + +`moduleName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name. + +`functionName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module. + +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if the module is not importable, the function is missing, or the +@step decorator is absent. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.SubprocessPythonExecutor.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.SubprocessPythonExecutor.md new file mode 100644 index 00000000..73fd5dd8 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.SubprocessPythonExecutor.md @@ -0,0 +1,143 @@ +# Class SubprocessPythonExecutor + +Namespace: [Flowthru.Extensions.Python.Execution](Flowthru.Extensions.Python.Execution.md) +Assembly: Flowthru.Extensions.Python.dll + +Subprocess Python executor — spawns an isolated Python worker process per instance. + +```csharp +public sealed class SubprocessPythonExecutor : IPythonExecutor, IDisposable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[SubprocessPythonExecutor](Flowthru.Extensions.Python.Execution.SubprocessPythonExecutor.md) + +#### Implements + +[IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md), +[IDisposable](https://learn.microsoft.com/dotnet/api/system.idisposable) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Each SubprocessPythonExecutor owns one child Python process. Isolation is at the +OS process boundary: separate interpreter, sys.modules, venv, and memory space. +No Python.NET, no GIL management on the C# side. +

+

+Protocol: newline-delimited JSON over stdin/stdout. +Tabular data is exchanged as base64-encoded Apache Arrow IPC buffers. +Dtype coercion specs are serialized from . +

+

+The worker script (flowthru_worker.py) must be present in +. +

+ +## Constructors + +### SubprocessPythonExecutor\(PythonRuntimeOptions, ILogger\) + +```csharp +public SubprocessPythonExecutor(PythonRuntimeOptions options, ILogger logger) +``` + +#### Parameters + +`options` [PythonRuntimeOptions](Flowthru.Extensions.Python.Runtime.PythonRuntimeOptions.md) + +`logger` [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger\-1)<[SubprocessPythonExecutor](Flowthru.Extensions.Python.Execution.SubprocessPythonExecutor.md)\> + +## Methods + +### Dispose\(\) + +Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + +```csharp +public void Dispose() +``` + +### Invoke\(string, string, TInput\) + +Invokes a Python function, marshalling input and output to/from C# types. + +```csharp +public TOutput Invoke(string moduleName, string functionName, TInput input) +``` + +#### Parameters + +`moduleName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted module name (e.g., "Flows.DataScience.train_model"). +Must be resolvable via the executor's configured sys.path. + +`functionName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module. + +`input` TInput + +Typed input value. + +#### Returns + + TOutput + +Typed output value returned by the Python function. + +#### Type Parameters + +`TInput` + +C# input type. May be a scalar, IEnumerable<TSchema> (tabular), +byte[] (raw bytes), or a ValueTuple of any of those (multi-input). + +`TOutput` + +C# output type. Same range as TInput. + +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if the module cannot be imported, the function cannot be resolved, +or marshalling fails. + +### ValidateStep\(string, string\) + +Validates that a Python step exists and satisfies Flowthru's @step contract. + +```csharp +public void ValidateStep(string moduleName, string functionName) +``` + +#### Parameters + +`moduleName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name. + +`functionName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module. + +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if the module is not importable, the function is missing, or the +@step decorator is absent. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.md new file mode 100644 index 00000000..37635f40 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Execution.md @@ -0,0 +1,18 @@ +# Namespace Flowthru.Extensions.Python.Execution + +### Classes + + [PythonNetExecutor](Flowthru.Extensions.Python.Execution.PythonNetExecutor.md) + +In-process Python executor using Python.NET. + + [SubprocessPythonExecutor](Flowthru.Extensions.Python.Execution.SubprocessPythonExecutor.md) + +Subprocess Python executor — spawns an isolated Python worker process per instance. + +### Interfaces + + [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Abstraction for executing Python code. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.ArrowMarshaller.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.ArrowMarshaller.md new file mode 100644 index 00000000..0d1fef2c --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.ArrowMarshaller.md @@ -0,0 +1,185 @@ +# Class ArrowMarshaller + +Namespace: [Flowthru.Extensions.Python.Marshalling](Flowthru.Extensions.Python.Marshalling.md) +Assembly: Flowthru.Extensions.Python.dll + +Marshals tabular data between C# IEnumerable<T> and Apache Arrow RecordBatch. + +```csharp +public static class ArrowMarshaller +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ArrowMarshaller](Flowthru.Extensions.Python.Marshalling.ArrowMarshaller.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Purpose: Bidirectional conversion for DataFrame interchange between +C# and Python via Arrow IPC (Inter-Process Communication) format. +

+

+C# → Arrow Flow: +

+
IEnumerable<T> → RecordBatch → IPC buffer → Python pyarrow.Table → pd.DataFrame
+

+Arrow → C# Flow: +

+
pd.DataFrame → pyarrow.Table → IPC buffer → RecordBatch → IEnumerable<T>
+

+Performance: Uses columnar processing (column-by-column, not row-by-row) +for efficient Arrow array construction. +

+ +## Methods + +### FromIpcBuffer\(byte\[\]\) + +Deserializes an Arrow IPC buffer (from Python) to a RecordBatch. + +```csharp +public static RecordBatch FromIpcBuffer(byte[] buffer) +``` + +#### Parameters + +`buffer` [byte](https://learn.microsoft.com/dotnet/api/system.byte)\[\] + +Byte array containing Arrow IPC stream + +#### Returns + + RecordBatch + +Arrow RecordBatch + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +Thrown when buffer is null + + [InvalidDataException](https://learn.microsoft.com/dotnet/api/system.io.invaliddataexception) + +Thrown when buffer is not a valid Arrow IPC stream. + +### FromRecordBatch\(RecordBatch\) + +Converts an Arrow RecordBatch to an IEnumerable of C# objects. + +```csharp +public static IEnumerable FromRecordBatch(RecordBatch batch) where T : notnull +``` + +#### Parameters + +`batch` RecordBatch + +The Arrow RecordBatch to convert + +#### Returns + + [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1) + +IEnumerable of C# objects + +#### Type Parameters + +`T` + +The C# schema type + +#### Remarks + +Uses SchemaActivator for instantiation to support required members. +Data is converted row-by-row (column values → object properties). + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +Thrown when batch is null + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown when schema mismatch or type conversion fails. + +### ToIpcBuffer\(RecordBatch\) + +Serializes an Arrow RecordBatch to an IPC buffer for Python.NET transfer. + +```csharp +public static byte[] ToIpcBuffer(RecordBatch batch) +``` + +#### Parameters + +`batch` RecordBatch + +The RecordBatch to serialize + +#### Returns + + [byte](https://learn.microsoft.com/dotnet/api/system.byte)\[\] + +Byte array containing Arrow IPC stream + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +Thrown when batch is null + +### ToRecordBatch\(IEnumerable\) + +Converts an IEnumerable of C# objects to an Arrow RecordBatch. + +```csharp +public static RecordBatch ToRecordBatch(IEnumerable rows) where T : notnull +``` + +#### Parameters + +`rows` [IEnumerable](https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable\-1) + +The rows to convert + +#### Returns + + RecordBatch + +Arrow RecordBatch containing the data + +#### Type Parameters + +`T` + +The C# schema type + +#### Remarks + +Data is processed column-wise for efficiency. All rows are materialized into memory +to build the RecordBatch. + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +Thrown when rows is null + + [NotSupportedException](https://learn.microsoft.com/dotnet/api/system.notsupportedexception) + +Thrown when a property type cannot be marshalled to Arrow. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.ArrowSchemaMapper.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.ArrowSchemaMapper.md new file mode 100644 index 00000000..b3757bdb --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.ArrowSchemaMapper.md @@ -0,0 +1,151 @@ +# Class ArrowSchemaMapper + +Namespace: [Flowthru.Extensions.Python.Marshalling](Flowthru.Extensions.Python.Marshalling.md) +Assembly: Flowthru.Extensions.Python.dll + +Maps C# schema types to Apache Arrow schemas for tabular data interchange. + +```csharp +public static class ArrowSchemaMapper +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ArrowSchemaMapper](Flowthru.Extensions.Python.Marshalling.ArrowSchemaMapper.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Purpose: Generates Arrow schemas from C# types annotated with [FlowthruSchema], +preserving field names (via [SerializedLabel]) and nullability for correct DataFrame marshalling. +

+

+Type Mapping (C# → Arrow): +

+
  • int, int? → Int32Type
  • long, long? → Int64Type
  • float, float? → FloatType
  • double, double? → DoubleType
  • bool, bool? → BooleanType
  • string → StringType (always nullable in Arrow)
  • DateTime, DateTime? → TimestampType (microsecond, UTC)
  • DateTimeOffset, DateTimeOffset? → TimestampType (microsecond, UTC)
  • TimeSpan, TimeSpan? → DurationType (microsecond)
  • Guid, Guid? → StringType (serialized as string)
  • byte[] → BinaryType
+

+Field Naming: Uses PropertyMappingHelper to respect [SerializedLabel] attributes, +ensuring consistency with CSV/Parquet/JSON serializers. +

+ +## Methods + +### BuildArrowSchema\(\) + +Generates an Apache Arrow schema from a C# schema type. + +```csharp +public static Schema BuildArrowSchema() where T : notnull +``` + +#### Returns + + Schema + +Arrow schema with fields matching the C# type's properties + +#### Type Parameters + +`T` + +The C# schema type to map + +#### Remarks + +The schema includes all public instance properties, with field names determined by +[SerializedLabel] attributes or property names. Nullability is preserved from the C# type. + +#### Exceptions + + [NotSupportedException](https://learn.microsoft.com/dotnet/api/system.notsupportedexception) + +Thrown when a property type cannot be mapped to Arrow. + +### BuildArrowSchema\(Type\) + +Generates an Apache Arrow schema from a C# type. + +```csharp +public static Schema BuildArrowSchema(Type type) +``` + +#### Parameters + +`type` [Type](https://learn.microsoft.com/dotnet/api/system.type) + +The C# type to map + +#### Returns + + Schema + +Arrow schema with fields matching the type's properties + +#### Exceptions + + [NotSupportedException](https://learn.microsoft.com/dotnet/api/system.notsupportedexception) + +Thrown when a property type cannot be mapped to Arrow. + +### BuildDtypeSpec\(\) + +Builds a Python dictionary (PyDict) mapping column names to pandas dtype strings. + +```csharp +public static object BuildDtypeSpec() where T : notnull +``` + +#### Returns + + [object](https://learn.microsoft.com/dotnet/api/system.object) + +PyDict with dtype specifications for df_to_ipc + +#### Type Parameters + +`T` + +The C# schema type + +#### Remarks + +Maps Arrow types to pandas dtype strings: +- Int32Type → 'int32' +- Int64Type → 'int64' +- FloatType → 'float32' +- DoubleType → 'float64' +- BooleanType → 'bool' +- StringType → 'object' + +### BuildDtypeSpecDictionary\(\) + +Builds a C# dictionary mapping column names to pandas dtype strings, +suitable for JSON serialization in subprocess protocol messages. + +```csharp +public static Dictionary BuildDtypeSpecDictionary() where T : notnull +``` + +#### Returns + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [string](https://learn.microsoft.com/dotnet/api/system.string)\> + +Dictionary with dtype specifications for the subprocess worker's df_to_ipc + +#### Type Parameters + +`T` + +The C# schema type + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.ScalarMarshaller.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.ScalarMarshaller.md new file mode 100644 index 00000000..f8e4ef4e --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.ScalarMarshaller.md @@ -0,0 +1,113 @@ +# Class ScalarMarshaller + +Namespace: [Flowthru.Extensions.Python.Marshalling](Flowthru.Extensions.Python.Marshalling.md) +Assembly: Flowthru.Extensions.Python.dll + +Handles bidirectional conversion between C# scalar values and Python objects. + +```csharp +public static class ScalarMarshaller +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ScalarMarshaller](Flowthru.Extensions.Python.Marshalling.ScalarMarshaller.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Supports: +

  • Primitives: int, long, double, float, string, bool
  • Nullable primitives: int?, double?, etc.
  • Simple records and classes via property iteration
+

+

+Supported since Phase 5: +

  • Arrays: T[] (converted to/from Python lists)
  • Nested records (via recursive marshalling)
+

+

+Not supported: +

  • Generic collections: IEnumerable<T>, List<T> (use arrays or tabular I/O)
  • Arrow/DataFrame interchange (use tabular I/O path)
+

+

+Thread-safety: All methods are thread-safe. +Caller is responsible for GIL acquisition. +

+ +## Methods + +### FromPython\(PyObject\) + +Converts a Python object to a C# value of type T. + +```csharp +public static T FromPython(PyObject pyObject) +``` + +#### Parameters + +`pyObject` PyObject + +Python object to convert. + +#### Returns + + T + +C# value of type T. + +#### Type Parameters + +`T` + +Target C# type. + +#### Remarks + +Must be called within a GIL-acquired context. + +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown when conversion fails or the type is not supported. + +### ToPython\(object?\) + +Converts a C# value to a Python object. + +```csharp +public static PyObject ToPython(object? value) +``` + +#### Parameters + +`value` [object](https://learn.microsoft.com/dotnet/api/system.object)? + +C# value to convert. May be null. + +#### Returns + + PyObject + +Python object representation of the value. + +#### Remarks + +Must be called within a GIL-acquired context. + +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown when the value type is not supported for marshalling. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.md new file mode 100644 index 00000000..ff59ad27 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Marshalling.md @@ -0,0 +1,16 @@ +# Namespace Flowthru.Extensions.Python.Marshalling + +### Classes + + [ArrowMarshaller](Flowthru.Extensions.Python.Marshalling.ArrowMarshaller.md) + +Marshals tabular data between C# IEnumerable<T> and Apache Arrow RecordBatch. + + [ArrowSchemaMapper](Flowthru.Extensions.Python.Marshalling.ArrowSchemaMapper.md) + +Maps C# schema types to Apache Arrow schemas for tabular data interchange. + + [ScalarMarshaller](Flowthru.Extensions.Python.Marshalling.ScalarMarshaller.md) + +Handles bidirectional conversion between C# scalar values and Python objects. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Nodes.PythonNodeFactory.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Nodes.PythonNodeFactory.md new file mode 100644 index 00000000..8a7ef5ad --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Nodes.PythonNodeFactory.md @@ -0,0 +1,5582 @@ +# Class PythonNodeFactory + +Namespace: [](.md) +Assembly: Flowthru.Extensions.Python.dll + +Extension methods for adding Python nodes to pipelines. + +```csharp +public static class PythonNodeFactory +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PythonNodeFactory](.PythonNodeFactory.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Phases 2-5 implementation: +Hand-written 1×1 AddPythonStep(Phase 2-4). +Source-generated N×M overloads for multi-I/O support (Phase 5). +

+

+Usage: +

+
public static Pipeline Create(
+    Catalog catalog,
+    IPythonExecutor executor,
+    PythonRuntime runtime)
+{
+    return FlowBuilder.CreateFlow(pipeline =>
+    {
+        pipeline.AddPythonStep(
+            label: "Transform",
+            module: "my_nodes.transform",
+            function: "process",
+            input: catalog.RawData,
+            output: catalog.ProcessedData,
+            executor: executor,
+            runtime: runtime
+        );
+    });
+}
+

+Future phases: +

  • Phase 6: Async support
+

+ +## Methods + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, IItem, IPythonExecutor, string\) + +Adds a Python node with single input and single output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance. + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node. + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model"). +Must be resolvable via sys.path. + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module. + +`input` IItem + +Catalog entry providing input data. + +`output` IItem + +Catalog entry to store output data. + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function. + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description. + +#### Returns + + FlowBuilder + +This builder for method chaining. + +#### Type Parameters + +`TInput` + +Input type (must match catalog entry type). + +`TOutput` + +Output type (must match catalog entry type). + +#### Remarks + +

+Compile-time type safety: +Generic type parameters are inferred from catalog entries. +Mismatched types produce compiler errors. +

+

+Registration-time validation (Phase 4): +

  • Module is importable (exists, no syntax errors)
  • Function exists in module
  • @node decorator is present
+

+

+Pre-flight validation (Phase 4): +

  • Decorator schemas match C# generic types
  • Function signature arity is correct
  • Dry-run with 0-row data validates output structure
+

+ +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 1 input and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 1 input and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 1 input and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 1 input and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 1 input and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 1 input and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 1 input and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python node with 2 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` IItem + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 2 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 2 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 2 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 2 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 2 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 2 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 2 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python node with 3 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` IItem + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 3 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 3 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 3 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 3 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 3 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 3 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 3 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python node with 4 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` IItem + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 4 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 4 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 4 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 4 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 4 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 4 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 4 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python node with 5 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` IItem + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 5 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 5 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 5 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 5 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 5 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 5 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 5 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python node with 6 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` IItem + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 6 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 6 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 6 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 6 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 6 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 6 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 6 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python node with 7 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` IItem + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 7 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 7 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 7 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 7 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 7 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 7 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 7 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python node with 8 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` IItem + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 8 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 8 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 8 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 8 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 8 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 8 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python node with 8 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Pipeline builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this node + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog entry or tuple of catalog entries to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional node description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Nodes.PythonNodeWrapper-2.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Nodes.PythonNodeWrapper-2.md new file mode 100644 index 00000000..e3d43faf --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Nodes.PythonNodeWrapper-2.md @@ -0,0 +1,68 @@ +# Class PythonStepWrapper + +Namespace: [](.md) +Assembly: Flowthru.Extensions.Python.dll + +Thin wrapper that binds an to a specific module/function pair, +exposing it as a typed Func<TInput, TOutput> for use with the pipeline builder. + +```csharp +public sealed class PythonStepWrapper +``` + +#### Type Parameters + +`TInput` + +`TOutput` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PythonStepWrapper](.PythonStepWrapper\-2.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +All marshalling (scalar, tabular, bytes, multi-I/O tuples) is delegated to the executor. + +## Constructors + +### PythonStepWrapper\(IPythonExecutor, string, string\) + +```csharp +public PythonStepWrapper(IPythonExecutor executor, string moduleName, string functionName) +``` + +#### Parameters + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +`moduleName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +`functionName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +## Methods + +### GetTransform\(\) + +Gets the transformation function that invokes the Python node. + +```csharp +public Func GetTransform() +``` + +#### Returns + + [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +A function that takes TInput and returns TOutput. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Nodes.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Nodes.md new file mode 100644 index 00000000..4461083f --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Nodes.md @@ -0,0 +1,13 @@ +# Namespace + +### Classes + + [PythonNodeFactory](.PythonNodeFactory.md) + +Extension methods for adding Python nodes to pipelines. + + [PythonStepWrapper](.PythonStepWrapper\-2.md) + +Thin wrapper that binds an to a specific module/function pair, +exposing it as a typed Func<TInput, TOutput> for use with the pipeline builder. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.PythonExecutionMode.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.PythonExecutionMode.md new file mode 100644 index 00000000..1aa41f15 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.PythonExecutionMode.md @@ -0,0 +1,29 @@ +# Enum PythonExecutionMode + +Namespace: [Flowthru.Extensions.Python.Runtime](Flowthru.Extensions.Python.Runtime.md) +Assembly: Flowthru.Extensions.Python.dll + +Controls how Python step execution is isolated between FlowthruService instances. + +```csharp +public enum PythonExecutionMode +``` + +## Fields + +`InProcess = 0` + +Executes Python steps in the same process via Python.NET. +Fast (no IPC overhead), but all services share one Python interpreter, +sys.modules, and GIL. Use when co-hosted flows are known to be compatible. + + + +`Subprocess = 1` + +Executes Python steps in an isolated child process per FlowthruService. +Each service gets its own Python interpreter, venv, sys.path, and module cache. +Default for multi-service scenarios. + + + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.PythonRuntime.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.PythonRuntime.md new file mode 100644 index 00000000..03c9f4cb --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.PythonRuntime.md @@ -0,0 +1,141 @@ +# Class PythonRuntime + +Namespace: [Flowthru.Extensions.Python.Runtime](Flowthru.Extensions.Python.Runtime.md) +Assembly: Flowthru.Extensions.Python.dll + +Manages the Python runtime lifecycle and GIL context. + +```csharp +public sealed class PythonRuntime : IDisposable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PythonRuntime](Flowthru.Extensions.Python.Runtime.PythonRuntime.md) + +#### Implements + +[IDisposable](https://learn.microsoft.com/dotnet/api/system.idisposable) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Wraps Python.NET's initialization and shutdown. +Registered as a singleton in DI — only one Python runtime per application. +

+

+Thread-safety: Python.NET manages GIL acquisition internally. +Use to ensure thread-safe access to Python objects. +

+ +## Constructors + +### PythonRuntime\(PythonRuntimeOptions, ILogger\) + +Initializes a new instance of . + +```csharp +public PythonRuntime(PythonRuntimeOptions options, ILogger logger) +``` + +#### Parameters + +`options` [PythonRuntimeOptions](Flowthru.Extensions.Python.Runtime.PythonRuntimeOptions.md) + +`logger` [ILogger](https://learn.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger\-1)<[PythonRuntime](Flowthru.Extensions.Python.Runtime.PythonRuntime.md)\> + +## Methods + +### AcquireGil\(\) + +Acquires the Python GIL (Global Interpreter Lock). + +```csharp +public IDisposable AcquireGil() +``` + +#### Returns + + [IDisposable](https://learn.microsoft.com/dotnet/api/system.idisposable) + +A disposable GIL token. Dispose to release the GIL. + +#### Remarks + +

+Use this to bracket any Python.NET interop code: +

using (runtime.AcquireGil())
+{
+    dynamic module = Py.Import("my_module");
+    var result = module.my_function(42);
+}
+

+

+Python.NET's GIL management is thread-safe — multiple threads can acquire/release +the GIL, but only one thread executes Python code at a time. +

+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if has not been called. + +### Dispose\(\) + +Disposes the Python runtime wrapper. + +```csharp +public void Dispose() +``` + +#### Remarks + +

+Does not call PythonEngine.Shutdown() — Python.NET initializes globally once per process, +and explicit shutdown during process teardown is redundant. The OS reclaims all resources on exit. +

+

+Additionally, PythonEngine.Shutdown() attempts to serialize runtime state using +BinaryFormatter, which has been removed in .NET 10+ (see https://aka.ms/binaryformatter). +Since the serialized state is never restored, shutdown is both unnecessary and incompatible. +

+

+After disposal, will throw . +

+ +### Initialize\(\) + +Initializes the Python runtime if not already initialized. + +```csharp +public void Initialize() +``` + +#### Remarks + +

+Idempotent — safe to call multiple times. +Applies configuration from on first call. +

+

+Sets: +

  • PYTHONNET_PYDLL from resolved DLL path
  • PYTHONHOME from resolved venv path (if applicable)
  • sys.path from resolved module search paths
+

+ +#### Exceptions + + [InvalidOperationException](https://learn.microsoft.com/dotnet/api/system.invalidoperationexception) + +Thrown if Python runtime cannot be initialized (missing DLL, ABI mismatch, etc.). + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.PythonRuntimeOptions.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.PythonRuntimeOptions.md new file mode 100644 index 00000000..d8ade1cb --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.PythonRuntimeOptions.md @@ -0,0 +1,241 @@ +# Class PythonRuntimeOptions + +Namespace: [Flowthru.Extensions.Python.Runtime](Flowthru.Extensions.Python.Runtime.md) +Assembly: Flowthru.Extensions.Python.dll + +Configuration options for the Python runtime. + +```csharp +public sealed class PythonRuntimeOptions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PythonRuntimeOptions](Flowthru.Extensions.Python.Runtime.PythonRuntimeOptions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Follows the .NET Options pattern for environment-specific configuration. +Resolution order mirrors . +

+

+Developer workflow: +Run uv sync in your project directory during development to create .venv/. +During build, pyproject.toml, uv.lock, and .python-version are copied +to the output directory. On first run, the application automatically executes uv sync --frozen +in the output directory to materialize .venv/ in-place. +

+

+Auto-detection hierarchy: +

  1. Explicit value set via UsePython(opts => opts.PythonDll = "...")
  2. Environment variable (PYTHONNET_PYDLL for containers/CI)
  3. Explicit VenvPath override
  4. Auto-initialization via uv sync --frozen in output directory
  5. Fallback to VIRTUAL_ENV if set (compatibility with uv run)
+

+ +## Properties + +### ExecutionMode + +Controls whether Python steps run in the same process or an isolated child process. +Defaults to for per-service isolation. +Set to to opt in to shared-interpreter mode. + +```csharp +public PythonExecutionMode ExecutionMode { get; set; } +``` + +#### Property Value + + [PythonExecutionMode](Flowthru.Extensions.Python.Runtime.PythonExecutionMode.md) + +### ModuleSearchPaths + +Directories to add to Python's sys.path for module resolution. + +```csharp +public List ModuleSearchPaths { get; set; } +``` + +#### Property Value + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +#### Remarks + +

+If empty, resolved in order: +

  1. FLOWTHRU_PYTHON_PATH environment variable (colon/semicolon-separated)
  2. Project root (directory containing .csproj)
+

+

+Python steps at Flows/DataScience/Steps/train_model.py are referenced as +"Flows.DataScience.Steps.train_model" when the project root is in sys.path. +

+ +### PythonDll + +Path to the Python shared library (e.g., libpython3.12.so, python312.dll). + +```csharp +public string? PythonDll { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +

+If not set, resolved in order: +

  1. PYTHONNET_PYDLL environment variable (explicit override)
  2. Explicit VenvPath override
  3. Auto-materialized .venv/ via uv sync --frozen in output directory
  4. VIRTUAL_ENV environment variable (compatibility with uv run)
+

+

+Container deployments typically set PYTHONNET_PYDLL to point to system Python. +Local development and deployables use uv sync to create .venv/ in-place. +

+ +### UvPath + +Path to the uv executable for virtual environment initialization. + +```csharp +public string UvPath { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +

+Defaults to "uv" (PATH lookup). +Set this to an absolute path for non-standard installations. +

+

+Used by auto-initialization when pyproject.toml and uv.lock exist in +the output directory. To disable auto-initialization entirely, set VenvPath +explicitly or set PYTHONNET_PYDLL to point to system Python. +

+ +### VenvPath + +Path to the Python virtual environment (e.g., .venv/). + +```csharp +public string? VenvPath { get; set; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +

+If not set, resolved in order: +

  1. Auto-materialized via uv sync --frozen in output directory
  2. VIRTUAL_ENV environment variable
  3. None (uses system Python packages)
+

+

+Setting this property explicitly skips uv sync auto-initialization. +Useful for pre-built containers or custom venv management. +

+ +## Methods + +### GetResolvedModuleSearchPaths\(\) + +Gets the resolved Python module search paths. + +```csharp +public List GetResolvedModuleSearchPaths() +``` + +#### Returns + + [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\> + +#### Remarks + +Returns configured module search paths, or the executing assembly's base directory if none specified. +Python automatically includes site-packages from VIRTUAL_ENV when set. + +### GetResolvedPythonDll\(\) + +Gets the resolved Python DLL path using the auto-detection hierarchy. + +```csharp +public string GetResolvedPythonDll() +``` + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +

+Flowthru uses uv to manage Python environments. pyproject.toml, uv.lock, +and .python-version are copied to the output directory during build. On first run, +the runtime executes uv sync --frozen to materialize .venv/ in-place. +

+

+Attempts resolution in order: explicit value → PYTHONNET_PYDLL → explicit VenvPath → +uv sync auto-init → VIRTUAL_ENV. +Throws if no Python runtime is found. +

+ +### GetResolvedPythonExe\(\) + +Gets the Python executable path for subprocess execution. + +```csharp +public string GetResolvedPythonExe() +``` + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Remarks + +

+Resolves in the same order as but returns the interpreter +binary rather than the shared library. Used by SubprocessPythonExecutor to spawn +the worker process. +

+

+Falls back to python3 (Unix) or python (Windows) on PATH if no venv is found. +

+ +### GetResolvedVenvPath\(\) + +Gets the resolved virtual environment path. + +```csharp +public string? GetResolvedVenvPath() +``` + +#### Returns + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +#### Remarks + +

+Checks in order: +

  1. Explicit VenvPath property
  2. Auto-materialized .venv/ via uv sync in output directory
  3. VIRTUAL_ENV environment variable
+

+

+Returns null if no virtual environment is configured. +

+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.md new file mode 100644 index 00000000..06dad053 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Runtime.md @@ -0,0 +1,18 @@ +# Namespace Flowthru.Extensions.Python.Runtime + +### Classes + + [PythonRuntime](Flowthru.Extensions.Python.Runtime.PythonRuntime.md) + +Manages the Python runtime lifecycle and GIL context. + + [PythonRuntimeOptions](Flowthru.Extensions.Python.Runtime.PythonRuntimeOptions.md) + +Configuration options for the Python runtime. + +### Enums + + [PythonExecutionMode](Flowthru.Extensions.Python.Runtime.PythonExecutionMode.md) + +Controls how Python step execution is isolated between FlowthruService instances. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Services.FlowthruServiceBuilderExtensions.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Services.FlowthruServiceBuilderExtensions.md new file mode 100644 index 00000000..4fbe0926 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Services.FlowthruServiceBuilderExtensions.md @@ -0,0 +1,123 @@ +# Class FlowthruServiceBuilderExtensions + +Namespace: [Flowthru.Extensions.Python.Services](Flowthru.Extensions.Python.Services.md) +Assembly: Flowthru.Extensions.Python.dll + +Extension methods for integrating Python support with . + +```csharp +public static class FlowthruServiceBuilderExtensions +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[FlowthruServiceBuilderExtensions](Flowthru.Extensions.Python.Services.FlowthruServiceBuilderExtensions.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### UsePython\(FlowthruServiceBuilder\) + +Registers Python runtime and executor with default configuration. + +```csharp +public static FlowthruServiceBuilder UsePython(this FlowthruServiceBuilder builder) +``` + +#### Parameters + +`builder` FlowthruServiceBuilder + +The Flowthru service builder + +#### Returns + + FlowthruServiceBuilder + +The builder for method chaining + +#### Remarks + +

+Uses auto-detection for all configuration: +

  • Python DLL: PYTHONNET_PYDLL.venv/ → system Python
  • Virtual environment: FLOWTHRU_PYTHON_VENV.venv/ → none
  • Module search paths: FLOWTHRU_PYTHON_PATH → project root
+

+

+Example (auto-detection): +

services.AddFlowthru(flowthru =>
+{
+    flowthru
+        .RegisterCatalog<MyCatalog>()
+        .UsePython();  // Auto-detects .venv/, project root, etc.
+});
+

+ +### UsePython\(FlowthruServiceBuilder, Action\) + +Registers Python runtime and executor with custom configuration. + +```csharp +public static FlowthruServiceBuilder UsePython(this FlowthruServiceBuilder builder, Action configure) +``` + +#### Parameters + +`builder` FlowthruServiceBuilder + +The Flowthru service builder + +`configure` [Action](https://learn.microsoft.com/dotnet/api/system.action\-1)<[PythonRuntimeOptions](Flowthru.Extensions.Python.Runtime.PythonRuntimeOptions.md)\> + +Action to configure Python runtime options + +#### Returns + + FlowthruServiceBuilder + +The builder for method chaining + +#### Remarks + +

+Explicit configuration overrides auto-detection. +Use this for: +

  • Container deployments with non-standard Python paths
  • Custom module search paths
  • Multiple Python versions (explicit DLL path)
+

+

+Example (explicit configuration): +

services.AddFlowthru(flowthru =>
+{
+    flowthru
+        .RegisterCatalog<MyCatalog>()
+        .UsePython(python =>
+        {
+            python.PythonDll = "/usr/lib/x86_64-linux-gnu/libpython3.12.so";
+            python.ModuleSearchPaths.Add("Flows");
+            python.ModuleSearchPaths.Add("SharedSteps");
+        });
+});
+

+

+Example (environment-variable driven, for containers): +

services.AddFlowthru(flowthru =>
+{
+    flowthru
+        .RegisterCatalog<MyCatalog>()
+        .UsePython(python =>
+        {
+            // Reads PYTHONNET_PYDLL, FLOWTHRU_PYTHON_VENV, FLOWTHRU_PYTHON_PATH
+            // Auto-detection still active for unset properties
+        });
+});
+

+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Services.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Services.md new file mode 100644 index 00000000..0c1ba0c3 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Services.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Extensions.Python.Services + +### Classes + + [FlowthruServiceBuilderExtensions](Flowthru.Extensions.Python.Services.FlowthruServiceBuilderExtensions.md) + +Extension methods for integrating Python support with . + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Steps.PythonStepFactory.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Steps.PythonStepFactory.md new file mode 100644 index 00000000..aff78c93 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Steps.PythonStepFactory.md @@ -0,0 +1,5582 @@ +# Class PythonStepFactory + +Namespace: [Flowthru.Extensions.Python.Steps](Flowthru.Extensions.Python.Steps.md) +Assembly: Flowthru.Extensions.Python.dll + +Extension methods for adding Python steps to flows. + +```csharp +public static class PythonStepFactory +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PythonStepFactory](Flowthru.Extensions.Python.Steps.PythonStepFactory.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Phases 2-5 implementation: +Hand-written 1×1 AddPythonStep(Phase 2-4). +Source-generated N×M overloads for multi-I/O support (Phase 5). +

+

+Usage: +

+
public static Flow Create(
+    Catalog catalog,
+    IPythonExecutor executor,
+    PythonRuntime runtime)
+{
+    return FlowBuilder.CreateFlow(flow =>
+    {
+        flow.AddPythonStep(
+            label: "Transform",
+            module: "my_steps.transform",
+            function: "process",
+            input: catalog.RawData,
+            output: catalog.ProcessedData,
+            executor: executor,
+            runtime: runtime
+        );
+    });
+}
+

+Future phases: +

  • Phase 6: Async support
+

+ +## Methods + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, IItem, IPythonExecutor, string\) + +Adds a Python step with single input and single output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance. + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step. + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model"). +Must be resolvable via sys.path. + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module. + +`input` IItem + +Catalog item providing input data. + +`output` IItem + +Catalog item to store output data. + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function. + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description. + +#### Returns + + FlowBuilder + +This builder for method chaining. + +#### Type Parameters + +`TInput` + +Input type (must match catalog item type). + +`TOutput` + +Output type (must match catalog item type). + +#### Remarks + +

+Compile-time type safety: +Generic type parameters are inferred from catalog items. +Mismatched types produce compiler errors. +

+

+Registration-time validation (Phase 4): +

  • Module is importable (exists, no syntax errors)
  • Function exists in module
  • @step decorator is present
+

+

+Pre-flight validation (Phase 4): +

  • Decorator schemas match C# generic types
  • Function signature arity is correct
  • Dry-run with 0-row data validates output structure
+

+ +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 1 input and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 1 input and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 1 input and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 1 input and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 1 input and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 1 input and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, IItem, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 1 input and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, IItem input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` IItem + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python step with 2 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` IItem + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 2 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 2 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 2 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 2 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 2 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 2 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 2 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python step with 3 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` IItem + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 3 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 3 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 3 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 3 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 3 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 3 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 3 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python step with 4 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` IItem + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 4 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 4 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 4 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 4 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 4 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 4 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 4 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python step with 5 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` IItem + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 5 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 5 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 5 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 5 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 5 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 5 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 5 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python step with 6 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` IItem + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 6 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 6 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 6 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 6 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 6 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 6 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 6 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python step with 7 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` IItem + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 7 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 7 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 7 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 7 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 7 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 7 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 7 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IItem, IPythonExecutor, string\) + +Adds a Python step with 8 inputs and 1 output. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, IItem output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` IItem + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 8 inputs and 2 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 8 inputs and 3 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 8 inputs and 4 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 8 inputs and 5 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 8 inputs and 6 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 8 inputs and 7 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +### AddPythonStep\(FlowBuilder, string, string, string, \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\), IPythonExecutor, string\) + +Adds a Python step with 8 inputs and 8 outputs. + +```csharp +public static FlowBuilder AddPythonStep(this FlowBuilder builder, string label, string module, string function, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) input, (IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem) output, IPythonExecutor executor, string description = "") +``` + +#### Parameters + +`builder` FlowBuilder + +Flow builder instance + +`label` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Unique identifier for this step + +`module` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Dotted Python module name (e.g., "Flows.DataScience.train_model") + +`function` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Python function name within the module + +`input` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items providing input data + +`output` \(IItem, IItem, IItem, IItem, IItem, IItem, IItem, IItem\) + +Catalog item or tuple of catalog items to store output data + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for invoking the function + +`description` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Optional step description + +#### Returns + + FlowBuilder + +This builder for method chaining + +#### Type Parameters + +`TIn1` + +Input type 1 + +`TIn2` + +Input type 2 + +`TIn3` + +Input type 3 + +`TIn4` + +Input type 4 + +`TIn5` + +Input type 5 + +`TIn6` + +Input type 6 + +`TIn7` + +Input type 7 + +`TIn8` + +Input type 8 + +`TOut1` + +Output type 1 + +`TOut2` + +Output type 2 + +`TOut3` + +Output type 3 + +`TOut4` + +Output type 4 + +`TOut5` + +Output type 5 + +`TOut6` + +Output type 6 + +`TOut7` + +Output type 7 + +`TOut8` + +Output type 8 + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Steps.PythonStepWrapper-2.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Steps.PythonStepWrapper-2.md new file mode 100644 index 00000000..f501f137 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Steps.PythonStepWrapper-2.md @@ -0,0 +1,74 @@ +# Class PythonStepWrapper + +Namespace: [Flowthru.Extensions.Python.Steps](Flowthru.Extensions.Python.Steps.md) +Assembly: Flowthru.Extensions.Python.dll + +Thin wrapper that binds an to a specific module/function pair, +exposing it as a typed Func<TInput, TOutput> for use with the flow builder. + +```csharp +public sealed class PythonStepWrapper +``` + +#### Type Parameters + +`TInput` + +`TOutput` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PythonStepWrapper](Flowthru.Extensions.Python.Steps.PythonStepWrapper\-2.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +All marshalling (scalar, tabular, bytes, multi-I/O tuples) is delegated to the executor. + +## Constructors + +### PythonStepWrapper\(IPythonExecutor, string, string\) + +Initializes the wrapper with the executor and step information. + +```csharp +public PythonStepWrapper(IPythonExecutor executor, string moduleName, string functionName) +``` + +#### Parameters + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +`moduleName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +`functionName` [string](https://learn.microsoft.com/dotnet/api/system.string) + +#### Exceptions + + [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception) + +## Methods + +### GetTransform\(\) + +Gets the transformation function that invokes the Python step. + +```csharp +public Func GetTransform() +``` + +#### Returns + + [Func](https://learn.microsoft.com/dotnet/api/system.func\-2) + +A function that takes TInput and returns TOutput. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Steps.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Steps.md new file mode 100644 index 00000000..f64d2b03 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Steps.md @@ -0,0 +1,13 @@ +# Namespace Flowthru.Extensions.Python.Steps + +### Classes + + [PythonStepFactory](Flowthru.Extensions.Python.Steps.PythonStepFactory.md) + +Extension methods for adding Python steps to flows. + + [PythonStepWrapper](Flowthru.Extensions.Python.Steps.PythonStepWrapper\-2.md) + +Thin wrapper that binds an to a specific module/function pair, +exposing it as a typed Func<TInput, TOutput> for use with the flow builder. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Validation.PythonNodeValidator.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Validation.PythonNodeValidator.md new file mode 100644 index 00000000..e5dab47e --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Validation.PythonNodeValidator.md @@ -0,0 +1,120 @@ +# Class PythonNodeValidator + +Namespace: [Flowthru.Extensions.Python.Validation](Flowthru.Extensions.Python.Validation.md) +Assembly: Flowthru.Extensions.Python.dll + +Validation hook for Python nodes. + +```csharp +public class PythonNodeValidator : IPipelineValidationHook +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PythonNodeValidator](Flowthru.Extensions.Python.Validation.PythonNodeValidator.md) + +#### Implements + +IPipelineValidationHook + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Phase 4 pre-flight validation: +Validates Python nodes during pipeline pre-flight to catch schema mismatches, +incorrect function signatures, and structural errors before execution. +

+

+Checks performed: +

  • @node decorator schemas match C# generic type parameters
  • Function signature arity is correct for input count
  • Dry-run with 0-row data validates output structure
+

+

+Integration: +Register this hook via Pipeline.ValidationHooks during pipeline setup. +The hook is automatically invoked during Pipeline.ValidateExternalInputsAsync(). +

+ +## Constructors + +### PythonNodeValidator\(IPythonExecutor, PythonRuntime\) + +Initializes a new instance of . + +```csharp +public PythonNodeValidator(IPythonExecutor executor, PythonRuntime runtime) +``` + +#### Parameters + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for function inspection + +`runtime` [PythonRuntime](Flowthru.Extensions.Python.Runtime.PythonRuntime.md) + +Python runtime for GIL management + +## Methods + +### ValidateAsync\(Pipeline, CancellationToken\) + +Validates pipeline nodes during pre-flight checks. + +```csharp +public Task ValidateAsync(Pipeline pipeline, CancellationToken cancellationToken) +``` + +#### Parameters + +`pipeline` Pipeline + +The pipeline being validated + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token for async operations + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1) + +Validation result containing any errors found + +#### Remarks + +

+Implementations should: +

  • Never throw exceptions (return errors in ValidationResult)
  • Be idempotent (safe to call multiple times)
  • Be reasonably fast (executed during pre-flight, blocks pipeline start)
  • Only validate nodes they understand (ignore other node types)
+

+

+Example implementation (Python extension): +

+
public async Task<ValidationResult> ValidateAsync(
+  Pipeline pipeline,
+  CancellationToken cancellationToken)
+{
+  var result = ValidationResult.Success();
+
+  foreach (var node in pipeline.Nodes)
+  {
+    if (IsPythonNode(node))
+    {
+      var nodeResult = await ValidatePythonNode(node, cancellationToken);
+      result.Merge(nodeResult);
+    }
+  }
+
+  return result;
+}
+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Validation.PythonStepValidator.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Validation.PythonStepValidator.md new file mode 100644 index 00000000..58d80f28 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Validation.PythonStepValidator.md @@ -0,0 +1,120 @@ +# Class PythonStepValidator + +Namespace: [Flowthru.Extensions.Python.Validation](Flowthru.Extensions.Python.Validation.md) +Assembly: Flowthru.Extensions.Python.dll + +Validation hook for Python steps. + +```csharp +public class PythonStepValidator : IFlowValidationHook +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[PythonStepValidator](Flowthru.Extensions.Python.Validation.PythonStepValidator.md) + +#### Implements + +IFlowValidationHook + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+Phase 4 pre-flight validation: +Validates Python steps during flow pre-flight to catch schema mismatches, +incorrect function signatures, and structural errors before execution. +

+

+Checks performed: +

  • @step decorator schemas match C# generic type parameters
  • Function signature arity is correct for input count
  • Dry-run with 0-row data validates output structure
+

+

+Integration: +Register this hook via Flow.ValidationHooks during flow setup. +The hook is automatically invoked during Flow.ValidateExternalInputsAsync(). +

+ +## Constructors + +### PythonStepValidator\(IPythonExecutor, PythonRuntime\) + +Initializes a new instance of . + +```csharp +public PythonStepValidator(IPythonExecutor executor, PythonRuntime runtime) +``` + +#### Parameters + +`executor` [IPythonExecutor](Flowthru.Extensions.Python.Execution.IPythonExecutor.md) + +Python executor for function inspection + +`runtime` [PythonRuntime](Flowthru.Extensions.Python.Runtime.PythonRuntime.md) + +Python runtime for GIL management + +## Methods + +### ValidateAsync\(Flow, CancellationToken\) + +Validates flow steps during pre-flight checks. + +```csharp +public Task ValidateAsync(Flow flow, CancellationToken cancellationToken) +``` + +#### Parameters + +`flow` Flow + +The flow being validated + +`cancellationToken` [CancellationToken](https://learn.microsoft.com/dotnet/api/system.threading.cancellationtoken) + +Cancellation token for async operations + +#### Returns + + [Task](https://learn.microsoft.com/dotnet/api/system.threading.tasks.task\-1) + +Validation result containing any errors found + +#### Remarks + +

+Implementations should: +

  • Never throw exceptions (return errors in ValidationResult)
  • Be idempotent (safe to call multiple times)
  • Be reasonably fast (executed during pre-flight, blocks flow start)
  • Only validate steps they understand (ignore other step types)
+

+

+Example implementation (Python extension): +

+
public async Task<ValidationResult> ValidateAsync(
+  Flow flow,
+  CancellationToken cancellationToken)
+{
+  var result = ValidationResult.Success();
+
+  foreach (var step in flow.Steps)
+  {
+    if (IsPythonStep(step))
+    {
+      var stepResult = await ValidatePythonStep(step, cancellationToken);
+      result.Merge(stepResult);
+    }
+  }
+
+  return result;
+}
+ diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Validation.md b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Validation.md new file mode 100644 index 00000000..8546bf8e --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.Validation.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Extensions.Python.Validation + +### Classes + + [PythonStepValidator](Flowthru.Extensions.Python.Validation.PythonStepValidator.md) + +Validation hook for Python steps. + diff --git a/docs/reference/src/extensions/Flowthru.Extensions.Python/toc.yml b/docs/reference/src/extensions/Flowthru.Extensions.Python/toc.yml new file mode 100644 index 00000000..0b956f36 --- /dev/null +++ b/docs/reference/src/extensions/Flowthru.Extensions.Python/toc.yml @@ -0,0 +1,59 @@ +### YamlMime:TableOfContent +- name: Flowthru.Extensions.Python.Build + href: Flowthru.Extensions.Python.Build.md + items: + - name: Classes + - name: PythonSchemaExportTask + href: Flowthru.Extensions.Python.Build.PythonSchemaExportTask.md +- name: Flowthru.Extensions.Python.Execution + href: Flowthru.Extensions.Python.Execution.md + items: + - name: Classes + - name: PythonNetExecutor + href: Flowthru.Extensions.Python.Execution.PythonNetExecutor.md + - name: SubprocessPythonExecutor + href: Flowthru.Extensions.Python.Execution.SubprocessPythonExecutor.md + - name: Interfaces + - name: IPythonExecutor + href: Flowthru.Extensions.Python.Execution.IPythonExecutor.md +- name: Flowthru.Extensions.Python.Marshalling + href: Flowthru.Extensions.Python.Marshalling.md + items: + - name: Classes + - name: ArrowMarshaller + href: Flowthru.Extensions.Python.Marshalling.ArrowMarshaller.md + - name: ArrowSchemaMapper + href: Flowthru.Extensions.Python.Marshalling.ArrowSchemaMapper.md + - name: ScalarMarshaller + href: Flowthru.Extensions.Python.Marshalling.ScalarMarshaller.md +- name: Flowthru.Extensions.Python.Runtime + href: Flowthru.Extensions.Python.Runtime.md + items: + - name: Classes + - name: PythonRuntime + href: Flowthru.Extensions.Python.Runtime.PythonRuntime.md + - name: PythonRuntimeOptions + href: Flowthru.Extensions.Python.Runtime.PythonRuntimeOptions.md + - name: Enums + - name: PythonExecutionMode + href: Flowthru.Extensions.Python.Runtime.PythonExecutionMode.md +- name: Flowthru.Extensions.Python.Services + href: Flowthru.Extensions.Python.Services.md + items: + - name: Classes + - name: FlowthruServiceBuilderExtensions + href: Flowthru.Extensions.Python.Services.FlowthruServiceBuilderExtensions.md +- name: Flowthru.Extensions.Python.Steps + href: Flowthru.Extensions.Python.Steps.md + items: + - name: Classes + - name: PythonStepFactory + href: Flowthru.Extensions.Python.Steps.PythonStepFactory.md + - name: PythonStepWrapper + href: Flowthru.Extensions.Python.Steps.PythonStepWrapper-2.md +- name: Flowthru.Extensions.Python.Validation + href: Flowthru.Extensions.Python.Validation.md + items: + - name: Classes + - name: PythonStepValidator + href: Flowthru.Extensions.Python.Validation.PythonStepValidator.md diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.CurveFitting.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.CurveFitting.md new file mode 100644 index 00000000..b29384a0 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.CurveFitting.md @@ -0,0 +1,71 @@ +# Class CurveFitting + +Namespace: [Flowthru.Misc.ML.UMAP.Core](Flowthru.Misc.ML.UMAP.Core.md) +Assembly: Flowthru.Misc.ML.dll + +Helper for computing UMAP curve fitting parameters. + +```csharp +public static class CurveFitting +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[CurveFitting](Flowthru.Misc.ML.UMAP.Core.CurveFitting.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+UMAP uses a smooth curve to approximate the attractive force between points: +

+
weight(dist) = 1 / (1 + a * dist^(2b))
+

+Parameters a and b are fit to match an exponential decay curve +based on the spread and min_dist hyperparameters: +

+
  • For dist < min_dist: weight = 1.0 (fully connected)
  • For dist ≥ min_dist: weight = exp(-(dist - min_dist) / spread)
+

+Python UMAP reference: find_ab_params() in umap_.py (lines 1393-1408) +

+

+This implementation uses the Levenberg-Marquardt algorithm to match Python's +scipy.optimize.curve_fit behavior, ensuring identical parameter values. +

+ +## Methods + +### FindABParams\(float, float\) + +Computes curve parameters a and b from spread and min_dist using curve fitting. + +```csharp +public static (float a, float b) FindABParams(float spread, float minDist) +``` + +#### Parameters + +`spread` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Effective scale of embedded points. + +`minDist` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Minimum distance between embedded points. + +#### Returns + + \([float](https://learn.microsoft.com/dotnet/api/system.single) [a](https://learn.microsoft.com/dotnet/api/system.valuetuple\-system.single,system.single\-.a), [float](https://learn.microsoft.com/dotnet/api/system.single) [b](https://learn.microsoft.com/dotnet/api/system.valuetuple\-system.single,system.single\-.b)\) + +Tuple of (a, b) parameters. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.DataShape.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.DataShape.md new file mode 100644 index 00000000..70f15d30 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.DataShape.md @@ -0,0 +1,180 @@ +# Class DataShape + +Namespace: [Flowthru.Misc.ML.UMAP.Core](Flowthru.Misc.ML.UMAP.Core.md) +Assembly: Flowthru.Misc.ML.dll + +Describes the shape and characteristics of input data. +Used by strategy factories to select appropriate default strategies. + +```csharp +public sealed record DataShape : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[DataShape](Flowthru.Misc.ML.UMAP.Core.DataShape.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Analyzing data shape allows the UMAP pipeline to automatically choose +optimal strategies. For example: +- Small datasets (< 4096 samples) can use exact k-NN +- Large datasets benefit from approximate nearest neighbor search +- Sparse data requires specialized algorithms +- High-dimensional data may benefit from PCA initialization + +## Properties + +### EstimatedMemoryBytes + +Approximate memory footprint in bytes. + +```csharp +public long EstimatedMemoryBytes { get; init; } +``` + +#### Property Value + + [long](https://learn.microsoft.com/dotnet/api/system.int64) + +### Features + +Number of features (columns) in the dataset. + +```csharp +public required int Features { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### IsHighDimensional + +Whether the dataset is high-dimensional (typically > 100 features). +High-dimensional data may benefit from dimensionality reduction in initialization. + +```csharp +public bool IsHighDimensional { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### IsLargeDataset + +Whether the dataset is considered "large" (typically ≥ 4096 samples). +Large datasets should use approximate algorithms. + +```csharp +public bool IsLargeDataset { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### IsSmallDataset + +Whether the dataset is considered "small" (typically < 4096 samples). +Small datasets can use exact algorithms. + +```csharp +public bool IsSmallDataset { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### IsSparse + +Whether the data is stored in a sparse format. + +```csharp +public required bool IsSparse { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### IsVeryHighDimensional + +Whether the dataset is very high-dimensional (typically > 1000 features). +Very high-dimensional data may require PCA pre-processing. + +```csharp +public bool IsVeryHighDimensional { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### RecommendedEpochs + +Recommended number of training epochs based on dataset size. +Follows Python UMAP heuristics: 500 for small datasets, 200 for large. + +```csharp +public int RecommendedEpochs { get; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### RecommendedNeighbors + +Recommended number of nearest neighbors based on dataset size. +Follows Python UMAP heuristics: typically 15, but adjusted for very small datasets. + +```csharp +public int RecommendedNeighbors { get; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### Samples + +Number of samples (rows) in the dataset. + +```csharp +public required int Samples { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### SparsityRatio + +Sparsity ratio (proportion of zero elements) if applicable. +Only meaningful when is true. + +```csharp +public float? SparsityRatio { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)? + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md new file mode 100644 index 00000000..2d2f86a1 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md @@ -0,0 +1,110 @@ +# Interface IMetric + +Namespace: [Flowthru.Misc.ML.UMAP.Core.Markers](Flowthru.Misc.ML.UMAP.Core.Markers.md) +Assembly: Flowthru.Misc.ML.dll + +Base interface for distance metrics used in UMAP. +Provides the fundamental distance computation between points in high-dimensional space. + +```csharp +public interface IMetric +``` + +## Remarks + +

+Metrics define how distances are measured in the input space during k-NN search +and graph construction. Different metrics capture different notions of similarity. +

+

+Common implementations: Euclidean (L2), Manhattan (L1), Cosine (angular). +

+ +## Properties + +### DisconnectionDistance + +Maximum meaningful distance for bounded metrics, or null for unbounded metrics. +Used to handle disconnected components in the k-NN graph. + +```csharp +float? DisconnectionDistance { get; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)? + +#### Remarks + +

+Examples: +- Euclidean: null (unbounded) +- Cosine: 2.0 (ranges from 0 to 2) +- Jaccard: 1.0 (ranges from 0 to 1) +

+

+When set, distances at or beyond this value indicate maximally dissimilar points +that should be treated as disconnected in the manifold approximation. +

+ +### Name + +Human-readable name of the metric (e.g., "euclidean", "cosine"). +Used for logging and serialization. + +```csharp +string Name { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### SupportsAngularProjection + +Whether this metric benefits from angular (cosine-based) random projection forests. +Angular metrics (cosine, correlation) use different RP tree splits than Euclidean metrics. + +```csharp +bool SupportsAngularProjection { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### Distance\(ReadOnlySpan, ReadOnlySpan\) + +Compute the distance between two points. + +```csharp +float Distance(ReadOnlySpan x, ReadOnlySpan y) +``` + +#### Parameters + +`x` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +First point + +`y` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Second point + +#### Returns + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +Distance value (non-negative) + +#### Remarks + +Must satisfy metric properties: +- Non-negativity: Distance(x, y) ≥ 0 +- Identity: Distance(x, x) = 0 +- Symmetry: Distance(x, y) = Distance(y, x) +- Triangle inequality: Distance(x, z) ≤ Distance(x, y) + Distance(y, z) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Markers.IOutputMetric.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Markers.IOutputMetric.md new file mode 100644 index 00000000..73e7e28c --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Markers.IOutputMetric.md @@ -0,0 +1,71 @@ +# Interface IOutputMetric + +Namespace: [Flowthru.Misc.ML.UMAP.Core.Markers](Flowthru.Misc.ML.UMAP.Core.Markers.md) +Assembly: Flowthru.Misc.ML.dll + +Output space metric that provides distance gradients for layout optimization. +Required for embedding into non-Euclidean spaces (spherical, hyperbolic, toroidal, etc.). + +```csharp +public interface IOutputMetric : IMetric +``` + +#### Implements + +[IMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md) + +## Remarks + +

+During layout optimization (SGD phase), UMAP needs both the distance and its gradient +to update point positions. Standard Euclidean SGD has a specialized, highly optimized +implementation. For other output spaces, the generic SGD implementation requires gradients. +

+

+Examples of non-Euclidean output spaces: +- Spherical (haversine distance): Embeddings constrained to sphere surface +- Hyperbolic (Poincaré/hyperboloid): For hierarchical data +- Toroidal (wrap-around): For periodic data +

+ +## Methods + +### DistanceWithGradient\(ReadOnlySpan, ReadOnlySpan, out float, Span\) + +Compute distance and its gradient with respect to the first argument. +Used during stochastic gradient descent to optimize the embedding layout. + +```csharp +void DistanceWithGradient(ReadOnlySpan x, ReadOnlySpan y, out float distance, Span gradient) +``` + +#### Parameters + +`x` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +First point (the point being optimized) + +`y` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Second point (reference/anchor point) + +`distance` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Output: distance between x and y + +`gradient` [Span](https://learn.microsoft.com/dotnet/api/system.span\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Output: gradient of distance with respect to x (∂distance/∂x). +Must be pre-allocated by caller with length equal to dimensionality. + +#### Remarks + +

+The gradient represents the direction and magnitude of steepest increase in distance +when moving x. During SGD, we use this to either attract or repel points. +

+

+For Euclidean distance d = ||x - y||: +- ∇d/∂x = (x - y) / ||x - y|| +

+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Markers.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Markers.md new file mode 100644 index 00000000..41013615 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Markers.md @@ -0,0 +1,14 @@ +# Namespace Flowthru.Misc.ML.UMAP.Core.Markers + +### Interfaces + + [IMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md) + +Base interface for distance metrics used in UMAP. +Provides the fundamental distance computation between points in high-dimensional space. + + [IOutputMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IOutputMetric.md) + +Output space metric that provides distance gradients for layout optimization. +Required for embedding into non-Euclidean spaces (spherical, hyperbolic, toroidal, etc.). + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapFitResult.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapFitResult.md new file mode 100644 index 00000000..32a4531b --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapFitResult.md @@ -0,0 +1,141 @@ +# Class UmapFitResult + +Namespace: [Flowthru.Misc.ML.UMAP.Core](Flowthru.Misc.ML.UMAP.Core.md) +Assembly: Flowthru.Misc.ML.dll + +Result of the complete UMAP FitTransform operation. +Contains the final embedding and all intermediate results. + +```csharp +public sealed record UmapFitResult : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[UmapFitResult](Flowthru.Misc.ML.UMAP.Core.UmapFitResult.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Constructors + +### UmapFitResult\(Matrix, UmapGraphResult, LayoutInitResult, SamplingScheduleResult, LayoutOptimizationResult, UmapRuntimeReport\) + +Result of the complete UMAP FitTransform operation. +Contains the final embedding and all intermediate results. + +```csharp +public UmapFitResult(Matrix Embedding, UmapGraphResult GraphResult, LayoutInitResult LayoutInitResult, SamplingScheduleResult SamplingScheduleResult, LayoutOptimizationResult OptimizationResult, UmapRuntimeReport RuntimeReport) +``` + +#### Parameters + +`Embedding` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Final optimized low-dimensional embedding. Shape: (n_samples, n_components) + +`GraphResult` [UmapGraphResult](Flowthru.Misc.ML.UMAP.Core.UmapGraphResult.md) + +Intermediate result from graph construction (Phases 1-4). + +`LayoutInitResult` [LayoutInitResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.LayoutInitResult.md) + +Intermediate result from layout initialization (Phase 5). + +`SamplingScheduleResult` [SamplingScheduleResult](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.SamplingScheduleResult.md) + +Intermediate result from sampling schedule computation (Phase 6). + +`OptimizationResult` [LayoutOptimizationResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.LayoutOptimizationResult.md) + +Result from layout optimization (Phase 7). + +`RuntimeReport` [UmapRuntimeReport](Flowthru.Misc.ML.UMAP.Core.UmapRuntimeReport.md) + +Performance timing metrics for each UMAP phase. + +## Properties + +### Embedding + +Final optimized low-dimensional embedding. Shape: (n_samples, n_components) + +```csharp +public Matrix Embedding { get; init; } +``` + +#### Property Value + + Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +### GraphResult + +Intermediate result from graph construction (Phases 1-4). + +```csharp +public UmapGraphResult GraphResult { get; init; } +``` + +#### Property Value + + [UmapGraphResult](Flowthru.Misc.ML.UMAP.Core.UmapGraphResult.md) + +### LayoutInitResult + +Intermediate result from layout initialization (Phase 5). + +```csharp +public LayoutInitResult LayoutInitResult { get; init; } +``` + +#### Property Value + + [LayoutInitResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.LayoutInitResult.md) + +### OptimizationResult + +Result from layout optimization (Phase 7). + +```csharp +public LayoutOptimizationResult OptimizationResult { get; init; } +``` + +#### Property Value + + [LayoutOptimizationResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.LayoutOptimizationResult.md) + +### RuntimeReport + +Performance timing metrics for each UMAP phase. + +```csharp +public UmapRuntimeReport RuntimeReport { get; init; } +``` + +#### Property Value + + [UmapRuntimeReport](Flowthru.Misc.ML.UMAP.Core.UmapRuntimeReport.md) + +### SamplingScheduleResult + +Intermediate result from sampling schedule computation (Phase 6). + +```csharp +public SamplingScheduleResult SamplingScheduleResult { get; init; } +``` + +#### Property Value + + [SamplingScheduleResult](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.SamplingScheduleResult.md) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md new file mode 100644 index 00000000..8e98313c --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md @@ -0,0 +1,220 @@ +# Class UmapFlowBuilder + +Namespace: [Flowthru.Misc.ML.UMAP.Core](Flowthru.Misc.ML.UMAP.Core.md) +Assembly: Flowthru.Misc.ML.dll + +```csharp +public sealed class UmapFlowBuilder +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### FitTransform\(float\[\]\[\]\) + +Fits UMAP and transforms data in one step. +Auto-selects strategies based on data characteristics if not explicitly set. + +```csharp +public float[][] FitTransform(float[][] data) +``` + +#### Parameters + +`data` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Input data as jagged array (n_samples, n_features) + +#### Returns + + [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Low-dimensional embedding (n_samples, n_components) + +### FitTransform\(Matrix\) + +Fits UMAP and transforms data in one step. +Auto-selects strategies based on data characteristics if not explicitly set. + +```csharp +public Matrix FitTransform(Matrix data) +``` + +#### Parameters + +`data` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Input data matrix (n_samples, n_features) + +#### Returns + + Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Low-dimensional embedding (n_samples, n_components) + +#### Remarks + +TODO: Consider deprecating this overload. Matrix<float> adds virtual call overhead +and intermediate allocations compared to float[][]. Only kept for compatibility with +SpectralInit which uses Math.Net for eigenvalue decomposition. + +### FitTransformWithReport\(Matrix\) + +Fits UMAP and transforms data in one step, returning full result including runtime report. +Auto-selects strategies based on data characteristics if not explicitly set. + +```csharp +public UmapFitResult FitTransformWithReport(Matrix data) +``` + +#### Parameters + +`data` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Input data matrix (n_samples, n_features) + +#### Returns + + [UmapFitResult](Flowthru.Misc.ML.UMAP.Core.UmapFitResult.md) + +Complete UMAP result including embedding and runtime metrics + +### WithGraphRefinement\(IGraphRefinementStrategy\) + +```csharp +public UmapFlowBuilder WithGraphRefinement(IGraphRefinementStrategy strategy) +``` + +#### Parameters + +`strategy` [IGraphRefinementStrategy](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.IGraphRefinementStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithInputMetric\(IMetric\) + +```csharp +public UmapFlowBuilder WithInputMetric(IMetric metric) +``` + +#### Parameters + +`metric` [IMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithLayoutInit\(ILayoutInitStrategy\) + +```csharp +public UmapFlowBuilder WithLayoutInit(ILayoutInitStrategy strategy) +``` + +#### Parameters + +`strategy` [ILayoutInitStrategy](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.ILayoutInitStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithLayoutOptimization\(ILayoutOptimizationStrategy\) + +```csharp +public UmapFlowBuilder WithLayoutOptimization(ILayoutOptimizationStrategy strategy) +``` + +#### Parameters + +`strategy` [ILayoutOptimizationStrategy](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.ILayoutOptimizationStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithLocalMetric\(ILocalMetricStrategy\) + +```csharp +public UmapFlowBuilder WithLocalMetric(ILocalMetricStrategy strategy) +``` + +#### Parameters + +`strategy` [ILocalMetricStrategy](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.ILocalMetricStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithMembershipStrength\(IMembershipStrengthStrategy\) + +```csharp +public UmapFlowBuilder WithMembershipStrength(IMembershipStrengthStrategy strategy) +``` + +#### Parameters + +`strategy` [IMembershipStrengthStrategy](Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.IMembershipStrengthStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithNeighborSearch\(INeighborSearchStrategy\) + +```csharp +public UmapFlowBuilder WithNeighborSearch(INeighborSearchStrategy strategy) +``` + +#### Parameters + +`strategy` [INeighborSearchStrategy](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.INeighborSearchStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithOutputMetric\(IOutputMetric\) + +```csharp +public UmapFlowBuilder WithOutputMetric(IOutputMetric metric) +``` + +#### Parameters + +`metric` [IOutputMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IOutputMetric.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithSamplingSchedule\(ISamplingScheduleStrategy\) + +```csharp +public UmapFlowBuilder WithSamplingSchedule(ISamplingScheduleStrategy strategy) +``` + +#### Parameters + +`strategy` [ISamplingScheduleStrategy](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.ISamplingScheduleStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapGraphResult.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapGraphResult.md new file mode 100644 index 00000000..fcd9b7af --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapGraphResult.md @@ -0,0 +1,139 @@ +# Class UmapGraphResult + +Namespace: [Flowthru.Misc.ML.UMAP.Core](Flowthru.Misc.ML.UMAP.Core.md) +Assembly: Flowthru.Misc.ML.dll + +Result of computing the UMAP graph (phases 1-3). + +```csharp +public sealed record UmapGraphResult : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[UmapGraphResult](Flowthru.Misc.ML.UMAP.Core.UmapGraphResult.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Constructors + +### UmapGraphResult\(SparseMatrix, int\[\]\[\], float\[\]\[\], float\[\], float\[\], object?\) + +Result of computing the UMAP graph (phases 1-3). + +```csharp +public UmapGraphResult(SparseMatrix Graph, int[][] KnnIndices, float[][] KnnDistances, float[] Sigmas, float[] Rhos, object? SearchIndex) +``` + +#### Parameters + +`Graph` SparseMatrix + +Fuzzy simplicial set as a sparse symmetric matrix. + +`KnnIndices` [int](https://learn.microsoft.com/dotnet/api/system.int32)\[\]\[\] + +K-nearest neighbor indices for each point. + +`KnnDistances` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +K-nearest neighbor distances for each point. + +`Sigmas` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Bandwidth parameters from local metric computation. + +`Rhos` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Local connectivity distances from local metric computation. + +`SearchIndex` [object](https://learn.microsoft.com/dotnet/api/system.object)? + +Optional search index for transform operations. + +## Properties + +### Graph + +Fuzzy simplicial set as a sparse symmetric matrix. + +```csharp +public SparseMatrix Graph { get; init; } +``` + +#### Property Value + + SparseMatrix + +### KnnDistances + +K-nearest neighbor distances for each point. + +```csharp +public float[][] KnnDistances { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +### KnnIndices + +K-nearest neighbor indices for each point. + +```csharp +public int[][] KnnIndices { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32)\[\]\[\] + +### Rhos + +Local connectivity distances from local metric computation. + +```csharp +public float[] Rhos { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +### SearchIndex + +Optional search index for transform operations. + +```csharp +public object? SearchIndex { get; init; } +``` + +#### Property Value + + [object](https://learn.microsoft.com/dotnet/api/system.object)? + +### Sigmas + +Bandwidth parameters from local metric computation. + +```csharp +public float[] Sigmas { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapParameters.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapParameters.md new file mode 100644 index 00000000..dbaad0a8 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapParameters.md @@ -0,0 +1,341 @@ +# Class UmapParameters + +Namespace: [Flowthru.Misc.ML.UMAP.Core](Flowthru.Misc.ML.UMAP.Core.md) +Assembly: Flowthru.Misc.ML.dll + +Core parameters for UMAP algorithm configuration. +These parameters control the mathematical behavior of the algorithm across all strategies. + +```csharp +public sealed record UmapParameters : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[UmapParameters](Flowthru.Misc.ML.UMAP.Core.UmapParameters.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +This record contains the fundamental UMAP hyperparameters that affect the global +structure of the embedding. Strategy-specific parameters are configured on individual +strategy instances. + +## Properties + +### A + +Curve fitting parameter 'a' for the low-dimensional fuzzy simplicial set. +If null, automatically computed from and . + +```csharp +public float? A { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)? + +#### Remarks + +Default: null (auto-compute). Manual setting is for advanced use only. +This parameter controls the attractive force curve in the embedding space. + +### B + +Curve fitting parameter 'b' for the low-dimensional fuzzy simplicial set. +If null, automatically computed from and . + +```csharp +public float? B { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)? + +#### Remarks + +Default: null (auto-compute). Manual setting is for advanced use only. +This parameter controls the attractive force curve in the embedding space. + +### LearningRate + +Initial learning rate for stochastic gradient descent. + +```csharp +public float LearningRate { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +#### Remarks + +Default: 1.0. Range: (0, ∞). Typical values: 0.5-2.0. +Learning rate decays linearly to 0 over training epochs. + +### LocalConnectivity + +Local connectivity required at the manifold level. +Number of nearest neighbors assumed to be connected locally. + +```csharp +public float LocalConnectivity { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +#### Remarks + +Default: 1.0. Range: [1, numberOfNeighbors]. Typical values: 1.0-5.0. +Higher values increase local connectivity, making the manifold more connected. +Should not exceed the local intrinsic dimension of the manifold. + +### MinDist + +Effective minimum distance between embedded points. +Controls how tightly points are packed in clusters. + +```csharp +public float MinDist { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +#### Remarks + +Default: 0.1. Range: [0, spread]. Typical values: 0.0-0.5. +- 0.0: Dense, tightly packed clusters +- 0.1: Balanced (default) +- 0.3-0.5: More spread out, emphasizes separation + +### NegativeSampleRate + +Number of negative samples per positive sample during optimization. + +```csharp +public int NegativeSampleRate { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +Default: 5. Range: [1, ∞). Typical values: 5-20. +Higher values = stronger repulsive force but slower training. + +### NumberOfComponents + +Dimensionality of the target embedding space. + +```csharp +public int NumberOfComponents { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +Default: 2 (for visualization). Range: [1, ∞). Typical values: 2-100. +- 2D: Visualization and exploratory analysis +- 3D: Interactive 3D visualization +- Higher: Feature extraction, downstream ML tasks + +### NumberOfEpochs + +Number of optimization epochs (training iterations). +If null, automatically determined based on dataset size. + +```csharp +public int? NumberOfEpochs { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32)? + +#### Remarks + +Default: null (auto). If set, range: [0, ∞). Auto values: 500 (small data), 200 (large data). +More epochs = better convergence but longer training time. + +### NumberOfNeighbors + +Number of nearest neighbors to consider for manifold approximation. +Larger values result in more global structure, smaller values preserve local details. + +```csharp +public int NumberOfNeighbors { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +Default: 15. Range: [2, ∞). Typical values: 5-50. +- Small values (5-10): Emphasize local structure, fine details +- Medium values (15-30): Balanced local and global structure +- Large values (50+): Emphasize global structure, may lose fine details + +### ProgressReporter + +Optional progress reporter for programmatic progress tracking. + +```csharp +public IProgress? ProgressReporter { get; init; } +``` + +#### Property Value + + [IProgress](https://learn.microsoft.com/dotnet/api/system.iprogress\-1)<[UmapProgress](Flowthru.Misc.ML.UMAP.Core.UmapProgress.md)\>? + +### RandomSeed + +Random seed for reproducible results. +If null, uses non-deterministic randomization. + +```csharp +public int? RandomSeed { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32)? + +### RepulsionStrength + +Weight applied to negative samples in optimization. +Controls repulsive force between non-neighboring points. + +```csharp +public float RepulsionStrength { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +#### Remarks + +Default: 1.0. Range: [0, ∞). Typical values: 0.5-2.0. +- Lower values: Less repulsion, denser embedding +- Higher values: More repulsion, more spread out + +### SetOpMixRatio + +Interpolation between fuzzy union and intersection for combining local simplicial sets. + +```csharp +public float SetOpMixRatio { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +#### Remarks + +Default: 1.0 (pure fuzzy union). Range: [0, 1]. +- 1.0: Pure fuzzy union (standard UMAP) +- 0.0: Pure fuzzy intersection (more conservative connectivity) +- 0.5: Balanced between union and intersection + +### Spread + +Effective scale of embedded points. +Works with to control clustering vs. dispersion. + +```csharp +public float Spread { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +#### Remarks + +Default: 1.0. Range: (0, ∞). Typical values: 0.5-2.0. +Controls the overall scale at which embedded points spread out. + +### Verbosity + +Verbosity level for progress reporting. + +```csharp +public int Verbosity { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +0 = Silent, 1 = Basic progress, 2 = Detailed progress + +## Methods + +### GetA\(\) + +Gets the curve parameter 'a', computing it from spread and min_dist if not explicitly set. + +```csharp +public float GetA() +``` + +#### Returns + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +### GetB\(\) + +Gets the curve parameter 'b', computing it from spread and min_dist if not explicitly set. + +```csharp +public float GetB() +``` + +#### Returns + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +### Validate\(\) + +Validates the parameters and throws if any are invalid. + +```csharp +public void Validate() +``` + +#### Exceptions + + [ArgumentException](https://learn.microsoft.com/dotnet/api/system.argumentexception) + +Thrown when parameters are out of valid ranges. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapPipeline.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapPipeline.md new file mode 100644 index 00000000..e21e76b3 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapPipeline.md @@ -0,0 +1,70 @@ +# Class UmapPipeline + +Namespace: [Flowthru.Misc.ML.UMAP.Core](Flowthru.Misc.ML.UMAP.Core.md) +Assembly: Flowthru.Misc.ML.dll + +Fluent builder for UMAP pipelines with automatic strategy selection. + +```csharp +public static class UmapPipeline +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[UmapPipeline](Flowthru.Misc.ML.UMAP.Core.UmapPipeline.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This provides a low skill floor (simple defaults) with a high skill ceiling (full customization). +Strategies are resolved lazily at FitTransform() time based on data characteristics. +

+

+Usage patterns: +

+
// Beginner: Full auto-configuration (Euclidean metric)
+var result = UmapPipeline.Create().FitTransform(data);
+
+// Intermediate: Custom metric
+var result = UmapPipeline.Create()
+  .WithInputMetric(CosineMetric.Instance)
+  .FitTransform(data);
+
+// Advanced: Custom strategies for testing/benchmarking
+var result = UmapPipeline.Create()
+  .WithNeighborSearch(new NNDescentSearch { MaxIterations = 50 })
+  .FitTransform(data);
+ +## Methods + +### Create\(UmapParameters?\) + +Creates a new UMAP pipeline with default settings. +Euclidean metric is used by default, and strategies will be auto-selected based on data shape. + +```csharp +public static UmapFlowBuilder Create(UmapParameters? parameters = null) +``` + +#### Parameters + +`parameters` [UmapParameters](Flowthru.Misc.ML.UMAP.Core.UmapParameters.md)? + +UMAP hyperparameters (n_neighbors, min_dist, etc.). +If null, uses defaults appropriate for the data. + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapPipelineBuilder.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapPipelineBuilder.md new file mode 100644 index 00000000..7addc9a5 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapPipelineBuilder.md @@ -0,0 +1,220 @@ +# Class UmapFlowBuilder + +Namespace: [Flowthru.Misc.ML.UMAP.Core](Flowthru.Misc.ML.UMAP.Core.md) +Assembly: Flowthru.Misc.ML.dll + +```csharp +public sealed class UmapFlowBuilder +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### FitTransform\(float\[\]\[\]\) + +Fits UMAP and transforms data in one step. +Auto-selects strategies based on data characteristics if not explicitly set. + +```csharp +public float[][] FitTransform(float[][] data) +``` + +#### Parameters + +`data` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Input data as jagged array (n_samples, n_features) + +#### Returns + + [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Low-dimensional embedding (n_samples, n_components) + +### FitTransform\(Matrix\) + +Fits UMAP and transforms data in one step. +Auto-selects strategies based on data characteristics if not explicitly set. + +```csharp +public Matrix FitTransform(Matrix data) +``` + +#### Parameters + +`data` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Input data matrix (n_samples, n_features) + +#### Returns + + Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Low-dimensional embedding (n_samples, n_components) + +#### Remarks + +TODO: Consider deprecating this overload. Matrix<float> adds virtual call overhead +and intermediate allocations compared to float[][]. Only kept for compatibility with +SpectralInit which uses Math.Net for eigenvalue decomposition. + +### FitTransformWithReport\(Matrix\) + +Fits UMAP and transforms data in one step, returning full result including runtime report. +Auto-selects strategies based on data characteristics if not explicitly set. + +```csharp +public UmapFitResult FitTransformWithReport(Matrix data) +``` + +#### Parameters + +`data` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Input data matrix (n_samples, n_features) + +#### Returns + + [UmapFitResult](Flowthru.Misc.ML.UMAP.Core.UmapFitResult.md) + +Complete UMAP result including embedding and runtime metrics + +### WithGraphRefinement\(IGraphRefinementStrategy\) + +```csharp +public UmapFlowBuilder WithGraphRefinement(IGraphRefinementStrategy strategy) +``` + +#### Parameters + +`strategy` [IGraphRefinementStrategy](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.IGraphRefinementStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithInputMetric\(IMetric\) + +```csharp +public UmapFlowBuilder WithInputMetric(IMetric metric) +``` + +#### Parameters + +`metric` [IMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithLayoutInit\(ILayoutInitStrategy\) + +```csharp +public UmapFlowBuilder WithLayoutInit(ILayoutInitStrategy strategy) +``` + +#### Parameters + +`strategy` [ILayoutInitStrategy](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.ILayoutInitStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithLayoutOptimization\(ILayoutOptimizationStrategy\) + +```csharp +public UmapFlowBuilder WithLayoutOptimization(ILayoutOptimizationStrategy strategy) +``` + +#### Parameters + +`strategy` [ILayoutOptimizationStrategy](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.ILayoutOptimizationStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithLocalMetric\(ILocalMetricStrategy\) + +```csharp +public UmapFlowBuilder WithLocalMetric(ILocalMetricStrategy strategy) +``` + +#### Parameters + +`strategy` [ILocalMetricStrategy](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.ILocalMetricStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithMembershipStrength\(IMembershipStrengthStrategy\) + +```csharp +public UmapFlowBuilder WithMembershipStrength(IMembershipStrengthStrategy strategy) +``` + +#### Parameters + +`strategy` [IMembershipStrengthStrategy](Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.IMembershipStrengthStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithNeighborSearch\(INeighborSearchStrategy\) + +```csharp +public UmapFlowBuilder WithNeighborSearch(INeighborSearchStrategy strategy) +``` + +#### Parameters + +`strategy` [INeighborSearchStrategy](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.INeighborSearchStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithOutputMetric\(IOutputMetric\) + +```csharp +public UmapFlowBuilder WithOutputMetric(IOutputMetric metric) +``` + +#### Parameters + +`metric` [IOutputMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IOutputMetric.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + +### WithSamplingSchedule\(ISamplingScheduleStrategy\) + +```csharp +public UmapFlowBuilder WithSamplingSchedule(ISamplingScheduleStrategy strategy) +``` + +#### Parameters + +`strategy` [ISamplingScheduleStrategy](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.ISamplingScheduleStrategy.md) + +#### Returns + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapProgress.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapProgress.md new file mode 100644 index 00000000..d152f2ff --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapProgress.md @@ -0,0 +1,91 @@ +# Class UmapProgress + +Namespace: [Flowthru.Misc.ML.UMAP.Core](Flowthru.Misc.ML.UMAP.Core.md) +Assembly: Flowthru.Misc.ML.dll + +Progress information reported during UMAP execution. + +```csharp +public sealed record UmapProgress : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[UmapProgress](Flowthru.Misc.ML.UMAP.Core.UmapProgress.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### CurrentEpoch + +Current epoch number, if applicable (during optimization). + +```csharp +public int? CurrentEpoch { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32)? + +### Details + +Optional detailed status message. + +```csharp +public string? Details { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string)? + +### Progress + +Progress within the current stage as a fraction [0.0, 1.0]. + +```csharp +public required float Progress { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +### Stage + +Name of the current pipeline stage (e.g., "K-NN", "Graph Construction", "Optimization"). + +```csharp +public required string Stage { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### TotalEpochs + +Total number of epochs, if applicable (during optimization). + +```csharp +public int? TotalEpochs { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32)? + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapRuntimeReport.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapRuntimeReport.md new file mode 100644 index 00000000..de36a588 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.UmapRuntimeReport.md @@ -0,0 +1,76 @@ +# Class UmapRuntimeReport + +Namespace: [Flowthru.Misc.ML.UMAP.Core](Flowthru.Misc.ML.UMAP.Core.md) +Assembly: Flowthru.Misc.ML.dll + +Runtime performance report for UMAP execution. + +```csharp +public sealed record UmapRuntimeReport : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[UmapRuntimeReport](Flowthru.Misc.ML.UMAP.Core.UmapRuntimeReport.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Generic schema capturing timing metrics for each UMAP algorithmic phase. +Does not include Flowthru serialization markers to keep it framework-agnostic. + +## Properties + +### Timings + +Timing measurements for each UMAP phase. +Key is the stage name, value is elapsed time in milliseconds. + +```csharp +public Dictionary Timings { get; init; } +``` + +#### Property Value + + [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [int](https://learn.microsoft.com/dotnet/api/system.int32)\> + +#### Remarks + +Expected stages: +- "NeighborSearch" - Phase 1: k-NN graph construction +- "LocalMetric" - Phase 2: Local metric parameter computation +- "GraphConstruction" - Phase 3: Fuzzy simplicial set construction +- "GraphRefinement" - Phase 4: Graph refinement (optional) +- "LayoutInit" - Phase 5: Low-dimensional layout initialization +- "SamplingSchedule" - Phase 6: Edge sampling schedule computation +- "LayoutOptimization" - Phase 7: Stochastic gradient descent optimization + +### TotalTimeMs + +Total elapsed time for the complete FitTransform operation, in milliseconds. + +```csharp +public int TotalTimeMs { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +Sum of all individual phase timings. Useful for quick performance assessment. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.CosineMetric.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.CosineMetric.md new file mode 100644 index 00000000..461ef9a9 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.CosineMetric.md @@ -0,0 +1,124 @@ +# Class CosineMetric + +Namespace: [Flowthru.Misc.ML.UMAP.Core.Utils](Flowthru.Misc.ML.UMAP.Core.Utils.md) +Assembly: Flowthru.Misc.ML.dll + +Cosine distance metric (angular distance). + +```csharp +public sealed class CosineMetric : IMetric +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[CosineMetric](Flowthru.Misc.ML.UMAP.Core.Utils.CosineMetric.md) + +#### Implements + +[IMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Cosine distance measures the angle between vectors: +d(x, y) = 1 - (x·y) / (||x|| ||y||) + +Range: [0, 2] where 0 = identical direction, 1 = orthogonal, 2 = opposite direction. +Ignores magnitude, only considers direction. + +## Properties + +### DisconnectionDistance + +Maximum meaningful distance for bounded metrics, or null for unbounded metrics. +Used to handle disconnected components in the k-NN graph. + +```csharp +public float? DisconnectionDistance { get; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)? + +#### Remarks + +

+Examples: +- Euclidean: null (unbounded) +- Cosine: 2.0 (ranges from 0 to 2) +- Jaccard: 1.0 (ranges from 0 to 1) +

+

+When set, distances at or beyond this value indicate maximally dissimilar points +that should be treated as disconnected in the manifold approximation. +

+ +### Instance + +Singleton instance of Cosine metric. +Use this to avoid allocations. + +```csharp +public static CosineMetric Instance { get; } +``` + +#### Property Value + + [CosineMetric](Flowthru.Misc.ML.UMAP.Core.Utils.CosineMetric.md) + +### Name + +Human-readable name of the metric (e.g., "euclidean", "cosine"). +Used for logging and serialization. + +```csharp +public string Name { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### SupportsAngularProjection + +Whether this metric benefits from angular (cosine-based) random projection forests. +Angular metrics (cosine, correlation) use different RP tree splits than Euclidean metrics. + +```csharp +public bool SupportsAngularProjection { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### Distance\(ReadOnlySpan, ReadOnlySpan\) + +Compute cosine distance: 1 - (dot product / product of norms). + +```csharp +public float Distance(ReadOnlySpan x, ReadOnlySpan y) +``` + +#### Parameters + +`x` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +`y` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +#### Returns + + [float](https://learn.microsoft.com/dotnet/api/system.single) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.CustomMetric.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.CustomMetric.md new file mode 100644 index 00000000..8113125d --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.CustomMetric.md @@ -0,0 +1,150 @@ +# Class CustomMetric + +Namespace: [Flowthru.Misc.ML.UMAP.Core.Utils](Flowthru.Misc.ML.UMAP.Core.Utils.md) +Assembly: Flowthru.Misc.ML.dll + +Custom metric wrapper for user-defined distance functions. + +```csharp +public sealed class CustomMetric : IMetric +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[CustomMetric](Flowthru.Misc.ML.UMAP.Core.Utils.CustomMetric.md) + +#### Implements + +[IMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Allows users to provide arbitrary distance functions while maintaining +the IMetric interface contract. Useful for experimentation and custom metrics. + +## Constructors + +### CustomMetric\(string, Func, ReadOnlySpan, float\>, float?, bool\) + +Creates a custom metric from a distance function. + +```csharp +public CustomMetric(string name, Func, ReadOnlySpan, float> distanceFunc, float? disconnectionDistance = null, bool supportsAngularProjection = false) +``` + +#### Parameters + +`name` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Human-readable name for the metric + +`distanceFunc` [Func](https://learn.microsoft.com/dotnet/api/system.func\-3)<[ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\>, [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\>, [float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Function computing distance between two points + +`disconnectionDistance` [float](https://learn.microsoft.com/dotnet/api/system.single)? + +Optional maximum distance for bounded metrics + +`supportsAngularProjection` [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +Whether angular RP forests benefit this metric + +## Properties + +### DisconnectionDistance + +Maximum meaningful distance for bounded metrics, or null for unbounded metrics. +Used to handle disconnected components in the k-NN graph. + +```csharp +public float? DisconnectionDistance { get; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)? + +#### Remarks + +

+Examples: +- Euclidean: null (unbounded) +- Cosine: 2.0 (ranges from 0 to 2) +- Jaccard: 1.0 (ranges from 0 to 1) +

+

+When set, distances at or beyond this value indicate maximally dissimilar points +that should be treated as disconnected in the manifold approximation. +

+ +### Name + +Human-readable name of the metric (e.g., "euclidean", "cosine"). +Used for logging and serialization. + +```csharp +public string Name { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### SupportsAngularProjection + +Whether this metric benefits from angular (cosine-based) random projection forests. +Angular metrics (cosine, correlation) use different RP tree splits than Euclidean metrics. + +```csharp +public bool SupportsAngularProjection { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### Distance\(ReadOnlySpan, ReadOnlySpan\) + +Compute the distance between two points. + +```csharp +public float Distance(ReadOnlySpan x, ReadOnlySpan y) +``` + +#### Parameters + +`x` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +First point + +`y` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Second point + +#### Returns + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +Distance value (non-negative) + +#### Remarks + +Must satisfy metric properties: +- Non-negativity: Distance(x, y) ≥ 0 +- Identity: Distance(x, x) = 0 +- Symmetry: Distance(x, y) = Distance(y, x) +- Triangle inequality: Distance(x, z) ≤ Distance(x, y) + Distance(y, z) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.EuclideanMetric.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.EuclideanMetric.md new file mode 100644 index 00000000..eb651979 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.EuclideanMetric.md @@ -0,0 +1,142 @@ +# Class EuclideanMetric + +Namespace: [Flowthru.Misc.ML.UMAP.Core.Utils](Flowthru.Misc.ML.UMAP.Core.Utils.md) +Assembly: Flowthru.Misc.ML.dll + +Euclidean (L2) distance metric with gradient support. + +```csharp +public sealed class EuclideanMetric : IOutputMetric, IMetric +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EuclideanMetric](Flowthru.Misc.ML.UMAP.Core.Utils.EuclideanMetric.md) + +#### Implements + +[IOutputMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IOutputMetric.md), +[IMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Euclidean distance is the straight-line distance in n-dimensional space: +d(x, y) = sqrt(sum((x[i] - y[i])^2)) + +This is the most common metric and has specialized optimizations in layout optimization. + +## Properties + +### DisconnectionDistance + +Maximum meaningful distance for bounded metrics, or null for unbounded metrics. +Used to handle disconnected components in the k-NN graph. + +```csharp +public float? DisconnectionDistance { get; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)? + +#### Remarks + +

+Examples: +- Euclidean: null (unbounded) +- Cosine: 2.0 (ranges from 0 to 2) +- Jaccard: 1.0 (ranges from 0 to 1) +

+

+When set, distances at or beyond this value indicate maximally dissimilar points +that should be treated as disconnected in the manifold approximation. +

+ +### Instance + +Singleton instance of Euclidean metric. +Use this to avoid allocations. + +```csharp +public static EuclideanMetric Instance { get; } +``` + +#### Property Value + + [EuclideanMetric](Flowthru.Misc.ML.UMAP.Core.Utils.EuclideanMetric.md) + +### Name + +Human-readable name of the metric (e.g., "euclidean", "cosine"). +Used for logging and serialization. + +```csharp +public string Name { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### SupportsAngularProjection + +Whether this metric benefits from angular (cosine-based) random projection forests. +Angular metrics (cosine, correlation) use different RP tree splits than Euclidean metrics. + +```csharp +public bool SupportsAngularProjection { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### Distance\(ReadOnlySpan, ReadOnlySpan\) + +Compute Euclidean distance: sqrt(sum of squared differences). + +```csharp +public float Distance(ReadOnlySpan x, ReadOnlySpan y) +``` + +#### Parameters + +`x` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +`y` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +#### Returns + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +### DistanceWithGradient\(ReadOnlySpan, ReadOnlySpan, out float, Span\) + +Compute Euclidean distance and its gradient: ∇d/∂x = (x - y) / ||x - y|| + +```csharp +public void DistanceWithGradient(ReadOnlySpan x, ReadOnlySpan y, out float distance, Span gradient) +``` + +#### Parameters + +`x` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +`y` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +`distance` [float](https://learn.microsoft.com/dotnet/api/system.single) + +`gradient` [Span](https://learn.microsoft.com/dotnet/api/system.span\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.ManhattanMetric.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.ManhattanMetric.md new file mode 100644 index 00000000..26b68a7f --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.ManhattanMetric.md @@ -0,0 +1,123 @@ +# Class ManhattanMetric + +Namespace: [Flowthru.Misc.ML.UMAP.Core.Utils](Flowthru.Misc.ML.UMAP.Core.Utils.md) +Assembly: Flowthru.Misc.ML.dll + +Manhattan (L1) distance metric. + +```csharp +public sealed class ManhattanMetric : IMetric +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ManhattanMetric](Flowthru.Misc.ML.UMAP.Core.Utils.ManhattanMetric.md) + +#### Implements + +[IMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +Manhattan distance is the sum of absolute differences: +d(x, y) = sum(|x[i] - y[i]|) + +Also known as taxicab or city block distance. + +## Properties + +### DisconnectionDistance + +Maximum meaningful distance for bounded metrics, or null for unbounded metrics. +Used to handle disconnected components in the k-NN graph. + +```csharp +public float? DisconnectionDistance { get; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)? + +#### Remarks + +

+Examples: +- Euclidean: null (unbounded) +- Cosine: 2.0 (ranges from 0 to 2) +- Jaccard: 1.0 (ranges from 0 to 1) +

+

+When set, distances at or beyond this value indicate maximally dissimilar points +that should be treated as disconnected in the manifold approximation. +

+ +### Instance + +Singleton instance of Manhattan metric. +Use this to avoid allocations. + +```csharp +public static ManhattanMetric Instance { get; } +``` + +#### Property Value + + [ManhattanMetric](Flowthru.Misc.ML.UMAP.Core.Utils.ManhattanMetric.md) + +### Name + +Human-readable name of the metric (e.g., "euclidean", "cosine"). +Used for logging and serialization. + +```csharp +public string Name { get; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + +### SupportsAngularProjection + +Whether this metric benefits from angular (cosine-based) random projection forests. +Angular metrics (cosine, correlation) use different RP tree splits than Euclidean metrics. + +```csharp +public bool SupportsAngularProjection { get; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### Distance\(ReadOnlySpan, ReadOnlySpan\) + +Compute Manhattan distance: sum of absolute differences. + +```csharp +public float Distance(ReadOnlySpan x, ReadOnlySpan y) +``` + +#### Parameters + +`x` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +`y` [ReadOnlySpan](https://learn.microsoft.com/dotnet/api/system.readonlyspan\-1)<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +#### Returns + + [float](https://learn.microsoft.com/dotnet/api/system.single) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.md new file mode 100644 index 00000000..d642538f --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.Utils.md @@ -0,0 +1,20 @@ +# Namespace Flowthru.Misc.ML.UMAP.Core.Utils + +### Classes + + [CosineMetric](Flowthru.Misc.ML.UMAP.Core.Utils.CosineMetric.md) + +Cosine distance metric (angular distance). + + [CustomMetric](Flowthru.Misc.ML.UMAP.Core.Utils.CustomMetric.md) + +Custom metric wrapper for user-defined distance functions. + + [EuclideanMetric](Flowthru.Misc.ML.UMAP.Core.Utils.EuclideanMetric.md) + +Euclidean (L2) distance metric with gradient support. + + [ManhattanMetric](Flowthru.Misc.ML.UMAP.Core.Utils.ManhattanMetric.md) + +Manhattan (L1) distance metric. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.md new file mode 100644 index 00000000..31506551 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Core.md @@ -0,0 +1,47 @@ +# Namespace Flowthru.Misc.ML.UMAP.Core + +### Namespaces + + [Flowthru.Misc.ML.UMAP.Core.Markers](Flowthru.Misc.ML.UMAP.Core.Markers.md) + + [Flowthru.Misc.ML.UMAP.Core.Utils](Flowthru.Misc.ML.UMAP.Core.Utils.md) + +### Classes + + [CurveFitting](Flowthru.Misc.ML.UMAP.Core.CurveFitting.md) + +Helper for computing UMAP curve fitting parameters. + + [DataShape](Flowthru.Misc.ML.UMAP.Core.DataShape.md) + +Describes the shape and characteristics of input data. +Used by strategy factories to select appropriate default strategies. + + [UmapFitResult](Flowthru.Misc.ML.UMAP.Core.UmapFitResult.md) + +Result of the complete UMAP FitTransform operation. +Contains the final embedding and all intermediate results. + + [UmapFlowBuilder](Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md) + + [UmapGraphResult](Flowthru.Misc.ML.UMAP.Core.UmapGraphResult.md) + +Result of computing the UMAP graph (phases 1-3). + + [UmapParameters](Flowthru.Misc.ML.UMAP.Core.UmapParameters.md) + +Core parameters for UMAP algorithm configuration. +These parameters control the mathematical behavior of the algorithm across all strategies. + + [UmapPipeline](Flowthru.Misc.ML.UMAP.Core.UmapPipeline.md) + +Fluent builder for UMAP pipelines with automatic strategy selection. + + [UmapProgress](Flowthru.Misc.ML.UMAP.Core.UmapProgress.md) + +Progress information reported during UMAP execution. + + [UmapRuntimeReport](Flowthru.Misc.ML.UMAP.Core.UmapRuntimeReport.md) + +Runtime performance report for UMAP execution. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.GraphRefinementResult.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.GraphRefinementResult.md new file mode 100644 index 00000000..6a1feafa --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.GraphRefinementResult.md @@ -0,0 +1,97 @@ +# Class GraphRefinementResult + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.md) +Assembly: Flowthru.Misc.ML.dll + +Result of graph refinement operation. + +```csharp +public sealed record GraphRefinementResult : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[GraphRefinementResult](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.GraphRefinementResult.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Constructors + +### GraphRefinementResult\(SparseMatrix, int, float\) + +Result of graph refinement operation. + +```csharp +public GraphRefinementResult(SparseMatrix RefinedGraph, int EdgesRemoved, float MinEdgeWeight) +``` + +#### Parameters + +`RefinedGraph` SparseMatrix + +The refined sparse graph with weak edges removed. +Shape: (n_samples, n_samples) + +`EdgesRemoved` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of edges removed during refinement. +Useful for diagnostics and logging. + +`MinEdgeWeight` [float](https://learn.microsoft.com/dotnet/api/system.single) + +The minimum edge weight threshold that was applied. +Edges below this value were removed. + +## Properties + +### EdgesRemoved + +Number of edges removed during refinement. +Useful for diagnostics and logging. + +```csharp +public int EdgesRemoved { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### MinEdgeWeight + +The minimum edge weight threshold that was applied. +Edges below this value were removed. + +```csharp +public float MinEdgeWeight { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +### RefinedGraph + +The refined sparse graph with weak edges removed. +Shape: (n_samples, n_samples) + +```csharp +public SparseMatrix RefinedGraph { get; init; } +``` + +#### Property Value + + SparseMatrix + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.IGraphRefinementStrategy.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.IGraphRefinementStrategy.md new file mode 100644 index 00000000..f787006e --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.IGraphRefinementStrategy.md @@ -0,0 +1,72 @@ +# Interface IGraphRefinementStrategy + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.md) +Assembly: Flowthru.Misc.ML.dll + +Strategy interface for refining the fuzzy simplicial set graph before layout optimization. +This is the fourth phase of the UMAP algorithm. + +```csharp +public interface IGraphRefinementStrategy +``` + +## Remarks + +

+The graph refinement phase prepares the fuzzy simplicial set for layout optimization by: +

+
  • Pruning weak edges that would have minimal impact on optimization
  • Reducing memory footprint and computational cost
  • Improving numerical stability by removing near-zero weights
+

+Standard approach (adaptive thresholding): +

+

+Edges with weight below max_weight / n_epochs are removed, as they would be +sampled less than once during optimization. This balances graph sparsity with fidelity. +

+

+Python UMAP reference: Lines 1063-1076 in simplicial_set_embedding() function +

+ +## Methods + +### RefineGraph\(SparseMatrix, int\) + +Refines the fuzzy simplicial set by pruning weak edges and normalizing edge weights. + +```csharp +GraphRefinementResult RefineGraph(SparseMatrix graph, int nEpochs) +``` + +#### Parameters + +`graph` SparseMatrix + +Input fuzzy simplicial set as a sparse symmetric matrix. +Shape: (n_samples, n_samples) +This matrix may be modified in-place for efficiency. + +`nEpochs` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of optimization epochs planned for layout optimization. +Used to determine the minimum edge weight threshold - edges sampled less than +once during optimization can be safely removed. +Must be positive. + +#### Returns + + [GraphRefinementResult](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.GraphRefinementResult.md) + +A refined sparse graph with weak edges removed and remaining edges normalized. +May return the same instance as input if modified in-place. + +#### Remarks + +

+Implementation requirements: +

+
  1. Determine edge weight threshold based on n_epochs
  2. Remove edges below threshold
  3. Eliminate zero entries from sparse matrix
  4. Preserve matrix symmetry
  5. Thread-safe for concurrent refinement operations
+

+Performance considerations: +

+
  • In-place modification is preferred to reduce memory allocation
  • Sparse matrix operations should preserve CSR/CSC format efficiency
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.AdaptiveThresholding.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.AdaptiveThresholding.md new file mode 100644 index 00000000..41d6996b --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.AdaptiveThresholding.md @@ -0,0 +1,82 @@ +# Class AdaptiveThresholding + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.md) +Assembly: Flowthru.Misc.ML.dll + +Standard UMAP graph refinement using adaptive threshold based on optimization epochs. + +```csharp +public sealed class AdaptiveThresholding : IGraphRefinementStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[AdaptiveThresholding](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.AdaptiveThresholding.md) + +#### Implements + +[IGraphRefinementStrategy](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.IGraphRefinementStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This implementation follows the standard UMAP algorithm's approach to graph refinement: +edges with weight below max_weight / n_epochs are removed, as they would be +sampled less than once during the optimization process. +

+

+Rationale: During stochastic gradient descent, edges are sampled proportionally +to their weights. An edge with weight w in a graph with maximum weight w_max +will be sampled approximately (w / w_max) × n_epochs times. Edges sampled less +than once have negligible impact on the final embedding. +

+

+Implementation: Uses direct CSR (Compressed Sparse Row) storage manipulation for +O(nnz) performance. Single-pass filter through non-zero entries only, avoiding O(n²) iteration. +

+

+Time complexity: O(nnz) where nnz is the number of non-zero entries in the graph +

+

+Space complexity: O(nnz) - creates new storage arrays during filtering +

+

+Python UMAP reference: Lines 1063-1076 in simplicial_set_embedding() +

+ +## Methods + +### RefineGraph\(SparseMatrix, int\) + +Refines the graph by removing edges below an adaptive threshold using CSR direct access. + +```csharp +public GraphRefinementResult RefineGraph(SparseMatrix graph, int nEpochs) +``` + +#### Parameters + +`graph` SparseMatrix + +Fuzzy simplicial set to refine (modified in-place). + +`nEpochs` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of optimization epochs planned. + +#### Returns + + [GraphRefinementResult](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.GraphRefinementResult.md) + +Refinement result with statistics. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.md new file mode 100644 index 00000000..ed6354de --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.md @@ -0,0 +1,8 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations + +### Classes + + [AdaptiveThresholding](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.AdaptiveThresholding.md) + +Standard UMAP graph refinement using adaptive threshold based on optimization epochs. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.md new file mode 100644 index 00000000..a038b053 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.md @@ -0,0 +1,19 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement + +### Namespaces + + [Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.md) + +### Classes + + [GraphRefinementResult](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.GraphRefinementResult.md) + +Result of graph refinement operation. + +### Interfaces + + [IGraphRefinementStrategy](Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.IGraphRefinementStrategy.md) + +Strategy interface for refining the fuzzy simplicial set graph before layout optimization. +This is the fourth phase of the UMAP algorithm. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.ILayoutInitStrategy.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.ILayoutInitStrategy.md new file mode 100644 index 00000000..67322acd --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.ILayoutInitStrategy.md @@ -0,0 +1,85 @@ +# Interface ILayoutInitStrategy + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LayoutInit](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.md) +Assembly: Flowthru.Misc.ML.dll + +Strategy interface for initializing the low-dimensional embedding before optimization. +This is the fifth phase of the UMAP algorithm. + +```csharp +public interface ILayoutInitStrategy +``` + +## Remarks + +

+The layout initialization phase creates an initial low-dimensional embedding that serves +as the starting point for stochastic gradient descent optimization. The quality of this +initialization significantly impacts: +

+
  • Convergence speed: Better initializations require fewer optimization epochs
  • Final quality: Good initializations help avoid poor local minima
  • Reproducibility: Deterministic initializations enable consistent results
+

+Common initialization strategies: +

+
  • Spectral: Eigendecomposition of graph Laplacian (high quality, O(n²) time, recommended for datasets < 10k samples)
  • PCA: Principal component analysis of original data (medium quality, O(n×d) time)
  • Random: Uniform random positions (low quality, O(n) time, fastest option)
  • Precomputed: User-provided initialization (quality varies)
+

+All initializations are normalized to the range [-10, 10] with small random noise +to prevent degenerate configurations and improve numerical stability. +

+

+Python UMAP reference: Lines 1078-1148 in simplicial_set_embedding() function +

+ +## Methods + +### InitializeLayout\(Matrix?, SparseMatrix, int, Random\) + +Initializes the low-dimensional embedding layout. + +```csharp +LayoutInitResult InitializeLayout(Matrix? data, SparseMatrix graph, int nComponents, Random random) +``` + +#### Parameters + +`data` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\>? + +Original high-dimensional data matrix. +Shape: (n_samples, n_features) +May be null for precomputed distance-based initialization. + +`graph` SparseMatrix + +Refined fuzzy simplicial set graph after pruning. +Shape: (n_samples, n_samples) +Used by spectral and graph-based initialization methods. + +`nComponents` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Target dimensionality of the embedding. +Typically 2 or 3 for visualization, or higher for downstream tasks. +Must be at least 1 and less than n_samples. + +`random` [Random](https://learn.microsoft.com/dotnet/api/system.random) + +Random number generator for reproducible randomization. +Used for noise injection and random initialization. + +#### Returns + + [LayoutInitResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.LayoutInitResult.md) + +Initial embedding matrix with coordinates normalized to [-10, 10] range. +Shape: (n_samples, n_components) + +#### Remarks + +

+Implementation requirements: +

+
  1. Generate or compute initial coordinates
  2. Add small random noise to avoid degeneracies
  3. Normalize to [-10, 10] range for numerical stability
  4. Ensure output is C-contiguous (row-major) for optimization
  5. Handle disconnected graph components gracefully
+

+Performance considerations: +

+
  • Spectral methods require eigenvalue decomposition: O(n²) to O(n³)
  • PCA methods require SVD: O(min(n,d) × n × d)
  • Random methods are O(n × k) where k is n_components
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.RandomInit.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.RandomInit.md new file mode 100644 index 00000000..57d9392e --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.RandomInit.md @@ -0,0 +1,84 @@ +# Class RandomInit + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.md) +Assembly: Flowthru.Misc.ML.dll + +Random uniform initialization for fast prototyping and debugging. + +```csharp +public sealed class RandomInit : ILayoutInitStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[RandomInit](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.RandomInit.md) + +#### Implements + +[ILayoutInitStrategy](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.ILayoutInitStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This strategy initializes embedding coordinates uniformly at random in the range [-10, 10]. +While this provides the fastest initialization, it typically requires more optimization +epochs to converge compared to spectral or PCA initialization. +

+

+Use cases: +

+
  • Quick prototyping and experimentation
  • Debugging optimization algorithms
  • When data/graph are unavailable for smarter initialization
  • Fallback when spectral initialization fails (disconnected graph)
+

+Time complexity: O(n × k) where n = n_samples, k = n_components +

+

+Space complexity: O(n × k) +

+

+Python UMAP reference: Lines 1078-1081 in simplicial_set_embedding() +

+ +## Methods + +### InitializeLayout\(Matrix?, SparseMatrix, int, Random\) + +Initializes embedding with uniform random coordinates. + +```csharp +public LayoutInitResult InitializeLayout(Matrix? data, SparseMatrix graph, int nComponents, Random random) +``` + +#### Parameters + +`data` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\>? + +Original data (unused for random initialization). + +`graph` SparseMatrix + +Graph (unused for random initialization). + +`nComponents` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Target embedding dimensionality. + +`random` [Random](https://learn.microsoft.com/dotnet/api/system.random) + +Random number generator for coordinate sampling. + +#### Returns + + [LayoutInitResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.LayoutInitResult.md) + +Random embedding normalized to [-10, 10] range. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.SpectralInit.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.SpectralInit.md new file mode 100644 index 00000000..ceed57ae --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.SpectralInit.md @@ -0,0 +1,83 @@ +# Class SpectralInit + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.md) +Assembly: Flowthru.Misc.ML.dll + +Spectral initialization via eigendecomposition of the graph Laplacian. +Produces a high-quality initialization for connected graphs on small-to-medium datasets. + +```csharp +public sealed class SpectralInit : ILayoutInitStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[SpectralInit](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.SpectralInit.md) + +#### Implements + +[ILayoutInitStrategy](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.ILayoutInitStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Methods + +### InitializeLayout\(Matrix?, SparseMatrix, int, Random\) + +Initializes the low-dimensional embedding layout. + +```csharp +public LayoutInitResult InitializeLayout(Matrix? data, SparseMatrix graph, int nComponents, Random random) +``` + +#### Parameters + +`data` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\>? + +Original high-dimensional data matrix. +Shape: (n_samples, n_features) +May be null for precomputed distance-based initialization. + +`graph` SparseMatrix + +Refined fuzzy simplicial set graph after pruning. +Shape: (n_samples, n_samples) +Used by spectral and graph-based initialization methods. + +`nComponents` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Target dimensionality of the embedding. +Typically 2 or 3 for visualization, or higher for downstream tasks. +Must be at least 1 and less than n_samples. + +`random` [Random](https://learn.microsoft.com/dotnet/api/system.random) + +Random number generator for reproducible randomization. +Used for noise injection and random initialization. + +#### Returns + + [LayoutInitResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.LayoutInitResult.md) + +Initial embedding matrix with coordinates normalized to [-10, 10] range. +Shape: (n_samples, n_components) + +#### Remarks + +

+Implementation requirements: +

+
  1. Generate or compute initial coordinates
  2. Add small random noise to avoid degeneracies
  3. Normalize to [-10, 10] range for numerical stability
  4. Ensure output is C-contiguous (row-major) for optimization
  5. Handle disconnected graph components gracefully
+

+Performance considerations: +

+
  • Spectral methods require eigenvalue decomposition: O(n²) to O(n³)
  • PCA methods require SVD: O(min(n,d) × n × d)
  • Random methods are O(n × k) where k is n_components
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.md new file mode 100644 index 00000000..e5c43f97 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.md @@ -0,0 +1,13 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations + +### Classes + + [RandomInit](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.RandomInit.md) + +Random uniform initialization for fast prototyping and debugging. + + [SpectralInit](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.SpectralInit.md) + +Spectral initialization via eigendecomposition of the graph Laplacian. +Produces a high-quality initialization for connected graphs on small-to-medium datasets. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.LayoutInitResult.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.LayoutInitResult.md new file mode 100644 index 00000000..92ae5499 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.LayoutInitResult.md @@ -0,0 +1,81 @@ +# Class LayoutInitResult + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LayoutInit](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.md) +Assembly: Flowthru.Misc.ML.dll + +Result of layout initialization. + +```csharp +public sealed record LayoutInitResult : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[LayoutInitResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.LayoutInitResult.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Constructors + +### LayoutInitResult\(Matrix, string\) + +Result of layout initialization. + +```csharp +public LayoutInitResult(Matrix Embedding, string InitializationMethod) +``` + +#### Parameters + +`Embedding` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Initial low-dimensional embedding coordinates. +Shape: (n_samples, n_components) +Values are normalized to approximately [-10, 10] range. + +`InitializationMethod` [string](https://learn.microsoft.com/dotnet/api/system.string) + +Human-readable description of the initialization method used. +Useful for logging and debugging. + +## Properties + +### Embedding + +Initial low-dimensional embedding coordinates. +Shape: (n_samples, n_components) +Values are normalized to approximately [-10, 10] range. + +```csharp +public Matrix Embedding { get; init; } +``` + +#### Property Value + + Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +### InitializationMethod + +Human-readable description of the initialization method used. +Useful for logging and debugging. + +```csharp +public string InitializationMethod { get; init; } +``` + +#### Property Value + + [string](https://learn.microsoft.com/dotnet/api/system.string) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.md new file mode 100644 index 00000000..6ef87b31 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.md @@ -0,0 +1,19 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.LayoutInit + +### Namespaces + + [Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.md) + +### Classes + + [LayoutInitResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.LayoutInitResult.md) + +Result of layout initialization. + +### Interfaces + + [ILayoutInitStrategy](Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.ILayoutInitStrategy.md) + +Strategy interface for initializing the low-dimensional embedding before optimization. +This is the fifth phase of the UMAP algorithm. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.GraphEdge.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.GraphEdge.md new file mode 100644 index 00000000..673edf2f --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.GraphEdge.md @@ -0,0 +1,86 @@ +# Struct GraphEdge + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.md) +Assembly: Flowthru.Misc.ML.dll + +Represents an edge in the fuzzy simplicial set graph. + +```csharp +public readonly record struct GraphEdge : IEquatable +``` + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Constructors + +### GraphEdge\(int, int, float\) + +Represents an edge in the fuzzy simplicial set graph. + +```csharp +public GraphEdge(int Head, int Tail, float Weight) +``` + +#### Parameters + +`Head` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Index of the head vertex (source). + +`Tail` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Index of the tail vertex (target). + +`Weight` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Membership strength of this edge. + +## Properties + +### Head + +Index of the head vertex (source). + +```csharp +public int Head { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### Tail + +Index of the tail vertex (target). + +```csharp +public int Tail { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### Weight + +Membership strength of this edge. + +```csharp +public float Weight { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.ILayoutOptimizationStrategy.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.ILayoutOptimizationStrategy.md new file mode 100644 index 00000000..a401d9b8 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.ILayoutOptimizationStrategy.md @@ -0,0 +1,82 @@ +# Interface ILayoutOptimizationStrategy + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.md) +Assembly: Flowthru.Misc.ML.dll + +Strategy interface for optimizing low-dimensional embeddings via stochastic gradient descent. +This is the seventh phase of the UMAP algorithm. + +```csharp +public interface ILayoutOptimizationStrategy +``` + +## Remarks + +

+The layout optimization phase refines the initial embedding by minimizing the fuzzy set +cross entropy between the high-dimensional and low-dimensional fuzzy simplicial sets. +This is done through stochastic gradient descent with two types of forces: +

+
  • Attractive forces: Pull connected points closer based on graph edge weights
  • Repulsive forces: Push non-connected points apart via negative sampling
+

+The force curves are parameterized by a and b, which are derived from +the min_dist and spread hyperparameters via curve fitting. +

+

+Python UMAP reference: optimize_layout_euclidean() in layouts.py (lines 238-441) +

+ +## Methods + +### Optimize\(Matrix, GraphEdge\[\], float\[\], int, OptimizationParameters, Random\) + +Optimizes the embedding layout using stochastic gradient descent. + +```csharp +LayoutOptimizationResult Optimize(Matrix initialEmbedding, GraphEdge[] graphEdges, float[] samplingSchedule, int nEpochs, OptimizationParameters parameters, Random random) +``` + +#### Parameters + +`initialEmbedding` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Initial embedding from layout initialization strategy. +Shape: (n_samples, n_components) +This matrix will be modified in-place during optimization. + +`graphEdges` [GraphEdge](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.GraphEdge.md)\[\] + +Edges in the fuzzy simplicial set to optimize. +Contains (head_index, tail_index, weight) tuples. + +`samplingSchedule` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Sampling schedule that determines how often each edge is sampled. +Array length matches number of edges. + +`nEpochs` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of optimization epochs to run. +Must match the value used to compute the sampling schedule. + +`parameters` [OptimizationParameters](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.OptimizationParameters.md) + +Optimization parameters including learning rate, repulsion strength, etc. + +`random` [Random](https://learn.microsoft.com/dotnet/api/system.random) + +Random number generator for negative sampling and reproducibility. + +#### Returns + + [LayoutOptimizationResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.LayoutOptimizationResult.md) + +The optimized embedding (same matrix as initialEmbedding, modified in-place). + +#### Remarks + +

+Implementation requirements: +

+
  1. Initialize epoch-tracking arrays for sampling schedule
  2. For each epoch:
  3. Report progress if verbosity enabled
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGD.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGD.md new file mode 100644 index 00000000..ff1fd57a --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGD.md @@ -0,0 +1,102 @@ +# Class EuclideanSGD + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.md) +Assembly: Flowthru.Misc.ML.dll + +Standard Euclidean distance SGD optimizer for UMAP layout optimization. + +```csharp +public sealed class EuclideanSGD : ILayoutOptimizationStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EuclideanSGD](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGD.md) + +#### Implements + +[ILayoutOptimizationStrategy](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.ILayoutOptimizationStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+⚠️ NOTE: This is the reference implementation retained for testing and historical purposes. +For production use, prefer which provides 1.5-2x speedup +through direct array access and early stopping while maintaining identical embedding quality. +

+

+This implementation follows the Python UMAP reference for Euclidean output spaces. +It uses stochastic gradient descent with: +

+
  • Attractive forces based on graph edge weights and a/b curve parameters
  • Repulsive forces from negative sampling of non-neighbors
  • Linear learning rate decay
+

+Time complexity: O(E × n_epochs + N × k × n_epochs) where E = edges, N = vertices, k = negative samples +

+

+Python UMAP reference: optimize_layout_euclidean() in layouts.py (lines 238-441) +

+ +## Methods + +### Optimize\(Matrix, GraphEdge\[\], float\[\], int, OptimizationParameters, Random\) + +Optimizes the embedding layout using stochastic gradient descent. + +```csharp +public LayoutOptimizationResult Optimize(Matrix initialEmbedding, GraphEdge[] graphEdges, float[] samplingSchedule, int nEpochs, OptimizationParameters parameters, Random random) +``` + +#### Parameters + +`initialEmbedding` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Initial embedding from layout initialization strategy. +Shape: (n_samples, n_components) +This matrix will be modified in-place during optimization. + +`graphEdges` [GraphEdge](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.GraphEdge.md)\[\] + +Edges in the fuzzy simplicial set to optimize. +Contains (head_index, tail_index, weight) tuples. + +`samplingSchedule` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Sampling schedule that determines how often each edge is sampled. +Array length matches number of edges. + +`nEpochs` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of optimization epochs to run. +Must match the value used to compute the sampling schedule. + +`parameters` [OptimizationParameters](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.OptimizationParameters.md) + +Optimization parameters including learning rate, repulsion strength, etc. + +`random` [Random](https://learn.microsoft.com/dotnet/api/system.random) + +Random number generator for negative sampling and reproducibility. + +#### Returns + + [LayoutOptimizationResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.LayoutOptimizationResult.md) + +The optimized embedding (same matrix as initialEmbedding, modified in-place). + +#### Remarks + +

+Implementation requirements: +

+
  1. Initialize epoch-tracking arrays for sampling schedule
  2. For each epoch:
  3. Report progress if verbosity enabled
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGDOptimized.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGDOptimized.md new file mode 100644 index 00000000..f13a9901 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGDOptimized.md @@ -0,0 +1,123 @@ +# Class EuclideanSGDOptimized + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.md) +Assembly: Flowthru.Misc.ML.dll + +Optimized Euclidean distance SGD optimizer for UMAP layout optimization (default implementation). + +```csharp +public sealed class EuclideanSGDOptimized : ILayoutOptimizationStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[EuclideanSGDOptimized](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGDOptimized.md) + +#### Implements + +[ILayoutOptimizationStrategy](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.ILayoutOptimizationStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+✓ This is the default layout optimization strategy as of November 2025. +Provides strict performance improvements over with identical embedding quality. +

+

+This implementation optimizes the standard UMAP SGD algorithm with: +

+
  • Direct array access: Uses for vectorized operations
  • Cache-friendly memory access: Exploits column-major layout for better locality
  • Early stopping: Monitors convergence and terminates when vertex movement stabilizes
  • Reduced overhead: Eliminates repeated matrix indexing overhead
+

+Validated performance improvements (Fashion MNIST 70k samples): +

+
  • Layout Optimization: 62.4s → ~42s (~33% faster)
  • Total UMAP Runtime: 121.7s → ~101s (~17% faster overall)
  • Embedding Quality: Identical (validated via neighborhood preservation)
+

+Usage: Automatically selected by UmapPipeline.Create(). To use the reference +implementation for testing, explicitly call .WithLayoutOptimization(new EuclideanSGD()). +

+

+Python UMAP reference: optimize_layout_euclidean() in layouts.py (lines 238-441) +

+ +## Constructors + +### EuclideanSGDOptimized\(float\) + +Initializes a new instance of the optimized SGD optimizer. + +```csharp +public EuclideanSGDOptimized(float convergenceThreshold = 0.001) +``` + +#### Parameters + +`convergenceThreshold` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Average vertex movement threshold for early stopping. +Default is 0.001 (0.1% of coordinate space). +Set to 0 to disable early stopping. + +## Methods + +### Optimize\(Matrix, GraphEdge\[\], float\[\], int, OptimizationParameters, Random\) + +Optimizes the embedding layout using stochastic gradient descent. + +```csharp +public LayoutOptimizationResult Optimize(Matrix initialEmbedding, GraphEdge[] graphEdges, float[] samplingSchedule, int nEpochs, OptimizationParameters parameters, Random random) +``` + +#### Parameters + +`initialEmbedding` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +Initial embedding from layout initialization strategy. +Shape: (n_samples, n_components) +This matrix will be modified in-place during optimization. + +`graphEdges` [GraphEdge](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.GraphEdge.md)\[\] + +Edges in the fuzzy simplicial set to optimize. +Contains (head_index, tail_index, weight) tuples. + +`samplingSchedule` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Sampling schedule that determines how often each edge is sampled. +Array length matches number of edges. + +`nEpochs` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of optimization epochs to run. +Must match the value used to compute the sampling schedule. + +`parameters` [OptimizationParameters](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.OptimizationParameters.md) + +Optimization parameters including learning rate, repulsion strength, etc. + +`random` [Random](https://learn.microsoft.com/dotnet/api/system.random) + +Random number generator for negative sampling and reproducibility. + +#### Returns + + [LayoutOptimizationResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.LayoutOptimizationResult.md) + +The optimized embedding (same matrix as initialEmbedding, modified in-place). + +#### Remarks + +

+Implementation requirements: +

+
  1. Initialize epoch-tracking arrays for sampling schedule
  2. For each epoch:
  3. Report progress if verbosity enabled
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.md new file mode 100644 index 00000000..b03abe19 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.md @@ -0,0 +1,12 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations + +### Classes + + [EuclideanSGD](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGD.md) + +Standard Euclidean distance SGD optimizer for UMAP layout optimization. + + [EuclideanSGDOptimized](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGDOptimized.md) + +Optimized Euclidean distance SGD optimizer for UMAP layout optimization (default implementation). + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.LayoutOptimizationResult.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.LayoutOptimizationResult.md new file mode 100644 index 00000000..01318ae3 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.LayoutOptimizationResult.md @@ -0,0 +1,105 @@ +# Class LayoutOptimizationResult + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.md) +Assembly: Flowthru.Misc.ML.dll + +Result of layout optimization. + +```csharp +public sealed record LayoutOptimizationResult : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[LayoutOptimizationResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.LayoutOptimizationResult.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Constructors + +### LayoutOptimizationResult\(Matrix, float?\) + +Result of layout optimization. + +```csharp +public LayoutOptimizationResult(Matrix OptimizedEmbedding, float? FinalLoss) +``` + +#### Parameters + +`OptimizedEmbedding` Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + +The final optimized embedding. +Shape: (n_samples, n_components) + +`FinalLoss` [float](https://learn.microsoft.com/dotnet/api/system.single)? + +Final cross-entropy loss (if computed). +Null if loss tracking is disabled. + +## Properties + +### ActualEpochs + +Actual number of epochs completed before termination. +May be less than requested epochs if early stopping was triggered. + +```csharp +public int? ActualEpochs { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32)? + +### EarlyStoppingSaved + +Number of epochs saved by early stopping. +Zero if optimization ran to completion or early stopping was disabled. + +```csharp +public int? EarlyStoppingSaved { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32)? + +### FinalLoss + +Final cross-entropy loss (if computed). +Null if loss tracking is disabled. + +```csharp +public float? FinalLoss { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)? + +### OptimizedEmbedding + +The final optimized embedding. +Shape: (n_samples, n_components) + +```csharp +public Matrix OptimizedEmbedding { get; init; } +``` + +#### Property Value + + Matrix<[float](https://learn.microsoft.com/dotnet/api/system.single)\> + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.OptimizationParameters.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.OptimizationParameters.md new file mode 100644 index 00000000..76c5bc82 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.OptimizationParameters.md @@ -0,0 +1,115 @@ +# Class OptimizationParameters + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.md) +Assembly: Flowthru.Misc.ML.dll + +Parameters for layout optimization. + +```csharp +public sealed record OptimizationParameters : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[OptimizationParameters](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.OptimizationParameters.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Properties + +### A + +Curve parameter 'a' for attractive force. + +```csharp +public required float A { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +### B + +Curve parameter 'b' for attractive force. + +```csharp +public required float B { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +### Gamma + +Weight applied to negative (repulsive) samples. + +```csharp +public required float Gamma { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +### InitialAlpha + +Initial learning rate (decays linearly to 0). + +```csharp +public required float InitialAlpha { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +### NegativeSampleRate + +Number of negative samples per positive sample. + +```csharp +public required int NegativeSampleRate { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### ProgressReporter + +Progress reporter for programmatic tracking. + +```csharp +public IProgress? ProgressReporter { get; init; } +``` + +#### Property Value + + [IProgress](https://learn.microsoft.com/dotnet/api/system.iprogress\-1)<[UmapProgress](Flowthru.Misc.ML.UMAP.Core.UmapProgress.md)\>? + +### Verbosity + +Verbosity level for progress reporting. + +```csharp +public int Verbosity { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.md new file mode 100644 index 00000000..2564dfa7 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.md @@ -0,0 +1,29 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization + +### Namespaces + + [Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.md) + +### Classes + + [LayoutOptimizationResult](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.LayoutOptimizationResult.md) + +Result of layout optimization. + + [OptimizationParameters](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.OptimizationParameters.md) + +Parameters for layout optimization. + +### Structs + + [GraphEdge](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.GraphEdge.md) + +Represents an edge in the fuzzy simplicial set graph. + +### Interfaces + + [ILayoutOptimizationStrategy](Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.ILayoutOptimizationStrategy.md) + +Strategy interface for optimizing low-dimensional embeddings via stochastic gradient descent. +This is the seventh phase of the UMAP algorithm. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.ILocalMetricStrategy.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.ILocalMetricStrategy.md new file mode 100644 index 00000000..67d6db57 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.ILocalMetricStrategy.md @@ -0,0 +1,82 @@ +# Interface ILocalMetricStrategy + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LocalMetric](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.md) +Assembly: Flowthru.Misc.ML.dll + +Strategy interface for computing smooth approximations of local distances. +This is the second phase of the UMAP algorithm. + +```csharp +public interface ILocalMetricStrategy +``` + +## Remarks + +

+The local metric phase computes bandwidth parameters (σᵢ and ρᵢ) for each point that +normalize the local neighborhood structure. This handles varying local densities in the data: +

+
  • σᵢ (sigma): Bandwidth of the exponential kernel for point i
  • ρᵢ (rho): Distance to the nearest connected neighbor for point i
+

+These parameters ensure that each point has roughly the same "effective" number of neighbors +regardless of the local density, which is crucial for constructing a consistent fuzzy +simplicial set representation of the manifold. +

+

+Mathematical goal: Find σᵢ such that the fuzzy cardinality of the neighborhood equals k: +

+
Σⱼ exp(-(dᵢⱼ - ρᵢ) / σᵢ) = log₂(k)
+

+Python UMAP reference: smooth_knn_dist() function in umap_.py (lines ~143-250) +

+ +## Methods + +### ComputeLocalMetrics\(float\[\]\[\], float, float, float\) + +Computes smooth local metric parameters (bandwidths and local connectivity distances). + +```csharp +LocalMetricResult ComputeLocalMetrics(float[][] knnDistances, float k, float localConnectivity = 1, float bandwidth = 1) +``` + +#### Parameters + +`knnDistances` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Distance to k-nearest neighbors for each point. +Array shape: (n_samples, n_neighbors) +Each row should be sorted in ascending order. + +`k` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Target number of effective neighbors (typically the same as n_neighbors). +Used to calibrate the bandwidth parameter. + +`localConnectivity` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Number of nearest neighbors that should be assumed to be connected at a local level. +Typically 1.0, meaning the nearest neighbor is always assumed connected (distance weight = 1.0). +Higher values (e.g., 2.0-5.0) increase local connectivity. +Range: [1.0, k] + +`bandwidth` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Target bandwidth multiplier for the exponential kernel. +Default: 1.0. Affects the target cardinality (target = log₂(k) × bandwidth). + +#### Returns + + [LocalMetricResult](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.LocalMetricResult.md) + +A result containing: +- Sigmas: Bandwidth parameter for each point (length n_samples) +- Rhos: Distance to nearest connected neighbor for each point (length n_samples) + +#### Remarks + +

+Implementation requirements: +

+
  1. Handle the case where points have fewer than k non-zero distances
  2. Apply minimum distance scaling to prevent numerical instability
  3. Ensure rho ≤ distance to k-th neighbor for all points
  4. Thread-safe for parallel processing of points
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.BinarySearchSmoothing.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.BinarySearchSmoothing.md new file mode 100644 index 00000000..cb925337 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.BinarySearchSmoothing.md @@ -0,0 +1,115 @@ +# Class BinarySearchSmoothing + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.md) +Assembly: Flowthru.Misc.ML.dll + +Binary search-based local metric smoothing. +Computes bandwidth parameters using iterative binary search to match target cardinality. + +```csharp +public sealed class BinarySearchSmoothing : ILocalMetricStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[BinarySearchSmoothing](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.BinarySearchSmoothing.md) + +#### Implements + +[ILocalMetricStrategy](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.ILocalMetricStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This is the standard UMAP approach for computing local metric parameters. For each point, +it uses binary search to find the bandwidth σ that makes the fuzzy cardinality of its +neighborhood equal to the target value (log₂(k)). +

+

+Algorithm: +

+
  1. Compute ρᵢ (distance to nearest connected neighbor) based on local connectivity
  2. Use binary search to find σᵢ such that Σⱼ exp(-(dᵢⱼ - ρᵢ)/σᵢ) ≈ log₂(k)
  3. Apply minimum distance scaling to prevent numerical instability
+

+Characteristics: +

+
  • Time complexity: O(n × k × log(max_iter)) ≈ O(n × k)
  • Space complexity: O(n) for output
  • Convergence: Typically within 10-20 iterations per point
  • Thread-safe: Yes (each point computed independently)
+

+Python reference: smooth_knn_dist() function in umap_.py (lines ~143-250). +This is a direct port of the numba-jitted Python implementation. +

+ +## Properties + +### MaxIterations + +Maximum number of binary search iterations per point. +Typically converges much faster, but this provides a safety limit. + +```csharp +public int MaxIterations { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +## Methods + +### ComputeLocalMetrics\(float\[\]\[\], float, float, float\) + +Computes smooth local metric parameters (bandwidths and local connectivity distances). + +```csharp +public LocalMetricResult ComputeLocalMetrics(float[][] knnDistances, float k, float localConnectivity = 1, float bandwidth = 1) +``` + +#### Parameters + +`knnDistances` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Distance to k-nearest neighbors for each point. +Array shape: (n_samples, n_neighbors) +Each row should be sorted in ascending order. + +`k` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Target number of effective neighbors (typically the same as n_neighbors). +Used to calibrate the bandwidth parameter. + +`localConnectivity` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Number of nearest neighbors that should be assumed to be connected at a local level. +Typically 1.0, meaning the nearest neighbor is always assumed connected (distance weight = 1.0). +Higher values (e.g., 2.0-5.0) increase local connectivity. +Range: [1.0, k] + +`bandwidth` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Target bandwidth multiplier for the exponential kernel. +Default: 1.0. Affects the target cardinality (target = log₂(k) × bandwidth). + +#### Returns + + [LocalMetricResult](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.LocalMetricResult.md) + +A result containing: +- Sigmas: Bandwidth parameter for each point (length n_samples) +- Rhos: Distance to nearest connected neighbor for each point (length n_samples) + +#### Remarks + +

+Implementation requirements: +

+
  1. Handle the case where points have fewer than k non-zero distances
  2. Apply minimum distance scaling to prevent numerical instability
  3. Ensure rho ≤ distance to k-th neighbor for all points
  4. Thread-safe for parallel processing of points
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.md new file mode 100644 index 00000000..119bae9d --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.md @@ -0,0 +1,9 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations + +### Classes + + [BinarySearchSmoothing](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.BinarySearchSmoothing.md) + +Binary search-based local metric smoothing. +Computes bandwidth parameters using iterative binary search to match target cardinality. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.LocalMetricResult.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.LocalMetricResult.md new file mode 100644 index 00000000..4bc5bc10 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.LocalMetricResult.md @@ -0,0 +1,79 @@ +# Class LocalMetricResult + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.LocalMetric](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.md) +Assembly: Flowthru.Misc.ML.dll + +Result of local metric computation. + +```csharp +public sealed record LocalMetricResult : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[LocalMetricResult](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.LocalMetricResult.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Constructors + +### LocalMetricResult\(float\[\], float\[\]\) + +Result of local metric computation. + +```csharp +public LocalMetricResult(float[] Sigmas, float[] Rhos) +``` + +#### Parameters + +`Sigmas` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Bandwidth parameters for exponential kernel. +Array shape: (n_samples,) + +`Rhos` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Distance to nearest connected neighbor. +Array shape: (n_samples,) + +## Properties + +### Rhos + +Distance to nearest connected neighbor. +Array shape: (n_samples,) + +```csharp +public float[] Rhos { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +### Sigmas + +Bandwidth parameters for exponential kernel. +Array shape: (n_samples,) + +```csharp +public float[] Sigmas { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.md new file mode 100644 index 00000000..b6e19eac --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.md @@ -0,0 +1,19 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.LocalMetric + +### Namespaces + + [Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.md) + +### Classes + + [LocalMetricResult](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.LocalMetricResult.md) + +Result of local metric computation. + +### Interfaces + + [ILocalMetricStrategy](Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.ILocalMetricStrategy.md) + +Strategy interface for computing smooth approximations of local distances. +This is the second phase of the UMAP algorithm. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.IMembershipStrengthStrategy.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.IMembershipStrengthStrategy.md new file mode 100644 index 00000000..92ba2801 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.IMembershipStrengthStrategy.md @@ -0,0 +1,89 @@ +# Interface IMembershipStrengthStrategy + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength](Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.md) +Assembly: Flowthru.Misc.ML.dll + +Strategy interface for computing fuzzy simplicial set membership strengths. +This is the third phase of the UMAP algorithm. + +```csharp +public interface IMembershipStrengthStrategy +``` + +## Remarks + +

+The membership strength phase converts k-NN distances into membership probabilities +for the fuzzy simplicial set. Each edge (i,j) gets a membership strength μᵢⱼ ∈ [0,1] +that represents how strongly point j belongs to the fuzzy neighborhood of point i. +

+

+Standard approach (exponential kernel): +

+
μᵢⱼ = exp(-(max(0, dᵢⱼ - ρᵢ)) / σᵢ)
+

+where dᵢⱼ is the distance, ρᵢ is the local connectivity distance, and σᵢ is the bandwidth. +

+

+After computing directed membership strengths, fuzzy set operations (union/intersection) +combine them into a symmetric global graph structure. +

+

+Python UMAP reference: compute_membership_strengths() and fuzzy_simplicial_set() +functions in umap_.py (lines ~260-450) +

+ +## Methods + +### ComputeMembershipStrengths\(int\[\]\[\], float\[\]\[\], float\[\], float\[\], float\) + +Computes membership strengths for the fuzzy simplicial set. + +```csharp +SparseMatrix ComputeMembershipStrengths(int[][] knnIndices, float[][] knnDistances, float[] sigmas, float[] rhos, float setOpMixRatio = 1) +``` + +#### Parameters + +`knnIndices` [int](https://learn.microsoft.com/dotnet/api/system.int32)\[\]\[\] + +Indices of k-nearest neighbors for each point. +Array shape: (n_samples, n_neighbors) + +`knnDistances` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Distances to k-nearest neighbors for each point. +Array shape: (n_samples, n_neighbors) + +`sigmas` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Bandwidth parameters from local metric computation. +Array shape: (n_samples,) + +`rhos` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Local connectivity distances from local metric computation. +Array shape: (n_samples,) + +`setOpMixRatio` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Interpolation between fuzzy union (1.0) and intersection (0.0). +Controls how local fuzzy sets are combined into global structure. +Range: [0.0, 1.0] + +#### Returns + + SparseMatrix + +A sparse matrix representing the fuzzy simplicial set. +Shape: (n_samples, n_samples) +Matrix[i,j] represents the membership strength of the edge from i to j. +After set operations, the matrix should be symmetric. + +#### Remarks + +

+Implementation requirements: +

+
  1. Compute directed membership strengths μᵢⱼ for each edge
  2. Apply fuzzy set operation: μ = α(μᵢⱼ + μⱼᵢ - μᵢⱼμⱼᵢ) + (1-α)μᵢⱼμⱼᵢ
  3. Eliminate zero entries from sparse matrix
  4. Ensure matrix is symmetric after set operations
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.ExponentialKernel.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.ExponentialKernel.md new file mode 100644 index 00000000..4538b5af --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.ExponentialKernel.md @@ -0,0 +1,112 @@ +# Class ExponentialKernel + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations](Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.md) +Assembly: Flowthru.Misc.ML.dll + +Exponential kernel-based membership strength computation. +Uses the standard UMAP exponential kernel to convert distances into probabilities. + +```csharp +public sealed class ExponentialKernel : IMembershipStrengthStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ExponentialKernel](Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.ExponentialKernel.md) + +#### Implements + +[IMembershipStrengthStrategy](Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.IMembershipStrengthStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This is the standard UMAP approach for computing fuzzy set membership strengths. +It applies an exponential kernel centered at the local connectivity distance (ρᵢ) +with bandwidth σᵢ: +

+
μᵢⱼ = {
+  1.0                           if dᵢⱼ ≤ ρᵢ or σᵢ = 0
+  exp(-(dᵢⱼ - ρᵢ) / σᵢ)        otherwise
+}
+

+After computing directed strengths, the algorithm applies fuzzy set operations to +symmetrize the graph. The set operation interpolates between fuzzy union and intersection: +

+
μ = α(μ_forward + μ_reverse - μ_forward × μ_reverse) + (1-α)(μ_forward × μ_reverse)
+

+where α is the set operation mix ratio (typically 1.0 for pure fuzzy union). +

+

+Characteristics: +

+
  • Time complexity: O(n × k) for computing strengths
  • Space complexity: O(n × k) sparse matrix
  • Graph density: Approximately k edges per node
  • Thread-safe: Yes for reading, exclusive write access needed
+

+Python reference: compute_membership_strengths() in umap_.py (lines ~260-330) +and fuzzy set operations in fuzzy_simplicial_set() (lines ~450-470). +

+ +## Methods + +### ComputeMembershipStrengths\(int\[\]\[\], float\[\]\[\], float\[\], float\[\], float\) + +Computes membership strengths for the fuzzy simplicial set. + +```csharp +public SparseMatrix ComputeMembershipStrengths(int[][] knnIndices, float[][] knnDistances, float[] sigmas, float[] rhos, float setOpMixRatio = 1) +``` + +#### Parameters + +`knnIndices` [int](https://learn.microsoft.com/dotnet/api/system.int32)\[\]\[\] + +Indices of k-nearest neighbors for each point. +Array shape: (n_samples, n_neighbors) + +`knnDistances` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Distances to k-nearest neighbors for each point. +Array shape: (n_samples, n_neighbors) + +`sigmas` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Bandwidth parameters from local metric computation. +Array shape: (n_samples,) + +`rhos` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Local connectivity distances from local metric computation. +Array shape: (n_samples,) + +`setOpMixRatio` [float](https://learn.microsoft.com/dotnet/api/system.single) + +Interpolation between fuzzy union (1.0) and intersection (0.0). +Controls how local fuzzy sets are combined into global structure. +Range: [0.0, 1.0] + +#### Returns + + SparseMatrix + +A sparse matrix representing the fuzzy simplicial set. +Shape: (n_samples, n_samples) +Matrix[i,j] represents the membership strength of the edge from i to j. +After set operations, the matrix should be symmetric. + +#### Remarks + +

+Implementation requirements: +

+
  1. Compute directed membership strengths μᵢⱼ for each edge
  2. Apply fuzzy set operation: μ = α(μᵢⱼ + μⱼᵢ - μᵢⱼμⱼᵢ) + (1-α)μᵢⱼμⱼᵢ
  3. Eliminate zero entries from sparse matrix
  4. Ensure matrix is symmetric after set operations
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.md new file mode 100644 index 00000000..cf182811 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.md @@ -0,0 +1,9 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations + +### Classes + + [ExponentialKernel](Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.ExponentialKernel.md) + +Exponential kernel-based membership strength computation. +Uses the standard UMAP exponential kernel to convert distances into probabilities. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.md new file mode 100644 index 00000000..bab9a33a --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.md @@ -0,0 +1,13 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength + +### Namespaces + + [Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations](Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.md) + +### Interfaces + + [IMembershipStrengthStrategy](Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.IMembershipStrengthStrategy.md) + +Strategy interface for computing fuzzy simplicial set membership strengths. +This is the third phase of the UMAP algorithm. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.INeighborSearchStrategy.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.INeighborSearchStrategy.md new file mode 100644 index 00000000..8759798b --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.INeighborSearchStrategy.md @@ -0,0 +1,73 @@ +# Interface INeighborSearchStrategy + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.md) +Assembly: Flowthru.Misc.ML.dll + +Strategy interface for computing k-nearest neighbors in high-dimensional space. +This is the first phase of the UMAP algorithm. + +```csharp +public interface INeighborSearchStrategy +``` + +## Remarks + +

+The neighbor search phase computes the k-nearest neighbors for each point in the dataset. +Different strategies provide different trade-offs between accuracy, speed, and memory usage: +

+
  • Exact methods (e.g., brute force): O(n²) time, 100% accurate, recommended for datasets < 4096 samples
  • Tree methods (e.g., KD-tree): O(n log n) time, exact or approximate, suitable for medium datasets with low-to-medium dimensions
  • Approximate methods (e.g., NN-Descent): O(n^1.14) time, ~99% accurate, recommended for datasets ≥ 4096 samples
  • Precomputed: O(1) time, user provides k-NN graph, suitable when neighbors are already known
+

+Python UMAP reference: nearest_neighbors() function in umap_.py (lines ~260-300) +

+ +## Methods + +### Search\(float\[\]\[\], int, IMetric, Random\) + +Computes k-nearest neighbors for all points in the dataset. + +```csharp +NeighborSearchResult Search(float[][] data, int nNeighbors, IMetric metric, Random random) +``` + +#### Parameters + +`data` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Input data as jagged array where each row represents a data point (n_samples × n_features). +data[i] is a float array of length n_features containing the feature values for sample i. +All rows must have the same length. + +`nNeighbors` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of nearest neighbors to find for each point. +Must be at least 2 and at most n_samples - 1. + +`metric` [IMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md) + +Distance metric for computing pairwise distances between points. + +`random` [Random](https://learn.microsoft.com/dotnet/api/system.random) + +Random number generator for any randomized algorithms (e.g., approximate search). +Ensures reproducibility when a seed is provided. + +#### Returns + + [NeighborSearchResult](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.NeighborSearchResult.md) + +A result containing: +- Indices: n_samples × n_neighbors array where Indices[i][j] is the index of the j-th nearest neighbor of point i +- Distances: n_samples × n_neighbors array where Distances[i][j] is the distance to that neighbor +- SearchIndex: Optional search index structure for future queries (e.g., for transform), or null if not applicable + +Note: Indices[i][0] should always be i (each point is its own nearest neighbor with distance 0). + +#### Remarks + +

+Implementation requirements: +

+
  1. Results must be sorted by distance (ascending) for each point
  2. First neighbor of each point should typically be itself (distance 0)
  3. For precomputed distances with disconnected components, use index -1 and distance ∞
  4. Thread-safe if marked as such in implementation
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.BruteForceSearch.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.BruteForceSearch.md new file mode 100644 index 00000000..4d6b291b --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.BruteForceSearch.md @@ -0,0 +1,101 @@ +# Class BruteForceSearch + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.md) +Assembly: Flowthru.Misc.ML.dll + +Brute-force exact k-nearest neighbor search. +Computes all pairwise distances - O(n²) time complexity. + +```csharp +public sealed class BruteForceSearch : INeighborSearchStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[BruteForceSearch](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.BruteForceSearch.md) + +#### Implements + +[INeighborSearchStrategy](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.INeighborSearchStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This implementation computes the exact k-nearest neighbors by calculating all pairwise +distances and selecting the k smallest for each point. While this is computationally +expensive for large datasets, it guarantees 100% accuracy and is the fastest approach +for small datasets (typically < 4096 samples). +

+

+Characteristics: +

+
  • Time complexity: O(n² × d) where n=samples, d=dimensions
  • Space complexity: O(n × k) for output
  • Accuracy: 100% (exact)
  • Recommended for: Small datasets (< 4096 samples)
  • Thread-safe: Yes (read-only operations)
+

+This is the reference implementation matching Python UMAP's behavior for small datasets +or when metric='precomputed' is not used. +

+

+Python reference: The brute-force path in nearest_neighbors() when exact k-NN +is computed via pairwise_distances() (Python UMAP lines ~2950-3000). +

+ +## Methods + +### Search\(float\[\]\[\], int, IMetric, Random\) + +Computes k-nearest neighbors for all points in the dataset. + +```csharp +public NeighborSearchResult Search(float[][] data, int nNeighbors, IMetric metric, Random random) +``` + +#### Parameters + +`data` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Input data as jagged array where each row represents a data point (n_samples × n_features). +data[i] is a float array of length n_features containing the feature values for sample i. +All rows must have the same length. + +`nNeighbors` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of nearest neighbors to find for each point. +Must be at least 2 and at most n_samples - 1. + +`metric` [IMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md) + +Distance metric for computing pairwise distances between points. + +`random` [Random](https://learn.microsoft.com/dotnet/api/system.random) + +Random number generator for any randomized algorithms (e.g., approximate search). +Ensures reproducibility when a seed is provided. + +#### Returns + + [NeighborSearchResult](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.NeighborSearchResult.md) + +A result containing: +- Indices: n_samples × n_neighbors array where Indices[i][j] is the index of the j-th nearest neighbor of point i +- Distances: n_samples × n_neighbors array where Distances[i][j] is the distance to that neighbor +- SearchIndex: Optional search index structure for future queries (e.g., for transform), or null if not applicable + +Note: Indices[i][0] should always be i (each point is its own nearest neighbor with distance 0). + +#### Remarks + +

+Implementation requirements: +

+
  1. Results must be sorted by distance (ascending) for each point
  2. First neighbor of each point should typically be itself (distance 0)
  3. For precomputed distances with disconnected components, use index -1 and distance ∞
  4. Thread-safe if marked as such in implementation
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.NNDescentSearch.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.NNDescentSearch.md new file mode 100644 index 00000000..9dbcb3b2 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.NNDescentSearch.md @@ -0,0 +1,216 @@ +# Class NNDescentSearch + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.md) +Assembly: Flowthru.Misc.ML.dll + +NN-Descent approximate k-nearest neighbor search. +Achieves ~99% accuracy with O(n^1.14) time complexity for large datasets. + +```csharp +public sealed class NNDescentSearch : INeighborSearchStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[NNDescentSearch](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.NNDescentSearch.md) + +#### Implements + +[INeighborSearchStrategy](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.INeighborSearchStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+NN-Descent is an iterative algorithm that efficiently constructs approximate k-nearest neighbor +graphs through a local join operation. It achieves sub-quadratic time complexity while maintaining +high accuracy (typically 99%+ recall). +

+

+Algorithm overview: +

+
  1. Initialize with random projection trees (RP-trees) for quality starting neighbors
  2. Fill remaining slots with random neighbors
  3. Iteratively refine via local join: compare candidate neighbor pairs
  4. Converge when update rate falls below threshold
+

+Performance characteristics: +

+
  • Time complexity: O(n^1.14 × d) empirically, vs O(n² × d) for brute-force
  • Space complexity: O(n × k + trees × n / leaf_size)
  • Accuracy: ~99% (approximate, configurable via parameters)
  • Recommended for: Large datasets (≥ 4096 samples)
  • Thread-safe: No (constructs new index per call)
+

+Based on: Dong, Moses, and Li. "Efficient K-Nearest Neighbor Graph Construction for Generic +Similarity Measures" (WWW 2011). Implementation follows PyNNDescent reference. +

+

+Python reference: nn_descent() function in pynndescent_.py and supporting functions +in utils.py and rp_trees.py from the PyNNDescent library. +

+ +## Properties + +### DeltaThreshold + +Convergence threshold as fraction of total edges. +Algorithm stops when updates per iteration drop below: delta × k × n. +Typical value: 0.001 (0.1% of edges changing). + +```csharp +public float DeltaThreshold { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single) + +### LeafSize + +Leaf size for random projection trees. +Smaller leaves increase tree depth and initialization quality. +Typical range: 10-20. + +```csharp +public int LeafSize { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +### LowMemory + +If true, uses block-based processing to reduce memory usage at cost of ~2x speed. +If false, maintains in-memory set for faster duplicate checking. + +```csharp +public bool LowMemory { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +### MaxCandidates + +Maximum number of candidate neighbors to consider per point per iteration. +If 0 (default), auto-configures as: min(60, k). +Higher values improve accuracy but increase iteration cost. + +```csharp +public int MaxCandidates { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +Controls the breadth of the local join search. Typical values: 30-60. +Each iteration costs O(n × max_candidates² × d). + +### MaxIterations + +Maximum number of NN-descent iterations. +If 0 (default), auto-configures as: max(5, round(log2(n))). +More iterations improve accuracy but increase runtime. + +```csharp +public int MaxIterations { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +Algorithm typically converges in 5-10 iterations via delta threshold. +Early stopping prevents unnecessary work. + +### NumTrees + +Number of random projection trees to build for initialization. +If 0 (default), auto-configures as: min(32, 5 + round(n^0.25)). +More trees improve initialization quality but increase build time. + +```csharp +public int NumTrees { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Remarks + +Python UMAP typically uses 5-32 trees depending on dataset size. +Each tree costs O(n log n × d) to build. + +### Verbose + +If true, prints progress information to console during search. + +```csharp +public bool Verbose { get; init; } +``` + +#### Property Value + + [bool](https://learn.microsoft.com/dotnet/api/system.boolean) + +## Methods + +### Search\(float\[\]\[\], int, IMetric, Random\) + +Computes k-nearest neighbors for all points in the dataset. + +```csharp +public NeighborSearchResult Search(float[][] data, int nNeighbors, IMetric metric, Random random) +``` + +#### Parameters + +`data` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Input data as jagged array where each row represents a data point (n_samples × n_features). +data[i] is a float array of length n_features containing the feature values for sample i. +All rows must have the same length. + +`nNeighbors` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Number of nearest neighbors to find for each point. +Must be at least 2 and at most n_samples - 1. + +`metric` [IMetric](Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md) + +Distance metric for computing pairwise distances between points. + +`random` [Random](https://learn.microsoft.com/dotnet/api/system.random) + +Random number generator for any randomized algorithms (e.g., approximate search). +Ensures reproducibility when a seed is provided. + +#### Returns + + [NeighborSearchResult](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.NeighborSearchResult.md) + +A result containing: +- Indices: n_samples × n_neighbors array where Indices[i][j] is the index of the j-th nearest neighbor of point i +- Distances: n_samples × n_neighbors array where Distances[i][j] is the distance to that neighbor +- SearchIndex: Optional search index structure for future queries (e.g., for transform), or null if not applicable + +Note: Indices[i][0] should always be i (each point is its own nearest neighbor with distance 0). + +#### Remarks + +

+Implementation requirements: +

+
  1. Results must be sorted by distance (ascending) for each point
  2. First neighbor of each point should typically be itself (distance 0)
  3. For precomputed distances with disconnected components, use index -1 and distance ∞
  4. Thread-safe if marked as such in implementation
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.md new file mode 100644 index 00000000..7408f2cd --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.md @@ -0,0 +1,14 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations + +### Classes + + [BruteForceSearch](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.BruteForceSearch.md) + +Brute-force exact k-nearest neighbor search. +Computes all pairwise distances - O(n²) time complexity. + + [NNDescentSearch](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.NNDescentSearch.md) + +NN-Descent approximate k-nearest neighbor search. +Achieves ~99% accuracy with O(n^1.14) time complexity for large datasets. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.NeighborSearchResult.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.NeighborSearchResult.md new file mode 100644 index 00000000..afdf614d --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.NeighborSearchResult.md @@ -0,0 +1,97 @@ +# Class NeighborSearchResult + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.md) +Assembly: Flowthru.Misc.ML.dll + +Result of a nearest neighbor search operation. + +```csharp +public sealed record NeighborSearchResult : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[NeighborSearchResult](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.NeighborSearchResult.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Constructors + +### NeighborSearchResult\(int\[\]\[\], float\[\]\[\], object?\) + +Result of a nearest neighbor search operation. + +```csharp +public NeighborSearchResult(int[][] Indices, float[][] Distances, object? SearchIndex) +``` + +#### Parameters + +`Indices` [int](https://learn.microsoft.com/dotnet/api/system.int32)\[\]\[\] + +Indices of k-nearest neighbors for each point. +Array shape: (n_samples, n_neighbors) + +`Distances` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +Distances to k-nearest neighbors for each point. +Array shape: (n_samples, n_neighbors) + +`SearchIndex` [object](https://learn.microsoft.com/dotnet/api/system.object)? + +Optional search index for future queries (used in transform operations). +May be null if the strategy doesn't support indexing. + +## Properties + +### Distances + +Distances to k-nearest neighbors for each point. +Array shape: (n_samples, n_neighbors) + +```csharp +public float[][] Distances { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)\[\]\[\] + +### Indices + +Indices of k-nearest neighbors for each point. +Array shape: (n_samples, n_neighbors) + +```csharp +public int[][] Indices { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32)\[\]\[\] + +### SearchIndex + +Optional search index for future queries (used in transform operations). +May be null if the strategy doesn't support indexing. + +```csharp +public object? SearchIndex { get; init; } +``` + +#### Property Value + + [object](https://learn.microsoft.com/dotnet/api/system.object)? + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.md new file mode 100644 index 00000000..a19d7fe7 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.md @@ -0,0 +1,19 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch + +### Namespaces + + [Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.md) + +### Classes + + [NeighborSearchResult](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.NeighborSearchResult.md) + +Result of a nearest neighbor search operation. + +### Interfaces + + [INeighborSearchStrategy](Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.INeighborSearchStrategy.md) + +Strategy interface for computing k-nearest neighbors in high-dimensional space. +This is the first phase of the UMAP algorithm. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.ISamplingScheduleStrategy.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.ISamplingScheduleStrategy.md new file mode 100644 index 00000000..f0bf09d8 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.ISamplingScheduleStrategy.md @@ -0,0 +1,76 @@ +# Interface ISamplingScheduleStrategy + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.md) +Assembly: Flowthru.Misc.ML.dll + +Strategy interface for computing edge sampling schedules during layout optimization. +This is the sixth phase of the UMAP algorithm. + +```csharp +public interface ISamplingScheduleStrategy +``` + +## Remarks + +

+The sampling schedule determines how frequently each edge in the fuzzy simplicial set +should be sampled during stochastic gradient descent. Edges with higher membership +strength (weight) are sampled more frequently. +

+

+Standard approach (proportional sampling): +

+

+Each edge is sampled proportionally to its weight. The number of epochs between samples +for an edge with weight w is: +

+
epochs_per_sample[i] = n_epochs / (n_epochs * weight[i] / max_weight)
+                     = max_weight / weight[i]
+

+This ensures that stronger edges (higher membership) are sampled more often, while +weaker edges may not be sampled at all if their expected sample count is less than 1. +

+

+Python UMAP reference: make_epochs_per_sample() function in umap_.py (lines 906-927) +

+ +## Methods + +### ComputeSchedule\(float\[\], int\) + +Computes the sampling schedule for edges during SGD optimization. + +```csharp +SamplingScheduleResult ComputeSchedule(float[] edgeWeights, int nEpochs) +``` + +#### Parameters + +`edgeWeights` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Array of edge weights from the fuzzy simplicial set. +These are the membership strengths after fuzzy set operations. +Length: number of edges in the graph + +`nEpochs` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Total number of optimization epochs to run. +Must be positive. + +#### Returns + + [SamplingScheduleResult](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.SamplingScheduleResult.md) + +Array of epochs-per-sample for each edge. +Value of epochs_per_sample[i] means edge i should be sampled +every epochs_per_sample[i] epochs (on average). +Edges with weight too small to be sampled are marked with -1. +Length: same as edgeWeights + +#### Remarks + +

+Implementation requirements: +

+
  1. Find maximum weight across all edges
  2. Compute expected number of samples per edge: n_epochs * (weight / max_weight)
  3. Invert to get epochs per sample: n_epochs / expected_samples
  4. Mark edges with expected_samples ≤ 0 as -1 (never sampled)
+ diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.ProportionalSampling.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.ProportionalSampling.md new file mode 100644 index 00000000..5c1f1cc0 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.ProportionalSampling.md @@ -0,0 +1,68 @@ +# Class ProportionalSampling + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.md) +Assembly: Flowthru.Misc.ML.dll + +Proportional sampling schedule where edges are sampled proportionally to their weights. +This is the standard UMAP sampling strategy. + +```csharp +public sealed class ProportionalSampling : ISamplingScheduleStrategy +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[ProportionalSampling](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.ProportionalSampling.md) + +#### Implements + +[ISamplingScheduleStrategy](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.ISamplingScheduleStrategy.md) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Remarks + +

+This implementation follows the Python UMAP reference implementation exactly. +Each edge is sampled with frequency proportional to its membership strength, +ensuring that stronger connections in the fuzzy simplicial set receive more +optimization attention. +

+

+Time complexity: O(E) where E is the number of edges +

+

+Space complexity: O(E) for the output array +

+

+Python UMAP reference: make_epochs_per_sample() in umap_.py (lines 906-927) +

+ +## Methods + +### ComputeSchedule\(float\[\], int\) + +Computes proportional sampling schedule for edges. + +```csharp +public SamplingScheduleResult ComputeSchedule(float[] edgeWeights, int nEpochs) +``` + +#### Parameters + +`edgeWeights` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +`nEpochs` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +#### Returns + + [SamplingScheduleResult](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.SamplingScheduleResult.md) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.md new file mode 100644 index 00000000..57f6c5a1 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.md @@ -0,0 +1,9 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations + +### Classes + + [ProportionalSampling](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.ProportionalSampling.md) + +Proportional sampling schedule where edges are sampled proportionally to their weights. +This is the standard UMAP sampling strategy. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.SamplingScheduleResult.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.SamplingScheduleResult.md new file mode 100644 index 00000000..aa00ab03 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.SamplingScheduleResult.md @@ -0,0 +1,81 @@ +# Class SamplingScheduleResult + +Namespace: [Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.md) +Assembly: Flowthru.Misc.ML.dll + +Result of sampling schedule computation. + +```csharp +public sealed record SamplingScheduleResult : IEquatable +``` + +#### Inheritance + +[object](https://learn.microsoft.com/dotnet/api/system.object) ← +[SamplingScheduleResult](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.SamplingScheduleResult.md) + +#### Implements + +[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1) + +#### Inherited Members + +[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)), +[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)), +[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode), +[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype), +[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals), +[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring) + +## Constructors + +### SamplingScheduleResult\(float\[\], int\) + +Result of sampling schedule computation. + +```csharp +public SamplingScheduleResult(float[] EpochsPerSample, int TotalExpectedSamples) +``` + +#### Parameters + +`EpochsPerSample` [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +Number of epochs between samples for each edge. +Array length matches the number of edges in the graph. +Value of -1 indicates the edge should never be sampled. + +`TotalExpectedSamples` [int](https://learn.microsoft.com/dotnet/api/system.int32) + +Total number of edge samples expected across all epochs. +Useful for progress estimation. + +## Properties + +### EpochsPerSample + +Number of epochs between samples for each edge. +Array length matches the number of edges in the graph. +Value of -1 indicates the edge should never be sampled. + +```csharp +public float[] EpochsPerSample { get; init; } +``` + +#### Property Value + + [float](https://learn.microsoft.com/dotnet/api/system.single)\[\] + +### TotalExpectedSamples + +Total number of edge samples expected across all epochs. +Useful for progress estimation. + +```csharp +public int TotalExpectedSamples { get; init; } +``` + +#### Property Value + + [int](https://learn.microsoft.com/dotnet/api/system.int32) + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.md b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.md new file mode 100644 index 00000000..bd192814 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.md @@ -0,0 +1,19 @@ +# Namespace Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule + +### Namespaces + + [Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.md) + +### Classes + + [SamplingScheduleResult](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.SamplingScheduleResult.md) + +Result of sampling schedule computation. + +### Interfaces + + [ISamplingScheduleStrategy](Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.ISamplingScheduleStrategy.md) + +Strategy interface for computing edge sampling schedules during layout optimization. +This is the sixth phase of the UMAP algorithm. + diff --git a/docs/reference/src/misc/Flowthru.Misc.ML/toc.yml b/docs/reference/src/misc/Flowthru.Misc.ML/toc.yml new file mode 100644 index 00000000..94d6bf67 --- /dev/null +++ b/docs/reference/src/misc/Flowthru.Misc.ML/toc.yml @@ -0,0 +1,156 @@ +### YamlMime:TableOfContent +- name: Flowthru.Misc.ML.UMAP.Core + href: Flowthru.Misc.ML.UMAP.Core.md + items: + - name: Classes + - name: CurveFitting + href: Flowthru.Misc.ML.UMAP.Core.CurveFitting.md + - name: DataShape + href: Flowthru.Misc.ML.UMAP.Core.DataShape.md + - name: UmapFitResult + href: Flowthru.Misc.ML.UMAP.Core.UmapFitResult.md + - name: UmapFlowBuilder + href: Flowthru.Misc.ML.UMAP.Core.UmapFlowBuilder.md + - name: UmapGraphResult + href: Flowthru.Misc.ML.UMAP.Core.UmapGraphResult.md + - name: UmapParameters + href: Flowthru.Misc.ML.UMAP.Core.UmapParameters.md + - name: UmapPipeline + href: Flowthru.Misc.ML.UMAP.Core.UmapPipeline.md + - name: UmapProgress + href: Flowthru.Misc.ML.UMAP.Core.UmapProgress.md + - name: UmapRuntimeReport + href: Flowthru.Misc.ML.UMAP.Core.UmapRuntimeReport.md +- name: Flowthru.Misc.ML.UMAP.Core.Markers + href: Flowthru.Misc.ML.UMAP.Core.Markers.md + items: + - name: Interfaces + - name: IMetric + href: Flowthru.Misc.ML.UMAP.Core.Markers.IMetric.md + - name: IOutputMetric + href: Flowthru.Misc.ML.UMAP.Core.Markers.IOutputMetric.md +- name: Flowthru.Misc.ML.UMAP.Core.Utils + href: Flowthru.Misc.ML.UMAP.Core.Utils.md + items: + - name: Classes + - name: CosineMetric + href: Flowthru.Misc.ML.UMAP.Core.Utils.CosineMetric.md + - name: CustomMetric + href: Flowthru.Misc.ML.UMAP.Core.Utils.CustomMetric.md + - name: EuclideanMetric + href: Flowthru.Misc.ML.UMAP.Core.Utils.EuclideanMetric.md + - name: ManhattanMetric + href: Flowthru.Misc.ML.UMAP.Core.Utils.ManhattanMetric.md +- name: Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement + href: Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.md + items: + - name: Classes + - name: GraphRefinementResult + href: Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.GraphRefinementResult.md + - name: Interfaces + - name: IGraphRefinementStrategy + href: Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.IGraphRefinementStrategy.md +- name: Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations + href: Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.md + items: + - name: Classes + - name: AdaptiveThresholding + href: Flowthru.Misc.ML.UMAP.Strategies.GraphRefinement.Implementations.AdaptiveThresholding.md +- name: Flowthru.Misc.ML.UMAP.Strategies.LayoutInit + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.md + items: + - name: Classes + - name: LayoutInitResult + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.LayoutInitResult.md + - name: Interfaces + - name: ILayoutInitStrategy + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.ILayoutInitStrategy.md +- name: Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.md + items: + - name: Classes + - name: RandomInit + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.RandomInit.md + - name: SpectralInit + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutInit.Implementations.SpectralInit.md +- name: Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.md + items: + - name: Classes + - name: LayoutOptimizationResult + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.LayoutOptimizationResult.md + - name: OptimizationParameters + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.OptimizationParameters.md + - name: Structs + - name: GraphEdge + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.GraphEdge.md + - name: Interfaces + - name: ILayoutOptimizationStrategy + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.ILayoutOptimizationStrategy.md +- name: Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.md + items: + - name: Classes + - name: EuclideanSGD + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGD.md + - name: EuclideanSGDOptimized + href: Flowthru.Misc.ML.UMAP.Strategies.LayoutOptimization.Implementations.EuclideanSGDOptimized.md +- name: Flowthru.Misc.ML.UMAP.Strategies.LocalMetric + href: Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.md + items: + - name: Classes + - name: LocalMetricResult + href: Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.LocalMetricResult.md + - name: Interfaces + - name: ILocalMetricStrategy + href: Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.ILocalMetricStrategy.md +- name: Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations + href: Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.md + items: + - name: Classes + - name: BinarySearchSmoothing + href: Flowthru.Misc.ML.UMAP.Strategies.LocalMetric.Implementations.BinarySearchSmoothing.md +- name: Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength + href: Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.md + items: + - name: Interfaces + - name: IMembershipStrengthStrategy + href: Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.IMembershipStrengthStrategy.md +- name: Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations + href: Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.md + items: + - name: Classes + - name: ExponentialKernel + href: Flowthru.Misc.ML.UMAP.Strategies.MembershipStrength.Implementations.ExponentialKernel.md +- name: Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch + href: Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.md + items: + - name: Classes + - name: NeighborSearchResult + href: Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.NeighborSearchResult.md + - name: Interfaces + - name: INeighborSearchStrategy + href: Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.INeighborSearchStrategy.md +- name: Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations + href: Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.md + items: + - name: Classes + - name: BruteForceSearch + href: Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.BruteForceSearch.md + - name: NNDescentSearch + href: Flowthru.Misc.ML.UMAP.Strategies.NeighborSearch.Implementations.NNDescentSearch.md +- name: Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule + href: Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.md + items: + - name: Classes + - name: SamplingScheduleResult + href: Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.SamplingScheduleResult.md + - name: Interfaces + - name: ISamplingScheduleStrategy + href: Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.ISamplingScheduleStrategy.md +- name: Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations + href: Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.md + items: + - name: Classes + - name: ProportionalSampling + href: Flowthru.Misc.ML.UMAP.Strategies.SamplingSchedule.Implementations.ProportionalSampling.md diff --git a/docs/tutorials/spaceflights/02-set-up-data.md b/docs/tutorials/spaceflights/02-set-up-data.md index 4631fd85..cdfe93d3 100644 --- a/docs/tutorials/spaceflights/02-set-up-data.md +++ b/docs/tutorials/spaceflights/02-set-up-data.md @@ -157,10 +157,10 @@ public partial class Catalog { /// ... - public ICatalogEntry> __NAME__ => + public IItem> __NAME__ => GetOrCreateEntry( () => - CatalogEntries.Enumerable.__FORMAT__<__SCHEMA__>( + Items.Enumerable.__FORMAT__<__SCHEMA__>( label: "__NAME__", path: "__PATH__" ) @@ -187,10 +187,10 @@ For our CompanySchema then: Our entry for Companies, then, will look like: ```cs - public ICatalogEntry> Companies => + public IItem> Companies => GetOrCreateEntry( () => - CatalogEntries.Enumerable.Csv( + Items.Enumerable.Csv( label: "Companies", path: $"{basePath}/Data/_01_Raw/Datasets/companies.csv" ) @@ -210,28 +210,28 @@ public partial class Catalog // Minimal "Names" Entry — keep it for now, to keep the project building. - public ICatalogEntry> Companies => + public IItem> Companies => GetOrCreateEntry( () => - CatalogEntries.Enumerable.Csv( + Items.Enumerable.Csv( label: "Companies", filePath: $"{_basePath}/_01_Raw/Datasets/companies.csv" ) ); - public ICatalogEntry> Reviews => + public IItem> Reviews => GetOrCreateEntry( () => - CatalogEntries.Enumerable.Csv( + Items.Enumerable.Csv( label: "Reviews", filePath: $"{_basePath}/_01_Raw/Datasets/reviews.csv" ) ); - public ICatalogEntry> Shuttles => + public IItem> Shuttles => GetOrCreateEntry( () => - CatalogEntries.Enumerable.Excel( + Items.Enumerable.Excel( label: "Shuttles", filePath: $"{_basePath}/_01_Raw/Datasets/shuttles.xlsx", sheetName: "Sheet1" diff --git a/examples/advanced/KedroSpaceflights.Custom/Data/Catalog.cs b/examples/advanced/KedroSpaceflights.Custom/Data/Catalog.cs index 4df30de8..d9ced3fe 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Data/Catalog.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Data/Catalog.cs @@ -19,7 +19,7 @@ namespace KedroSpaceflights.Custom.Data; /// _99_Reference: Reference data for validation /// /// -public partial class Catalog : DataCatalogBase +public partial class Catalog : CatalogAbstract { private readonly string _basePath; diff --git a/examples/advanced/KedroSpaceflights.Custom/Data/_01_Raw/Catalog.Raw.cs b/examples/advanced/KedroSpaceflights.Custom/Data/_01_Raw/Catalog.Raw.cs index 3df69275..bc086d7d 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Data/_01_Raw/Catalog.Raw.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Data/_01_Raw/Catalog.Raw.cs @@ -14,10 +14,10 @@ public partial class Catalog /// This is a critical Layer 0 input from an external source, configured for deep inspection /// to ensure data quality before pipeline execution. /// - public ICatalogEntry> Companies => - GetOrCreateEntry( + public IItem> Companies => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "RawCompanies", filePath: $"{_basePath}/_01_Raw/Datasets/companies.csv" ) @@ -31,10 +31,10 @@ public partial class Catalog /// This is a critical Layer 0 input from an external source, configured for deep inspection /// to ensure data quality before pipeline execution. /// - public ICatalogEntry> Reviews => - GetOrCreateEntry( + public IItem> Reviews => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "RawReviews", filePath: $"{_basePath}/_01_Raw/Datasets/reviews.csv" ) @@ -50,10 +50,10 @@ public partial class Catalog /// This is a critical Layer 0 input from an external source, configured for deep inspection /// to ensure data quality before pipeline execution. /// - public ICatalogEntry> Shuttles => - GetOrCreateEntry( + public IItem> Shuttles => + CreateItem( () => - CatalogEntries.Enumerable.Excel( + ItemFactory.Enumerable.Excel( label: "RawShuttles", filePath: $"{_basePath}/_01_Raw/Datasets/shuttles.xlsx", sheetName: "Sheet1" @@ -69,10 +69,10 @@ public partial class Catalog /// used for validation purposes in the DataDiagnostics pipeline. /// Uses KedroModelInputSchema to match the external Kedro CSV format. /// - public ICatalogEntry> KedroModelInputTable => - GetOrCreateEntry( + public IItem> KedroModelInputTable => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "KedroModelInputTable", filePath: $"{_basePath}/_01_Raw/Datasets/kedro_model_input_table.csv" ) diff --git a/examples/advanced/KedroSpaceflights.Custom/Data/_02_Intermediate/Catalog.Intermediate.cs b/examples/advanced/KedroSpaceflights.Custom/Data/_02_Intermediate/Catalog.Intermediate.cs index 72402a3a..e94dd20d 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Data/_02_Intermediate/Catalog.Intermediate.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Data/_02_Intermediate/Catalog.Intermediate.cs @@ -9,10 +9,10 @@ public partial class Catalog /// Preprocessed company data in Parquet format. /// Cleaned and validated company records. /// - public ICatalogEntry> CleanedCompanies => - GetOrCreateEntry( + public IItem> CleanedCompanies => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "CleanedCompanies", filePath: $"{_basePath}/_02_Intermediate/Datasets/cleaned_companies.parquet" ) @@ -22,10 +22,10 @@ public partial class Catalog /// Preprocessed shuttle data in Parquet format. /// Cleaned and validated shuttle records. /// - public ICatalogEntry> CleanedShuttles => - GetOrCreateEntry( + public IItem> CleanedShuttles => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "CleanedShuttles", filePath: $"{_basePath}/_02_Intermediate/Datasets/cleaned_shuttles.parquet" ) @@ -35,10 +35,10 @@ public partial class Catalog /// Preprocessed review data in Parquet format. /// Cleaned and validated review records with parsed numeric scores. /// - public ICatalogEntry> CleanedReviews => - GetOrCreateEntry( + public IItem> CleanedReviews => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "CleanedReviews", filePath: $"{_basePath}/_02_Intermediate/Datasets/cleaned_reviews.parquet" ) @@ -47,10 +47,10 @@ public partial class Catalog /// /// Preprocessed companies exported as CSV (for debugging). /// - public ICatalogEntry> CleanedCompaniesCsv => - GetOrCreateEntry( + public IItem> CleanedCompaniesCsv => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "CleanedCompaniesCsv", filePath: $"{_basePath}/_02_Intermediate/Datasets/cleaned_companies.csv" ) @@ -59,10 +59,10 @@ public partial class Catalog /// /// Preprocessed shuttles exported as CSV (for debugging). /// - public ICatalogEntry> CleanedShuttlesCsv => - GetOrCreateEntry( + public IItem> CleanedShuttlesCsv => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "CleanedShuttlesCsv", filePath: $"{_basePath}/_02_Intermediate/Datasets/cleaned_shuttles.csv" ) diff --git a/examples/advanced/KedroSpaceflights.Custom/Data/_03_Primary/Catalog.Primary.cs b/examples/advanced/KedroSpaceflights.Custom/Data/_03_Primary/Catalog.Primary.cs index 87beb44b..9408ab32 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Data/_03_Primary/Catalog.Primary.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Data/_03_Primary/Catalog.Primary.cs @@ -10,10 +10,10 @@ public partial class Catalog /// Model input table in Parquet format. /// Joined dataset ready for ML training. /// - public ICatalogEntry> ModelInputTable => - GetOrCreateEntry( + public IItem> ModelInputTable => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "ModelInputTable", filePath: $"{_basePath}/_03_Primary/Datasets/model_input_table.parquet" ) @@ -22,10 +22,10 @@ public partial class Catalog /// /// Model input table exported as minified JSON (compact, production-ready format). /// - public ICatalogEntry> ModelInputTableJsonMinified => - GetOrCreateEntry( + public IItem> ModelInputTableJsonMinified => + CreateItem( () => - CatalogEntries.Enumerable.Json( + ItemFactory.Enumerable.Json( label: "ModelInputTableJsonMinified", filePath: $"{_basePath}/_03_Primary/Datasets/model_input_table.min.json" ) @@ -34,10 +34,10 @@ public partial class Catalog /// /// Model input table exported as CSV (for debugging). /// - public ICatalogEntry> ModelInputTableCsv => - GetOrCreateEntry( + public IItem> ModelInputTableCsv => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "ModelInputTableCsv", filePath: $"{_basePath}/_03_Primary/Datasets/model_input_table.csv" ) @@ -48,18 +48,18 @@ public partial class Catalog /// Feature vectors for model training. /// Stored in memory as it's only used within the DataScience pipeline. /// - public ICatalogEntry> XTrain => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "XTrain")); + public IItem> XTrain => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "XTrain")); /// /// Testing features (X_test). /// Feature vectors for model evaluation. /// Stored as Parquet to enable cross-pipeline usage (DataEvaluation depends on this). /// - public ICatalogEntry> XTest => - GetOrCreateEntry( + public IItem> XTest => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "XTest", filePath: $"{_basePath}/_03_Primary/Datasets/x_test.parquet" ) @@ -70,18 +70,18 @@ public partial class Catalog /// Target prices for model training. /// Stored in memory as it's only used within the DataScience pipeline. /// - public ICatalogEntry> YTrain => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "YTrain")); + public IItem> YTrain => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "YTrain")); /// /// Testing targets (y_test). /// Target prices for model evaluation. /// Stored as Parquet to enable cross-pipeline usage (DataEvaluation depends on this). /// - public ICatalogEntry> YTest => - GetOrCreateEntry( + public IItem> YTest => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "YTest", filePath: $"{_basePath}/_03_Primary/Datasets/y_test.parquet" ) diff --git a/examples/advanced/KedroSpaceflights.Custom/Data/_04_Models/Catalog.Models.cs b/examples/advanced/KedroSpaceflights.Custom/Data/_04_Models/Catalog.Models.cs index 9cd6df7f..4ef5dac6 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Data/_04_Models/Catalog.Models.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Data/_04_Models/Catalog.Models.cs @@ -10,10 +10,10 @@ public partial class Catalog /// Contains intercept and coefficients for price prediction. /// Stored as JSON to enable cross-pipeline usage (DataEvaluation depends on this). /// - public ICatalogEntry Regressor => - GetOrCreateEntry( + public IItem Regressor => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "Regressor", filePath: $"{_basePath}/_04_Models/Datasets/regressor.json" ) diff --git a/examples/advanced/KedroSpaceflights.Custom/Data/_05_ModelOutput/Catalog.ModelOutput.cs b/examples/advanced/KedroSpaceflights.Custom/Data/_05_ModelOutput/Catalog.ModelOutput.cs index bcb7d0af..f828df88 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Data/_05_ModelOutput/Catalog.ModelOutput.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Data/_05_ModelOutput/Catalog.ModelOutput.cs @@ -10,10 +10,10 @@ public partial class Catalog /// Contains R², MAE, RMSE, etc. /// Stored as a singleton object (pipeline produces single metrics object). /// - public ICatalogEntry ModelMetrics => - GetOrCreateEntry( + public IItem ModelMetrics => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "ModelMetrics", filePath: $"{_basePath}/_05_ModelOutput/Datasets/model_metrics.json" ) @@ -23,10 +23,10 @@ public partial class Catalog /// Model predictions with actual and predicted values. /// Used for evaluation and visualization. /// - public ICatalogEntry> ModelPredictions => - GetOrCreateEntry( + public IItem> ModelPredictions => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "ModelPredictions", filePath: $"{_basePath}/_05_ModelOutput/Datasets/model_predictions.csv" ) diff --git a/examples/advanced/KedroSpaceflights.Custom/Data/_06_Reporting/Catalog.Reporting.cs b/examples/advanced/KedroSpaceflights.Custom/Data/_06_Reporting/Catalog.Reporting.cs index e4af2e4f..a6e539b8 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Data/_06_Reporting/Catalog.Reporting.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Data/_06_Reporting/Catalog.Reporting.cs @@ -11,10 +11,10 @@ public partial class Catalog /// Contains metrics for each fold, mean, std dev, and comparison to Kedro. /// Stored as JSON to preserve nested List<FoldMetric> structure. /// - public ICatalogEntry CrossValidationResults => - GetOrCreateEntry( + public IItem CrossValidationResults => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "CrossValidationResults", filePath: $"{_basePath}/_06_Reporting/Datasets/cross_validation_results.json" ) @@ -24,10 +24,10 @@ public partial class Catalog /// Cross-validation summary report in Markdown format. /// Human-readable report summarizing model performance and validation results. /// - public ICatalogEntry CrossValidationReport => - GetOrCreateEntry( + public IItem CrossValidationReport => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "CrossValidationReport", filePath: $"{_basePath}/_06_Reporting/Datasets/cross_validation_report.md" ) @@ -37,9 +37,9 @@ public partial class Catalog /// Shuttle passenger capacity bar chart (in-memory GenericChart). /// Intermediate chart object stored in memory for downstream export to multiple formats. /// - public ICatalogEntry ShuttlePassengerCapacityChart => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "ShuttlePassengerCapacityChart") + public IItem ShuttlePassengerCapacityChart => + CreateItem( + () => ItemFactory.Single.Memory(label: "ShuttlePassengerCapacityChart") ); /// @@ -52,10 +52,10 @@ public partial class Catalog /// figure specification with data traces and layout configuration. Can be rendered in browsers /// using plotly.js or converted to static images using Plotly.NET.ImageExport. /// - public ICatalogEntry ShuttlePassengerCapacityPlot => - GetOrCreateEntry( + public IItem ShuttlePassengerCapacityPlot => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "ShuttlePassengerCapacityPlot", filePath: $"{_basePath}/_06_Reporting/Datasets/shuttle_passenger_capacity_plot.json" ) @@ -65,10 +65,8 @@ public partial class Catalog /// Confusion matrix heatmap (in-memory GenericChart). /// Intermediate chart object stored in memory for downstream export to multiple formats. /// - public ICatalogEntry ConfusionMatrixChart => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "ConfusionMatrixChart") - ); + public IItem ConfusionMatrixChart => + CreateItem(() => ItemFactory.Single.Memory(label: "ConfusionMatrixChart")); /// /// Confusion matrix heatmap visualization (Plotly JSON). @@ -80,10 +78,10 @@ public partial class Catalog /// The heatmap displays a 2x2 confusion matrix with color-coded cells showing classification /// performance. JSON format allows browser-based rendering and potential conversion to PNG. /// - public ICatalogEntry ConfusionMatrixPlot => - GetOrCreateEntry( + public IItem ConfusionMatrixPlot => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "ConfusionMatrixPlot", filePath: $"{_basePath}/_06_Reporting/Datasets/confusion_matrix_plot.json" ) @@ -95,13 +93,13 @@ public partial class Catalog /// Stored as binary PNG file. /// /// - /// Uses CatalogEntries.Binary factory method to store actual PNG binary data with proper file format. + /// Uses ItemFactory.Binary factory method to store actual PNG binary data with proper file format. /// The PNG file can be opened directly in image viewers or embedded in reports. /// - public ICatalogEntry ShuttlePassengerCapacityPlotPng => - GetOrCreateEntry( + public IItem ShuttlePassengerCapacityPlotPng => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "ShuttlePassengerCapacityPlotPng", filePath: $"{_basePath}/_06_Reporting/Datasets/shuttle_passenger_capacity_plot.png" ) @@ -113,13 +111,13 @@ public partial class Catalog /// Stored as binary PNG file. /// /// - /// Uses CatalogEntries.Binary factory method to store actual PNG binary data with proper file format. + /// Uses ItemFactory.Binary factory method to store actual PNG binary data with proper file format. /// The PNG file can be opened directly in image viewers or embedded in reports. /// - public ICatalogEntry ConfusionMatrixPlotPng => - GetOrCreateEntry( + public IItem ConfusionMatrixPlotPng => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "ConfusionMatrixPlotPng", filePath: $"{_basePath}/_06_Reporting/Datasets/confusion_matrix_plot.png" ) @@ -130,10 +128,8 @@ public partial class Catalog /// Intermediate chart object showing R² distribution analysis with scatter plot, /// normal curve, mean line, and Kedro reference line. /// - public ICatalogEntry CrossValidationChart => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "CrossValidationChart") - ); + public IItem CrossValidationChart => + CreateItem(() => ItemFactory.Single.Memory(label: "CrossValidationChart")); /// /// Cross-validation visualization (Plotly JSON). @@ -146,10 +142,10 @@ public partial class Catalog /// /// Stored as Plotly JSON specification for interactive visualization. /// - public ICatalogEntry CrossValidationPlot => - GetOrCreateEntry( + public IItem CrossValidationPlot => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "CrossValidationPlot", filePath: $"{_basePath}/_06_Reporting/Datasets/cross_validation_plot.json" ) @@ -161,13 +157,13 @@ public partial class Catalog /// Stored as binary PNG file. /// /// - /// Uses CatalogEntries.Binary factory method to store actual PNG binary data with proper file format. + /// Uses ItemFactory.Binary factory method to store actual PNG binary data with proper file format. /// The PNG file can be opened directly in image viewers or embedded in reports. /// - public ICatalogEntry CrossValidationPlotPng => - GetOrCreateEntry( + public IItem CrossValidationPlotPng => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "CrossValidationPlotPng", filePath: $"{_basePath}/_06_Reporting/Datasets/cross_validation_plot.png" ) @@ -178,10 +174,8 @@ public partial class Catalog /// Intermediate chart object showing actual vs predicted values with color-coded dots /// (yellow for over-estimates, red for under-estimates) and a 1:1 identity reference line. /// - public ICatalogEntry PredictionScatterChart => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "PredictionScatterChart") - ); + public IItem PredictionScatterChart => + CreateItem(() => ItemFactory.Single.Memory(label: "PredictionScatterChart")); /// /// Prediction scatter plot visualization (Plotly JSON). @@ -194,10 +188,10 @@ public partial class Catalog /// /// Stored as Plotly JSON specification for interactive visualization. /// - public ICatalogEntry PredictionScatterPlot => - GetOrCreateEntry( + public IItem PredictionScatterPlot => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "PredictionScatterPlot", filePath: $"{_basePath}/_06_Reporting/Datasets/prediction_scatter_plot.json" ) @@ -209,13 +203,13 @@ public partial class Catalog /// Stored as binary PNG file at 600x600 resolution. /// /// - /// Uses CatalogEntries.Binary factory method to store actual PNG binary data with proper file format. + /// Uses ItemFactory.Binary factory method to store actual PNG binary data with proper file format. /// The PNG file can be opened directly in image viewers or embedded in reports. /// - public ICatalogEntry PredictionScatterPlotPng => - GetOrCreateEntry( + public IItem PredictionScatterPlotPng => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "PredictionScatterPlotPng", filePath: $"{_basePath}/_06_Reporting/Datasets/prediction_scatter_plot.png" ) diff --git a/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataDiagnostics/DataDiagnosticsPipeline.cs b/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataDiagnostics/DataDiagnosticsPipeline.cs index f41445bd..45e4e062 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataDiagnostics/DataDiagnosticsPipeline.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataDiagnostics/DataDiagnosticsPipeline.cs @@ -1,5 +1,5 @@ -using Flowthru.Nodes; -using Flowthru.Pipelines; +using Flowthru.Flows; +using Flowthru.Steps; using KedroSpaceflights.Custom.Data; using KedroSpaceflights.Custom.Data._02_Intermediate.Schemas; using KedroSpaceflights.Custom.Data._03_Primary.Schemas; @@ -31,12 +31,12 @@ namespace KedroSpaceflights.Custom.Pipelines.DataDiagnostics; /// public static class DataDiagnosticsPipeline { - public static Pipeline Create(Catalog catalog) + public static Flow Create(Catalog catalog) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // Node 1: Validate model input table against Kedro reference output (demonstrates NoData output pattern) - pipeline.AddNode( + pipeline.AddStep( label: "ValidateModelInputTableAgainstKedroSource", transform: ValidateAgainstKedroNode.Create(), input: (catalog.ModelInputTable, catalog.KedroModelInputTable), @@ -44,7 +44,7 @@ public static Pipeline Create(Catalog catalog) ); // Node 2: Export cleaned companies to CSV for manual inspection - pipeline.AddNode( + pipeline.AddStep( label: "ExportCompaniesToDiagnosticCsv", transform: PassthroughInputToOutputNode.Create(), input: catalog.CleanedCompanies, @@ -52,7 +52,7 @@ public static Pipeline Create(Catalog catalog) ); // Node 3: Export cleaned shuttles to CSV for manual inspection - pipeline.AddNode( + pipeline.AddStep( label: "ExportShuttlesToDiagnosticCsv", transform: PassthroughInputToOutputNode.Create(), input: catalog.CleanedShuttles, @@ -60,7 +60,7 @@ public static Pipeline Create(Catalog catalog) ); // Node 4: Export model input table to CSV for manual inspection - pipeline.AddNode( + pipeline.AddStep( label: "ExportModelInputTableToDiagnosticCsv", transform: PassthroughInputToOutputNode.Create(), input: catalog.ModelInputTable, @@ -68,7 +68,7 @@ public static Pipeline Create(Catalog catalog) ); // Node 5: Export model input table to minified JSON for production/compact storage - pipeline.AddNode( + pipeline.AddStep( label: "ExportModelInputTableToMinifiedJson", transform: PassthroughInputToOutputNode.Create(), input: catalog.ModelInputTable, diff --git a/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataDiagnostics/Nodes/ValidateAgainstKedroNode.cs b/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataDiagnostics/Nodes/ValidateAgainstKedroNode.cs index d0f7db30..a3813ca4 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataDiagnostics/Nodes/ValidateAgainstKedroNode.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataDiagnostics/Nodes/ValidateAgainstKedroNode.cs @@ -1,5 +1,5 @@ using System.Reflection; -using Flowthru.Nodes; +using Flowthru.Steps; using KedroSpaceflights.Custom.Data._01_Raw.Schemas; using KedroSpaceflights.Custom.Data._03_Primary.Schemas; diff --git a/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataEvaluation/DataEvaluationPipeline.cs b/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataEvaluation/DataEvaluationPipeline.cs index d3f6becb..bbe5c5b4 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataEvaluation/DataEvaluationPipeline.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataEvaluation/DataEvaluationPipeline.cs @@ -1,5 +1,5 @@ -using Flowthru.Nodes; -using Flowthru.Pipelines; +using Flowthru.Flows; +using Flowthru.Steps; using KedroSpaceflights.Custom.Data; using KedroSpaceflights.Custom.Data._03_Primary.Schemas; using KedroSpaceflights.Custom.Data._04_Models.Schemas; @@ -43,12 +43,12 @@ public class Params public CrossValidateModelNode.Params CrossValidationParams { get; init; } = new(); } - public static Pipeline Create(Catalog catalog, Params parameters) + public static Flow Create(Catalog catalog, Params parameters) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // Node 1: Evaluate OLS model (multi-input → multi-output) - pipeline.AddNode( + pipeline.AddStep( label: "EvaluateOLSModel", transform: EvaluateModelNode.Create(), input: (catalog.Regressor, catalog.XTest, catalog.YTest), @@ -56,7 +56,7 @@ public static Pipeline Create(Catalog catalog, Params parameters) ); // Node 2: Cross-validation for R² distribution analysis and comparison to Kedro - pipeline.AddNode( + pipeline.AddStep( label: "PerformCrossValidatedOLSRegressionTest", transform: CrossValidateModelNode.Create(parameters.CrossValidationParams), input: catalog.ModelInputTable, diff --git a/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataProcessing/DataProcessingPipeline.cs b/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataProcessing/DataProcessingPipeline.cs index 14962d15..46359e38 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataProcessing/DataProcessingPipeline.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataProcessing/DataProcessingPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using KedroSpaceflights.Custom.Data; using KedroSpaceflights.Custom.Pipelines.DataProcessing.Nodes; @@ -27,12 +27,12 @@ namespace KedroSpaceflights.Custom.Pipelines.DataProcessing; /// public static class DataProcessingPipeline { - public static Pipeline Create(Catalog catalog) + public static Flow Create(Catalog catalog) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // Node 1: Preprocess companies (simple: single input → single output) - pipeline.AddNode( + pipeline.AddStep( label: "PreprocessCompanies", transform: PreprocessCompaniesNode.Create(), input: catalog.Companies, @@ -40,7 +40,7 @@ public static Pipeline Create(Catalog catalog) ); // Node 2: Preprocess shuttles (simple: single input → single output) - pipeline.AddNode( + pipeline.AddStep( label: "PreprocessShuttles", transform: PreprocessShuttlesNode.Create(), input: catalog.Shuttles, @@ -50,7 +50,7 @@ public static Pipeline Create(Catalog catalog) // Node 3: Preprocess reviews (simple: single input → single output) // Note: Minor refactor compared to Kedro - we preprocess reviews separately // rather than handling raw reviews in create_model_input_table - pipeline.AddNode( + pipeline.AddStep( label: "PreprocessReviews", transform: PreprocessReviewsNode.Create(), input: catalog.Reviews, @@ -58,7 +58,7 @@ public static Pipeline Create(Catalog catalog) ); // Node 4: Create model input table (multi-input: 3 inputs → single output) - pipeline.AddNode( + pipeline.AddStep( label: "CreateModelInputTable", transform: CreateModelInputTableNode.Create(), input: (catalog.CleanedShuttles, catalog.CleanedCompanies, catalog.CleanedReviews), diff --git a/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataScience/DataSciencePipeline.cs b/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataScience/DataSciencePipeline.cs index e6b06614..054e3929 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataScience/DataSciencePipeline.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Pipelines/DataScience/DataSciencePipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using KedroSpaceflights.Custom.Data; using KedroSpaceflights.Custom.Data._03_Primary.Schemas; using KedroSpaceflights.Custom.Pipelines.DataEvaluation.Nodes; @@ -44,12 +44,12 @@ public record Params public CreateTestTrainSplitNode.TestTrainSplitParams ModelParams { get; init; } = new(); } - public static Pipeline Create(Catalog catalog, Params parameters) + public static Flow Create(Catalog catalog, Params parameters) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // Node 1: Split data into train/test sets (single input → multi-output) - pipeline.AddNode( + pipeline.AddStep( label: "CreateTestTrainSplitDatasets", transform: CreateTestTrainSplitNode.Create(parameters: parameters.ModelParams), input: catalog.ModelInputTable, @@ -57,7 +57,7 @@ public static Pipeline Create(Catalog catalog, Params parameters) ); // Node 2: Train OLS regression model (multi-input → single output) - pipeline.AddNode( + pipeline.AddStep( label: "TrainOLSModel", transform: TrainModelNode.Create(), input: (catalog.XTrain, catalog.YTrain), diff --git a/examples/advanced/KedroSpaceflights.Custom/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs b/examples/advanced/KedroSpaceflights.Custom/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs index 21292fca..63e92379 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs @@ -11,13 +11,13 @@ namespace KedroSpaceflights.Custom.Pipelines.Reporting.Nodes; /// /// This node handles the PNG export concern separately from chart generation, /// enabling a clean separation of visualization logic from output format concerns. -/// The PNG binary data can be stored in a BinaryFileCatalogEntry<byte[]>. +/// The PNG binary data can be stored in a BinaryFileItem<byte[]>. /// /// /// Input: GenericChart object from memory catalog /// /// -/// Output: PNG binary data as byte[] (can be stored in BinaryFileCatalogEntry<byte[]>) +/// Output: PNG binary data as byte[] (can be stored in BinaryFileItem<byte[]>) /// /// /// Reusability: This node can be used for any Plotly chart type diff --git a/examples/advanced/KedroSpaceflights.Custom/Pipelines/Reporting/ReportingPipeline.cs b/examples/advanced/KedroSpaceflights.Custom/Pipelines/Reporting/ReportingPipeline.cs index c8ffeb4d..50b762cc 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Pipelines/Reporting/ReportingPipeline.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Pipelines/Reporting/ReportingPipeline.cs @@ -1,5 +1,5 @@ -using Flowthru.Nodes; -using Flowthru.Pipelines; +using Flowthru.Flows; +using Flowthru.Steps; using KedroSpaceflights.Custom.Data; using KedroSpaceflights.Custom.Pipelines.Reporting.Nodes; @@ -42,14 +42,14 @@ namespace KedroSpaceflights.Custom.Pipelines.Reporting; /// public static class ReportingPipeline { - public static Pipeline Create(Catalog catalog) + public static Flow Create(Catalog catalog) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // ===== Shuttle Passenger Capacity Visualization ===== // Step 1: Generate chart from processed shuttle data - pipeline.AddNode( + pipeline.AddStep( label: "GeneratePassengerCapacityChart", transform: ComparePassengerCapacityNode.Create(), input: catalog.CleanedShuttles, @@ -57,7 +57,7 @@ public static Pipeline Create(Catalog catalog) ); // Step 2: Export chart to JSON for interactive visualization - pipeline.AddNode( + pipeline.AddStep( label: "ExportPassengerCapacityJson", transform: PlotlyJsonExportNode.Create(), input: catalog.ShuttlePassengerCapacityChart, @@ -66,7 +66,7 @@ public static Pipeline Create(Catalog catalog) // Removed to cut down Chromium dependency for test runs. // // Step 3: Export chart to base64-encoded PNG for static reports - // pipeline.AddNode( + // pipeline.AddStep( // label: "ExportPassengerCapacityPng", // transform: PlotlyImageExportNode.Create(), // input: catalog.ShuttlePassengerCapacityChart, @@ -76,7 +76,7 @@ public static Pipeline Create(Catalog catalog) // ===== Confusion Matrix Visualization ===== // Step 1: Generate confusion matrix heatmap from company data - pipeline.AddNode( + pipeline.AddStep( label: "GenerateConfusionMatrixChart", transform: CreateConfusionMatrixNode.Create(), input: catalog.CleanedCompanies, @@ -84,7 +84,7 @@ public static Pipeline Create(Catalog catalog) ); // Step 2: Export chart to JSON for interactive visualization - pipeline.AddNode( + pipeline.AddStep( label: "ExportConfusionMatrixJson", transform: PlotlyJsonExportNode.Create(), input: catalog.ConfusionMatrixChart, @@ -93,7 +93,7 @@ public static Pipeline Create(Catalog catalog) // Removed to cut down Chromium dependency for test runs. // // Step 3: Export chart to base64-encoded PNG for static reports - // pipeline.AddNode( + // pipeline.AddStep( // label: "ExportConfusionMatrixPng", // transform: PlotlyImageExportNode.Create(), // input: catalog.ConfusionMatrixChart, @@ -103,7 +103,7 @@ public static Pipeline Create(Catalog catalog) // ===== Cross-Validation Results Visualization ===== // Step 1: Generate comprehensive cross-validation chart - pipeline.AddNode( + pipeline.AddStep( label: "GenerateCrossValidationChart", transform: VisualizeCrossValidationNode.Create(), input: catalog.CrossValidationResults, @@ -111,7 +111,7 @@ public static Pipeline Create(Catalog catalog) ); // Step 2: Export chart to JSON for interactive visualization - pipeline.AddNode( + pipeline.AddStep( label: "ExportCrossValidationJson", transform: PlotlyJsonExportNode.Create(), input: catalog.CrossValidationChart, @@ -120,7 +120,7 @@ public static Pipeline Create(Catalog catalog) // Removed to cut down Chromium dependency for test runs. // Step 3: Export chart to base64-encoded PNG for static reports - // pipeline.AddNode( + // pipeline.AddStep( // label: "ExportCrossValidationPng", // transform: PlotlyImageExportNode.Create(), // input: catalog.CrossValidationChart, @@ -128,7 +128,7 @@ public static Pipeline Create(Catalog catalog) // ); // Node 6: Generate human-readable Markdown report from cross-validation results - pipeline.AddNode( + pipeline.AddStep( label: "GenerateCrossValidationReport", transform: GenerateCrossValidationReportNode.Create(), input: catalog.CrossValidationResults, @@ -138,7 +138,7 @@ public static Pipeline Create(Catalog catalog) // ===== Prediction Scatter Plot Visualization ===== // Step 1: Generate scatter plot from model metrics and predictions - pipeline.AddNode( + pipeline.AddStep( label: "GeneratePredictionScatterChart", transform: GeneratePredictionScatterNode.Create(), input: (catalog.ModelMetrics, catalog.ModelPredictions), @@ -146,7 +146,7 @@ public static Pipeline Create(Catalog catalog) ); // Step 2: Export chart to JSON for interactive visualization - pipeline.AddNode( + pipeline.AddStep( label: "ExportPredictionScatterJson", transform: PlotlyJsonExportNode.Create(), input: catalog.PredictionScatterChart, @@ -155,7 +155,7 @@ public static Pipeline Create(Catalog catalog) // Removed to cut down Chromium dependency for test runs. // // Step 3: Export chart to PNG for static reports - // pipeline.AddNode( + // pipeline.AddStep( // label: "ExportPredictionScatterPng", // transform: PlotlyImageExportNode.Create(), // input: catalog.PredictionScatterChart, diff --git a/examples/advanced/KedroSpaceflights.Custom/Program.cs b/examples/advanced/KedroSpaceflights.Custom/Program.cs index 2a38f76d..a7862b8e 100644 --- a/examples/advanced/KedroSpaceflights.Custom/Program.cs +++ b/examples/advanced/KedroSpaceflights.Custom/Program.cs @@ -53,7 +53,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa // Enable configuration loading from appsettings.json files // This loads: appsettings.json (base) -> appsettings.{Environment}.json -> appsettings.Local.json flowthru.UseConfiguration(opts => opts.ConfigurationPath = basePath); - flowthru.UseCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); + flowthru.RegisterCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); flowthru.ConfigureMetadata(meta => { var metadataPath = Path.Combine(basePath, "Metadata"); @@ -66,33 +66,33 @@ private static void ConfigureServices(IServiceCollection services, string basePa }); flowthru - .RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create) + .RegisterFlow(label: "DataProcessing", flow: DataProcessingPipeline.Create) .WithDescription("Preprocesses raw data and creates model input table"); flowthru - .RegisterPipeline( + .RegisterFlow( label: "DataScience", - pipeline: DataSciencePipeline.Create, + flow: DataSciencePipeline.Create, configurationSection: "Flowthru:Pipelines:DataScience" ) .WithDescription("Trains ML model"); flowthru - .RegisterPipeline(label: "DataDiagnostics", pipeline: DataDiagnosticsPipeline.Create) + .RegisterFlow(label: "DataDiagnostics", flow: DataDiagnosticsPipeline.Create) .WithDescription( "Validates pipeline outputs against Kedro reference and exports diagnostic data" ); flowthru - .RegisterPipeline( + .RegisterFlow( label: "DataEvaluation", - pipeline: DataEvaluationPipeline.Create, + flow: DataEvaluationPipeline.Create, configurationSection: "Flowthru:Pipelines:DataEvaluation" ) .WithDescription("Evaluates ML model performance and cross-validation"); flowthru - .RegisterPipeline(label: "Reporting", pipeline: ReportingPipeline.Create) + .RegisterFlow(label: "Reporting", flow: ReportingPipeline.Create) .WithDescription("Generates reports and visualizations"); }); diff --git a/examples/advanced/RetailDataMultipipeline/Data/CoreCatalog.cs b/examples/advanced/RetailDataMultipipeline/Data/CoreCatalog.cs index e3799ffc..65c70e9b 100644 --- a/examples/advanced/RetailDataMultipipeline/Data/CoreCatalog.cs +++ b/examples/advanced/RetailDataMultipipeline/Data/CoreCatalog.cs @@ -2,7 +2,7 @@ namespace RetailDataMultipipeline.Data; -public partial class CoreCatalog : DataCatalogBase +public partial class CoreCatalog : CatalogAbstract { private readonly string _basePath; diff --git a/examples/advanced/RetailDataMultipipeline/Data/_01_Raw/CoreCatalog.Raw.cs b/examples/advanced/RetailDataMultipipeline/Data/_01_Raw/CoreCatalog.Raw.cs index 45db49e6..2a8078d6 100644 --- a/examples/advanced/RetailDataMultipipeline/Data/_01_Raw/CoreCatalog.Raw.cs +++ b/examples/advanced/RetailDataMultipipeline/Data/_01_Raw/CoreCatalog.Raw.cs @@ -9,10 +9,10 @@ public partial class CoreCatalog /// All daily retail transaction CSV files from the by-day directory, read as a /// single concatenated sequence. Read-only — immutable raw source data. /// - public ICatalogEntry> RetailTransactionsRaw => - GetOrCreateEntry( + public IItem> RetailTransactionsRaw => + CreateItem( () => - CatalogEntries.Enumerable.CsvDirectory( + ItemFactory.Enumerable.CsvDirectory( label: "RetailTransactionsRaw", directoryPath: $"{_basePath}/_01_Raw/Datasets" ) @@ -21,10 +21,10 @@ public partial class CoreCatalog /// /// Country-to-currency mapping. Maintained independently of the OFX feed. /// - public ICatalogEntry> CountryCurrencies => - GetOrCreateEntry( + public IItem> CountryCurrencies => + CreateItem( () => - CatalogEntries.Enumerable.Json( + ItemFactory.Enumerable.Json( label: "CountryCurrencies", filePath: $"{_basePath}/_01_Raw/Datasets/country_currencies.json" ) @@ -33,10 +33,10 @@ public partial class CoreCatalog /// /// Stubbed OFX XXX/GBP/1000 responses — one per source currency. /// - public ICatalogEntry> OfxRates => - GetOrCreateEntry( + public IItem> OfxRates => + CreateItem( () => - CatalogEntries.Enumerable.Json( + ItemFactory.Enumerable.Json( label: "OfxRates", filePath: $"{_basePath}/_01_Raw/Datasets/ofx_rates.json" ) diff --git a/examples/advanced/RetailDataMultipipeline/Data/_02_Intermediate/CoreCatalog.Intermediate.cs b/examples/advanced/RetailDataMultipipeline/Data/_02_Intermediate/CoreCatalog.Intermediate.cs index 73c98b85..ddae918b 100644 --- a/examples/advanced/RetailDataMultipipeline/Data/_02_Intermediate/CoreCatalog.Intermediate.cs +++ b/examples/advanced/RetailDataMultipipeline/Data/_02_Intermediate/CoreCatalog.Intermediate.cs @@ -9,10 +9,10 @@ public partial class CoreCatalog /// All retail transactions consolidated from the daily CSV files into a single /// Parquet dataset with fully-typed columns. This is the canonical "full history" view. /// - public ICatalogEntry> AllRetailTransactions => - GetOrCreateEntry( + public IItem> AllRetailTransactions => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "AllRetailTransactions", filePath: $"{_basePath}/_02_Intermediate/Datasets/all_retail_transactions.parquet" ) diff --git a/examples/advanced/RetailDataMultipipeline/Data/_03_Primary/CoreCatalog.Primary.cs b/examples/advanced/RetailDataMultipipeline/Data/_03_Primary/CoreCatalog.Primary.cs index a05a95b8..d3f23ca3 100644 --- a/examples/advanced/RetailDataMultipipeline/Data/_03_Primary/CoreCatalog.Primary.cs +++ b/examples/advanced/RetailDataMultipipeline/Data/_03_Primary/CoreCatalog.Primary.cs @@ -12,10 +12,10 @@ public partial class CoreCatalog /// All per-country weekly DTU shards concatenated into a single Parquet dataset. /// Produced by the Consolidation pipeline as the fan-in of all shard outputs. /// - public ICatalogEntry> AllCountriesWeeklyDtu => - GetOrCreateEntry( + public IItem> AllCountriesWeeklyDtu => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "AllCountriesWeeklyDtu", filePath: $"{_basePath}/_03_Primary/Datasets/all_countries_weekly_dtu.parquet" ) diff --git a/examples/advanced/RetailDataMultipipeline/Data/_03_Primary/CountryShardCatalog.cs b/examples/advanced/RetailDataMultipipeline/Data/_03_Primary/CountryShardCatalog.cs index 79bc2658..2d6199b0 100644 --- a/examples/advanced/RetailDataMultipipeline/Data/_03_Primary/CountryShardCatalog.cs +++ b/examples/advanced/RetailDataMultipipeline/Data/_03_Primary/CountryShardCatalog.cs @@ -11,7 +11,7 @@ namespace RetailDataMultipipeline.Data; /// receive distinct qualified identifiers in the DAG metadata: /// e.g., UnitedKingdomShardCatalog.WeeklyDtu. /// -public class CountryShardCatalog : DataCatalogBase +public class CountryShardCatalog : CatalogAbstract { private readonly string _basePath; @@ -28,10 +28,10 @@ public CountryShardCatalog(string country, string basePath) /// /// Per-country weekly DTU Parquet shard, currency-converted to GBP. /// - public ICatalogEntry> WeeklyDtu => - GetOrCreateEntry( + public IItem> WeeklyDtu => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: $"WeeklyDtu_{Slugify(Country)}", filePath: $"{_basePath}/_03_Primary/Datasets/weekly_dtu_{Slugify(Country)}.parquet" ) diff --git a/examples/advanced/RetailDataMultipipeline/Data/_08_Reporting/CoreCatalog.Reporting.cs b/examples/advanced/RetailDataMultipipeline/Data/_08_Reporting/CoreCatalog.Reporting.cs index 98a2b063..c41459b5 100644 --- a/examples/advanced/RetailDataMultipipeline/Data/_08_Reporting/CoreCatalog.Reporting.cs +++ b/examples/advanced/RetailDataMultipipeline/Data/_08_Reporting/CoreCatalog.Reporting.cs @@ -5,40 +5,40 @@ namespace RetailDataMultipipeline.Data; public partial class CoreCatalog { - public ICatalogEntry> CountryTransactionSummary => - GetOrCreateEntry( + public IItem> CountryTransactionSummary => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "CountryTransactionSummary", filePath: $"{_basePath}/_08_Reporting/Datasets/country_transaction_summary.csv" ) ); /// Daily GBP revenue per country — line chart (PNG). - public ICatalogEntry DollarsChart => - GetOrCreateEntry( + public IItem DollarsChart => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "DollarsChart", filePath: $"{_basePath}/_08_Reporting/Charts/dollars_chart.png" ) ); /// Daily transaction count per country — line chart (PNG). - public ICatalogEntry TransactionsChart => - GetOrCreateEntry( + public IItem TransactionsChart => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "TransactionsChart", filePath: $"{_basePath}/_08_Reporting/Charts/transactions_chart.png" ) ); /// Daily unique customers per country — line chart (PNG). - public ICatalogEntry UsersChart => - GetOrCreateEntry( + public IItem UsersChart => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "UsersChart", filePath: $"{_basePath}/_08_Reporting/Charts/users_chart.png" ) diff --git a/examples/advanced/RetailDataMultipipeline/Pipelines/Analysis/AnalysisPipeline.cs b/examples/advanced/RetailDataMultipipeline/Pipelines/Analysis/AnalysisPipeline.cs index 3bd912b1..0c45e7a4 100644 --- a/examples/advanced/RetailDataMultipipeline/Pipelines/Analysis/AnalysisPipeline.cs +++ b/examples/advanced/RetailDataMultipipeline/Pipelines/Analysis/AnalysisPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using RetailDataMultipipeline.Data; using RetailDataMultipipeline.Pipelines.Analysis.Nodes; @@ -6,15 +6,15 @@ namespace RetailDataMultipipeline.Pipelines.Analysis; /// /// Computes weekly DTU metrics for a single country shard. -/// Instantiated once per via UsePipelines in Program.cs. +/// Instantiated once per via RegisterPipelines in Program.cs. /// public static class AnalysisPipeline { - public static Pipeline Create(CoreCatalog core, CountryShardCatalog shard) + public static Flow Create(CoreCatalog core, CountryShardCatalog shard) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "ComputeWeeklyDtu", description: $"Converts currency and aggregates weekly DTU metrics for {shard.Country}.", transform: ComputeWeeklyDtuNode.Create(shard.Country), diff --git a/examples/advanced/RetailDataMultipipeline/Pipelines/Consolidation/ConsolidationPipeline.cs b/examples/advanced/RetailDataMultipipeline/Pipelines/Consolidation/ConsolidationPipeline.cs index 1d8ea983..c69e6a06 100644 --- a/examples/advanced/RetailDataMultipipeline/Pipelines/Consolidation/ConsolidationPipeline.cs +++ b/examples/advanced/RetailDataMultipipeline/Pipelines/Consolidation/ConsolidationPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using RetailDataMultipipeline.Data; namespace RetailDataMultipipeline.Pipelines.Consolidation; @@ -14,16 +14,16 @@ namespace RetailDataMultipipeline.Pipelines.Consolidation; /// public static class ConsolidationPipeline { - public static Pipeline Create(CoreCatalog core, List shards) + public static Flow Create(CoreCatalog core, List shards) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "ConsolidateShards", description: "Concatenates all per-country weekly DTU shards into a single Parquet dataset.", inputs: shards.Select(s => s.WeeklyDtu).ToList(), output: core.AllCountriesWeeklyDtu, - node: batches => batches.SelectMany(b => b) + step: batches => batches.SelectMany(b => b) ); }); } diff --git a/examples/advanced/RetailDataMultipipeline/Pipelines/DataIngestion/DataIngestionPipeline.cs b/examples/advanced/RetailDataMultipipeline/Pipelines/DataIngestion/DataIngestionPipeline.cs index 883c036f..9d1ac78a 100644 --- a/examples/advanced/RetailDataMultipipeline/Pipelines/DataIngestion/DataIngestionPipeline.cs +++ b/examples/advanced/RetailDataMultipipeline/Pipelines/DataIngestion/DataIngestionPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using RetailDataMultipipeline.Data; using RetailDataMultipipeline.Pipelines.DataIngestion.Nodes; @@ -10,11 +10,11 @@ namespace RetailDataMultipipeline.Pipelines.DataIngestion; /// public static class DataIngestionPipeline { - public static Pipeline Create(CoreCatalog catalog) + public static Flow Create(CoreCatalog catalog) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "ConsolidateDailyFiles", description: "Reads all daily CSV files from the raw directory and writes a unified Parquet dataset.", transform: ConsolidateDailyFilesNode.Create(), diff --git a/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/GraphingPipeline.cs b/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/GraphingPipeline.cs index d9151a73..1f61ba76 100644 --- a/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/GraphingPipeline.cs +++ b/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/GraphingPipeline.cs @@ -1,6 +1,6 @@ using Flowthru.Extensions.Python.Execution; -using Flowthru.Extensions.Python.Nodes; -using Flowthru.Pipelines; +using Flowthru.Extensions.Python.Steps; +using Flowthru.Flows; using RetailDataMultipipeline.Data; namespace RetailDataMultipipeline.Pipelines.Graphing; @@ -12,11 +12,11 @@ namespace RetailDataMultipipeline.Pipelines.Graphing; /// public static class GraphingPipeline { - public static Pipeline Create(CoreCatalog catalog, IPythonExecutor executor) + public static Flow Create(CoreCatalog catalog, IPythonExecutor executor) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "PlotDollarsChart", description: "Line chart of weekly GBP revenue per country (Plotly PNG).", module: "Pipelines.Graphing.Nodes.plot_dtu_charts", @@ -26,7 +26,7 @@ public static Pipeline Create(CoreCatalog catalog, IPythonExecutor executor) executor: executor ); - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "PlotTransactionsChart", description: "Line chart of weekly transaction count per country (Plotly PNG).", module: "Pipelines.Graphing.Nodes.plot_dtu_charts", @@ -36,7 +36,7 @@ public static Pipeline Create(CoreCatalog catalog, IPythonExecutor executor) executor: executor ); - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "PlotUsersChart", description: "Line chart of weekly unique customers per country (Plotly PNG).", module: "Pipelines.Graphing.Nodes.plot_dtu_charts", diff --git a/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/Nodes/__init__.py b/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/Nodes/__init__.py index cbc9bd1c..8339fb8e 100644 --- a/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/Nodes/__init__.py +++ b/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/Nodes/__init__.py @@ -1 +1 @@ -"""Graphing pipeline nodes.""" +"""Graphing pipeline steps.""" diff --git a/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/Nodes/plot_dtu_charts.py b/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/Nodes/plot_dtu_charts.py index 306fb497..a024c45e 100644 --- a/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/Nodes/plot_dtu_charts.py +++ b/examples/advanced/RetailDataMultipipeline/Pipelines/Graphing/Nodes/plot_dtu_charts.py @@ -1,6 +1,6 @@ -"""Daily DTU chart generation nodes. +"""Daily DTU chart generation steps. -Each node reads the consolidated all-countries daily DTU dataset and produces a +Each step reads the consolidated all-countries daily DTU dataset and produces a PNG line chart (one trace per country) for a single metric. """ @@ -10,12 +10,12 @@ import pandas as pd import plotly.express as px import plotly.io as pio -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["WeeklyDtuSchema"], outputs="DollarsChart") +@step(inputs=["WeeklyDtuSchema"], outputs="DollarsChart") def plot_dollars_chart(daily_dtu: pd.DataFrame) -> bytes: """Line chart of weekly GBP revenue per country. @@ -45,7 +45,7 @@ def plot_dollars_chart(daily_dtu: pd.DataFrame) -> bytes: return pio.to_image(fig, format="png") -@node(inputs=["WeeklyDtuSchema"], outputs="TransactionsChart") +@step(inputs=["WeeklyDtuSchema"], outputs="TransactionsChart") def plot_transactions_chart(daily_dtu: pd.DataFrame) -> bytes: """Line chart of weekly transaction count per country. @@ -74,7 +74,7 @@ def plot_transactions_chart(daily_dtu: pd.DataFrame) -> bytes: return pio.to_image(fig, format="png") -@node(inputs=["WeeklyDtuSchema"], outputs="UsersChart") +@step(inputs=["WeeklyDtuSchema"], outputs="UsersChart") def plot_users_chart(daily_dtu: pd.DataFrame) -> bytes: """Line chart of weekly unique customer count per country. diff --git a/examples/advanced/RetailDataMultipipeline/Pipelines/Reporting/ReportingPipeline.cs b/examples/advanced/RetailDataMultipipeline/Pipelines/Reporting/ReportingPipeline.cs index bcce5b48..20b83e4c 100644 --- a/examples/advanced/RetailDataMultipipeline/Pipelines/Reporting/ReportingPipeline.cs +++ b/examples/advanced/RetailDataMultipipeline/Pipelines/Reporting/ReportingPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using RetailDataMultipipeline.Data; using RetailDataMultipipeline.Pipelines.Reporting.Nodes; @@ -9,11 +9,11 @@ namespace RetailDataMultipipeline.Pipelines.Reporting; /// public static class ReportingPipeline { - public static Pipeline Create(CoreCatalog catalog) + public static Flow Create(CoreCatalog catalog) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "SummarizeByCountry", description: "Counts debit and credit line items per country across the full transaction history.", transform: SummarizeByCountryNode.Create(), diff --git a/examples/advanced/RetailDataMultipipeline/Program.cs b/examples/advanced/RetailDataMultipipeline/Program.cs index 4e873228..f58c2f1a 100644 --- a/examples/advanced/RetailDataMultipipeline/Program.cs +++ b/examples/advanced/RetailDataMultipipeline/Program.cs @@ -21,27 +21,44 @@ public class Program public static Task Main(string[] args) => FlowthruCli.RunStandaloneAsync( args, - services => ConfigureServices(services, Directory.GetCurrentDirectory()) + services => + ConfigureServices( + services, + Directory.GetCurrentDirectory(), + AppDomain.CurrentDomain.BaseDirectory + ) ); - public static IServiceProvider ConfigureServices(string? basePath = null) + public static IServiceProvider ConfigureServices( + string? basePath = null, + string? outputPath = null + ) { var services = new ServiceCollection(); - ConfigureServices(services, basePath ?? Directory.GetCurrentDirectory()); + ConfigureServices( + services, + basePath ?? Directory.GetCurrentDirectory(), + outputPath ?? AppDomain.CurrentDomain.BaseDirectory + ); return services.BuildServiceProvider(); } - private static void ConfigureServices(IServiceCollection services, string basePath) + private static void ConfigureServices( + IServiceCollection services, + string basePath, + string outputPath + ) { services.AddFlowthru(flowthru => { flowthru.UseConfiguration(opts => opts.ConfigurationPath = basePath); - // Configure Python runtime — makes Pipelines/ importable and exposes the @node decorator + // Configure Python runtime — makes Pipelines/ importable and exposes the @step decorator flowthru.UsePython(python => { python.ModuleSearchPaths.Add(basePath); - python.ModuleSearchPaths.Add(AppDomain.CurrentDomain.BaseDirectory); + python.ModuleSearchPaths.Add(outputPath); + python.VenvPath = outputPath; }); var dataPath = Path.Combine(basePath, "Data"); @@ -57,24 +74,24 @@ private static void ConfigureServices(IServiceCollection services, string basePa // Core catalog: raw inputs → intermediate → reporting var coreCatalog = new CoreCatalog(dataPath); - flowthru.UseCatalog(coreCatalog); + flowthru.RegisterCatalog(coreCatalog); // Shard catalogs: one per country, labelled {Country}ShardCatalog var shardCatalogs = countries.Select(c => new CountryShardCatalog(c, dataPath)).ToList(); - flowthru.UseCatalogs(shardCatalogs); + flowthru.RegisterCatalogs(shardCatalogs); // Static pipelines resolved via DI - flowthru.RegisterPipeline( + flowthru.RegisterFlow( "DataIngestion", (CoreCatalog cat) => DataIngestionPipeline.Create(cat) ); - flowthru.RegisterPipeline("Reporting", (CoreCatalog cat) => ReportingPipeline.Create(cat)); - flowthru.RegisterPipeline("Graphing", GraphingPipeline.Create); + flowthru.RegisterFlow("Reporting", (CoreCatalog cat) => ReportingPipeline.Create(cat)); + flowthru.RegisterFlow("Graphing", GraphingPipeline.Create); // Dynamic per-country analysis pipelines + fan-in consolidation - flowthru.UsePipelines(_ => + flowthru.RegisterFlows(_ => { - var pipelines = new Dictionary(); + var pipelines = new Dictionary(); foreach (var shard in shardCatalogs) { pipelines[$"Analysis_{shard.Country.Replace(' ', '_')}"] = AnalysisPipeline.Create( diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/DataProcessingCatalog.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/DataProcessingCatalog.cs index bf65151c..323b260c 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/DataProcessingCatalog.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/DataProcessingCatalog.cs @@ -6,7 +6,7 @@ namespace SpaceflightsDistributed.DataProcessing.Data; /// Data catalog for the DataProcessing pipeline library. /// Owns all raw, intermediate, and primary data layers for the Spaceflights domain. /// -public partial class DataProcessingCatalog : DataCatalogBase +public partial class DataProcessingCatalog : CatalogAbstract { private readonly string _basePath; diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_01_Raw/Catalog.Raw.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_01_Raw/Catalog.Raw.cs index 141932b6..9defbde2 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_01_Raw/Catalog.Raw.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_01_Raw/Catalog.Raw.cs @@ -5,28 +5,28 @@ namespace SpaceflightsDistributed.DataProcessing.Data; public partial class DataProcessingCatalog { - public ICatalogEntry> Companies => - GetOrCreateEntry( + public IItem> Companies => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Companies", filePath: $"{_basePath}/_01_Raw/Datasets/companies.csv" ) ); - public ICatalogEntry> Reviews => - GetOrCreateEntry( + public IItem> Reviews => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Reviews", filePath: $"{_basePath}/_01_Raw/Datasets/reviews.csv" ) ); - public ICatalogEntry> Shuttles => - GetOrCreateEntry( + public IItem> Shuttles => + CreateItem( () => - CatalogEntries.Enumerable.Excel( + ItemFactory.Enumerable.Excel( label: "Shuttles", filePath: $"{_basePath}/_01_Raw/Datasets/shuttles.xlsx", sheetName: "Sheet1" diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_02_Intermediate/Catalog.Intermediate.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_02_Intermediate/Catalog.Intermediate.cs index a12407d8..8c71198e 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_02_Intermediate/Catalog.Intermediate.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_02_Intermediate/Catalog.Intermediate.cs @@ -5,19 +5,19 @@ namespace SpaceflightsDistributed.DataProcessing.Data; public partial class DataProcessingCatalog { - public ICatalogEntry> PreprocessedCompanies => - GetOrCreateEntry( + public IItem> PreprocessedCompanies => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "PreprocessedCompanies", filePath: $"{_basePath}/_02_Intermediate/Datasets/preprocessed_companies.parquet" ) ); - public ICatalogEntry> PreprocessedShuttles => - GetOrCreateEntry( + public IItem> PreprocessedShuttles => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "PreprocessedShuttles", filePath: $"{_basePath}/_02_Intermediate/Datasets/preprocessed_shuttles.parquet" ) diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_03_Primary/Catalog.Primary.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_03_Primary/Catalog.Primary.cs index f453f4d2..d50fb9a0 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_03_Primary/Catalog.Primary.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Data/_03_Primary/Catalog.Primary.cs @@ -5,10 +5,10 @@ namespace SpaceflightsDistributed.DataProcessing.Data; public partial class DataProcessingCatalog { - public ICatalogEntry> ModelInputTable => - GetOrCreateEntry( + public IItem> ModelInputTable => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "ModelInputTable", filePath: $"{_basePath}/_03_Primary/Datasets/model_input_table.parquet" ) diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Pipelines/DataProcessing/DataProcessingPipeline.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Pipelines/DataProcessing/DataProcessingPipeline.cs index 7b0a9635..4e62df27 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Pipelines/DataProcessing/DataProcessingPipeline.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataProcessing/Pipelines/DataProcessing/DataProcessingPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using SpaceflightsDistributed.DataProcessing.Data; using SpaceflightsDistributed.DataProcessing.Pipelines.DataProcessing.Nodes; @@ -10,11 +10,11 @@ namespace SpaceflightsDistributed.DataProcessing.Pipelines.DataProcessing; /// public static class DataProcessingPipeline { - public static Pipeline Create(DataProcessingCatalog catalog) + public static Flow Create(DataProcessingCatalog catalog) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "PreprocessCompanies", description: "Cleans and preprocesses raw company data.", transform: PreprocessCompaniesNode.Create(), @@ -22,7 +22,7 @@ public static Pipeline Create(DataProcessingCatalog catalog) output: catalog.PreprocessedCompanies ); - pipeline.AddNode( + pipeline.AddStep( label: "PreprocessShuttles", description: "Cleans and preprocesses raw shuttle data.", transform: PreprocessShuttlesNode.Create(), @@ -30,7 +30,7 @@ public static Pipeline Create(DataProcessingCatalog catalog) output: catalog.PreprocessedShuttles ); - pipeline.AddNode( + pipeline.AddStep( label: "CreateModelInputTable", description: "Joins preprocessed shuttle and company data with review scores.", transform: CreateModelInputTableNode.Create(), diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/DataScienceCatalog.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/DataScienceCatalog.cs index 39fdc11a..6c9f2c9e 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/DataScienceCatalog.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/DataScienceCatalog.cs @@ -6,7 +6,7 @@ namespace SpaceflightsDistributed.DataScience.Data; /// Data catalog for the DataScience pipeline library. /// Owns the model input splits, trained model, and model output layers. /// -public partial class DataScienceCatalog : DataCatalogBase +public partial class DataScienceCatalog : CatalogAbstract { private readonly string _basePath; diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_05_ModelInput/Catalog.ModelInput.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_05_ModelInput/Catalog.ModelInput.cs index b6db90f2..853eb78a 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_05_ModelInput/Catalog.ModelInput.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_05_ModelInput/Catalog.ModelInput.cs @@ -6,10 +6,10 @@ namespace SpaceflightsDistributed.DataScience.Data; public partial class DataScienceCatalog { /// Training split stored in memory — transient between pipeline runs. - public ICatalogEntry> TrainSplit => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "XTrain")); + public IItem> TrainSplit => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "XTrain")); /// Test split stored in memory — transient between pipeline runs. - public ICatalogEntry> TestSplit => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "XTest")); + public IItem> TestSplit => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "XTest")); } diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_06_Models/Catalog.Models.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_06_Models/Catalog.Models.cs index 15115df9..e7d9f85e 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_06_Models/Catalog.Models.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_06_Models/Catalog.Models.cs @@ -5,10 +5,10 @@ namespace SpaceflightsDistributed.DataScience.Data; public partial class DataScienceCatalog { - public ICatalogEntry Regressor => - GetOrCreateEntry( + public IItem Regressor => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "Regressor", filePath: $"{_basePath}/_06_Models/Datasets/regressor.json" ) diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_07_ModelOutput/Catalog.ModelOutput.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_07_ModelOutput/Catalog.ModelOutput.cs index d0bc39b7..67d9b01e 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_07_ModelOutput/Catalog.ModelOutput.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Data/_07_ModelOutput/Catalog.ModelOutput.cs @@ -5,19 +5,19 @@ namespace SpaceflightsDistributed.DataScience.Data; public partial class DataScienceCatalog { - public ICatalogEntry ModelMetrics => - GetOrCreateEntry( + public IItem ModelMetrics => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "ModelMetrics", filePath: $"{_basePath}/_07_ModelOutput/Datasets/model_metrics.json" ) ); - public ICatalogEntry> ModelPredictions => - GetOrCreateEntry( + public IItem> ModelPredictions => + CreateItem( () => - CatalogEntries.Enumerable.Json( + ItemFactory.Enumerable.Json( label: "ModelPredictions", filePath: $"{_basePath}/_07_ModelOutput/Datasets/model_predictions.json" ) diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Pipelines/DataScience/DataSciencePipeline.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Pipelines/DataScience/DataSciencePipeline.cs index 91362453..ef49a521 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Pipelines/DataScience/DataSciencePipeline.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.DataScience/Pipelines/DataScience/DataSciencePipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using SpaceflightsDistributed.DataProcessing.Data; using SpaceflightsDistributed.DataScience.Data; using SpaceflightsDistributed.DataScience.Pipelines.DataScience.Nodes; @@ -29,11 +29,11 @@ public record Params /// The data processing catalog supplying the model input table. /// The data science catalog receiving splits, model, and metrics. /// Configuration parameters for the pipeline. - public static Pipeline Create(DataProcessingCatalog dp, DataScienceCatalog ds, Params parameters) + public static Flow Create(DataProcessingCatalog dp, DataScienceCatalog ds, Params parameters) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "SplitData", description: "Splits the model input table into training and test sets.", transform: SplitDataNode.Create(parameters.ModelOptions), @@ -41,7 +41,7 @@ public static Pipeline Create(DataProcessingCatalog dp, DataScienceCatalog ds, P output: (ds.TrainSplit, ds.TestSplit) ); - pipeline.AddNode( + pipeline.AddStep( label: "TrainModel", description: "Trains a regression model to predict shuttle prices.", transform: TrainModelNode.Create(), @@ -49,7 +49,7 @@ public static Pipeline Create(DataProcessingCatalog dp, DataScienceCatalog ds, P output: ds.Regressor ); - pipeline.AddNode( + pipeline.AddStep( label: "EvaluateModel", description: "Evaluates the trained model on the test set and computes metrics.", transform: EvaluateModelNode.Create(), diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Data/ReportingCatalog.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Data/ReportingCatalog.cs index d743a0bb..20a652c2 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Data/ReportingCatalog.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Data/ReportingCatalog.cs @@ -6,7 +6,7 @@ namespace SpaceflightsDistributed.Reporting.Data; /// Data catalog for the Reporting pipeline library. /// Owns reporting outputs — capacity reports and visualization charts. /// -public partial class ReportingCatalog : DataCatalogBase +public partial class ReportingCatalog : CatalogAbstract { private readonly string _basePath; diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Data/_08_Reporting/Catalog.Reporting.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Data/_08_Reporting/Catalog.Reporting.cs index 75097d88..6eba1724 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Data/_08_Reporting/Catalog.Reporting.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Data/_08_Reporting/Catalog.Reporting.cs @@ -9,22 +9,20 @@ namespace SpaceflightsDistributed.Reporting.Data; /// public partial class ReportingCatalog { - public ICatalogEntry> ShuttleCapacityReport => - GetOrCreateEntry( + public IItem> ShuttleCapacityReport => + CreateItem( () => - CatalogEntries.Enumerable.Json( + ItemFactory.Enumerable.Json( label: "ShuttleCapacityReport", filePath: $"{_basePath}/_08_Reporting/Datasets/shuttle_capacity_report.json" ) ); - public ICatalogEntry ShuttlePassengerCapacityChart => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "ShuttlePassengerCapacityChart") + public IItem ShuttlePassengerCapacityChart => + CreateItem( + () => ItemFactory.Single.Memory(label: "ShuttlePassengerCapacityChart") ); - public ICatalogEntry ConfusionMatrixChart => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "ConfusionMatrixChart") - ); + public IItem ConfusionMatrixChart => + CreateItem(() => ItemFactory.Single.Memory(label: "ConfusionMatrixChart")); } diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Pipelines/Reporting/ReportingPipeline.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Pipelines/Reporting/ReportingPipeline.cs index 62362757..a4691246 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Pipelines/Reporting/ReportingPipeline.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed.Reporting/Pipelines/Reporting/ReportingPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using SpaceflightsDistributed.DataProcessing.Data; using SpaceflightsDistributed.DataScience.Data; using SpaceflightsDistributed.Reporting.Data; @@ -26,16 +26,16 @@ public record Params /// The data science catalog supplying model predictions. /// The reporting catalog receiving all report and chart outputs. /// Configuration for the pipeline. - public static Pipeline Create( + public static Flow Create( DataProcessingCatalog dp, DataScienceCatalog ds, ReportingCatalog r, Params parameters ) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "ComparePassengerCapacity", description: "Aggregates average shuttle passenger capacity grouped by shuttle type.", transform: ComparePassengerCapacityNode.Create(), @@ -43,7 +43,7 @@ Params parameters output: r.ShuttleCapacityReport ); - pipeline.AddNode( + pipeline.AddStep( label: "GeneratePassengerCapacityChart", description: "Generates a bar chart of passenger capacity rankings by shuttle type.", transform: GeneratePassengerCapacityChartNode.Create(), @@ -51,7 +51,7 @@ Params parameters output: r.ShuttlePassengerCapacityChart ); - pipeline.AddNode( + pipeline.AddStep( label: "GenerateConfusionMatrixChart", description: "Generates a confusion matrix heatmap from model price predictions.", transform: CreateConfusionMatrixNode.Create(parameters.ConfusionMatrixOptions), diff --git a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed/Program.cs b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed/Program.cs index 792fcced..f732461d 100644 --- a/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed/Program.cs +++ b/examples/advanced/SpaceflightsDistributed/SpaceflightsDistributed/Program.cs @@ -55,28 +55,28 @@ private static void ConfigureServices(IServiceCollection services, string basePa // ─── Catalog Registration ────────────────────────────────────────────── // Each library owns its own catalog. - flowthru.UseCatalog(_ => new DataProcessingCatalog(dataPath)); - flowthru.UseCatalog(_ => new DataScienceCatalog(dataPath)); - flowthru.UseCatalog(_ => new ReportingCatalog(dataPath)); + flowthru.RegisterCatalog(_ => new DataProcessingCatalog(dataPath)); + flowthru.RegisterCatalog(_ => new DataScienceCatalog(dataPath)); + flowthru.RegisterCatalog(_ => new ReportingCatalog(dataPath)); // ─── Pipeline Registration ───────────────────────────────────────────── flowthru - .RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create) + .RegisterFlow(label: "DataProcessing", flow: DataProcessingPipeline.Create) .WithDescription("Preprocesses companies and shuttles data into a model input table"); flowthru - .RegisterPipeline( + .RegisterFlow( label: "DataScience", - pipeline: DataSciencePipeline.Create, + flow: DataSciencePipeline.Create, configurationSection: "Flowthru:Pipelines:DataScience" ) .WithDescription("Trains linear regression model for shuttle price prediction"); flowthru - .RegisterPipeline( + .RegisterFlow( label: "Reporting", - pipeline: ReportingPipeline.Create, + flow: ReportingPipeline.Create, configurationSection: "Flowthru:Pipelines:Reporting" ) .WithDescription( diff --git a/examples/advanced/SpaceflightsPythonEFCore/Data/Catalog.cs b/examples/advanced/SpaceflightsPythonEFCore/Data/Catalog.cs index ddeac484..d4975176 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Data/Catalog.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Data/Catalog.cs @@ -7,7 +7,7 @@ namespace SpaceflightsPythonEFCore.Data; /// Data catalog for the SpaceflightsPythonEFCore pipeline. /// Combines EFCore storage (DataProcessing and ModelPredictions) with Python-consumed file and memory entries. /// -public partial class Catalog : DataCatalogBase +public partial class Catalog : CatalogAbstract { private readonly string _basePath; private readonly IDbContextFactory _contextFactory; diff --git a/examples/advanced/SpaceflightsPythonEFCore/Data/_01_Raw/Catalog.Raw.cs b/examples/advanced/SpaceflightsPythonEFCore/Data/_01_Raw/Catalog.Raw.cs index 98882977..a9c0533c 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Data/_01_Raw/Catalog.Raw.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Data/_01_Raw/Catalog.Raw.cs @@ -5,28 +5,28 @@ namespace SpaceflightsPythonEFCore.Data; public partial class Catalog { - public ICatalogEntry> Companies => - GetOrCreateEntry( + public IItem> Companies => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Companies", filePath: $"{_basePath}/_01_Raw/Datasets/companies.csv" ) ); - public ICatalogEntry> Reviews => - GetOrCreateEntry( + public IItem> Reviews => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Reviews", filePath: $"{_basePath}/_01_Raw/Datasets/reviews.csv" ) ); - public ICatalogEntry> Shuttles => - GetOrCreateEntry( + public IItem> Shuttles => + CreateItem( () => - CatalogEntries.Enumerable.Excel( + ItemFactory.Enumerable.Excel( label: "Shuttles", filePath: $"{_basePath}/_01_Raw/Datasets/shuttles.xlsx", sheetName: "Sheet1" diff --git a/examples/advanced/SpaceflightsPythonEFCore/Data/_02_Intermediate/Catalog.Intermediate.cs b/examples/advanced/SpaceflightsPythonEFCore/Data/_02_Intermediate/Catalog.Intermediate.cs index 81335acc..3192d964 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Data/_02_Intermediate/Catalog.Intermediate.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Data/_02_Intermediate/Catalog.Intermediate.cs @@ -11,10 +11,10 @@ namespace SpaceflightsPythonEFCore.Data; /// public partial class Catalog { - public ICatalogEntry> PreprocessedCompanies => - GetOrCreateEntry( + public IItem> PreprocessedCompanies => + CreateItem( () => - EFCoreCatalogEntries + EFCoreItemFactory .Enumerable.EFCore( label: "PreprocessedCompanies", contextFactory: _contextFactory @@ -22,10 +22,10 @@ public partial class Catalog .WithInspectionLevel(InspectionLevel.Shallow) ); - public ICatalogEntry> PreprocessedShuttles => - GetOrCreateEntry( + public IItem> PreprocessedShuttles => + CreateItem( () => - EFCoreCatalogEntries + EFCoreItemFactory .Enumerable.EFCore( label: "PreprocessedShuttles", contextFactory: _contextFactory diff --git a/examples/advanced/SpaceflightsPythonEFCore/Data/_03_Primary/Catalog.Primary.cs b/examples/advanced/SpaceflightsPythonEFCore/Data/_03_Primary/Catalog.Primary.cs index aa159b62..3dbb5914 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Data/_03_Primary/Catalog.Primary.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Data/_03_Primary/Catalog.Primary.cs @@ -11,10 +11,10 @@ namespace SpaceflightsPythonEFCore.Data; /// public partial class Catalog { - public ICatalogEntry> ModelInputTable => - GetOrCreateEntry( + public IItem> ModelInputTable => + CreateItem( () => - EFCoreCatalogEntries.Enumerable.EFCore( + EFCoreItemFactory.Enumerable.EFCore( label: "ModelInputTable", contextFactory: _contextFactory, queryCustomizer: q => q.OrderBy(r => r.ShuttleId) diff --git a/examples/advanced/SpaceflightsPythonEFCore/Data/_05_ModelInput/Catalog.ModelInput.cs b/examples/advanced/SpaceflightsPythonEFCore/Data/_05_ModelInput/Catalog.ModelInput.cs index 5d92372d..ea5b6e28 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Data/_05_ModelInput/Catalog.ModelInput.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Data/_05_ModelInput/Catalog.ModelInput.cs @@ -9,15 +9,15 @@ namespace SpaceflightsPythonEFCore.Data; /// public partial class Catalog { - public ICatalogEntry> XTest => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "XTest")); + public IItem> XTest => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "XTest")); - public ICatalogEntry> XTrain => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "XTrain")); + public IItem> XTrain => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "XTrain")); - public ICatalogEntry> YTest => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "YTest")); + public IItem> YTest => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "YTest")); - public ICatalogEntry> YTrain => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "YTrain")); + public IItem> YTrain => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "YTrain")); } diff --git a/examples/advanced/SpaceflightsPythonEFCore/Data/_06_Models/Catalog.Models.cs b/examples/advanced/SpaceflightsPythonEFCore/Data/_06_Models/Catalog.Models.cs index b2eade27..6b190be7 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Data/_06_Models/Catalog.Models.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Data/_06_Models/Catalog.Models.cs @@ -9,10 +9,10 @@ namespace SpaceflightsPythonEFCore.Data; /// public partial class Catalog { - public ICatalogEntry Regressor => - GetOrCreateEntry( + public IItem Regressor => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "Regressor", filePath: $"{_basePath}/_06_Models/Datasets/regressor.json" ) diff --git a/examples/advanced/SpaceflightsPythonEFCore/Data/_07_ModelOutput/Catalog.ModelOutput.cs b/examples/advanced/SpaceflightsPythonEFCore/Data/_07_ModelOutput/Catalog.ModelOutput.cs index 8755763c..6e3af1ff 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Data/_07_ModelOutput/Catalog.ModelOutput.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Data/_07_ModelOutput/Catalog.ModelOutput.cs @@ -15,19 +15,19 @@ namespace SpaceflightsPythonEFCore.Data; /// public partial class Catalog { - public ICatalogEntry ModelMetrics => - GetOrCreateEntry( + public IItem ModelMetrics => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "ModelMetrics", filePath: $"{_basePath}/_07_ModelOutput/Datasets/model_metrics.json" ) ); - public ICatalogEntry> ModelPredictions => - GetOrCreateEntry( + public IItem> ModelPredictions => + CreateItem( () => - EFCoreCatalogEntries.Enumerable.EFCore( + EFCoreItemFactory.Enumerable.EFCore( label: "ModelPredictions", contextFactory: _contextFactory, saveFunc: BulkSavePredictions diff --git a/examples/advanced/SpaceflightsPythonEFCore/Data/_08_Reporting/Catalog.Reporting.cs b/examples/advanced/SpaceflightsPythonEFCore/Data/_08_Reporting/Catalog.Reporting.cs index e6a2b722..cb4aee35 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Data/_08_Reporting/Catalog.Reporting.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Data/_08_Reporting/Catalog.Reporting.cs @@ -7,28 +7,28 @@ namespace SpaceflightsPythonEFCore.Data; /// public partial class Catalog { - public ICatalogEntry CapacityPlotExpress => - GetOrCreateEntry( + public IItem CapacityPlotExpress => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "CapacityPlotExpress", filePath: $"{_basePath}/_08_Reporting/Datasets/shuttle_passenger_capacity_plot_exp.json" ) ); - public ICatalogEntry CapacityPlotGraphObj => - GetOrCreateEntry( + public IItem CapacityPlotGraphObj => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "CapacityPlotGraphObj", filePath: $"{_basePath}/_08_Reporting/Datasets/shuttle_passenger_capacity_plot_go.json" ) ); - public ICatalogEntry ConfusionMatrix => - GetOrCreateEntry( + public IItem ConfusionMatrix => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "ConfusionMatrix", filePath: $"{_basePath}/_08_Reporting/Images/confusion_matrix.png" ) diff --git a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataProcessing/DataProcessingPipeline.cs b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataProcessing/DataProcessingPipeline.cs index 19ea9f0d..7962cbeb 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataProcessing/DataProcessingPipeline.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataProcessing/DataProcessingPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using SpaceflightsPythonEFCore.Data; using SpaceflightsPythonEFCore.Pipelines.DataProcessing.Nodes; @@ -10,11 +10,11 @@ namespace SpaceflightsPythonEFCore.Pipelines.DataProcessing; /// public static class DataProcessingPipeline { - public static Pipeline Create(Catalog catalog) + public static Flow Create(Catalog catalog) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "PreprocessCompanies", description: "Parse and validate raw company data (C#). Stores result in EFCore.", transform: PreprocessCompaniesNode.Create(), @@ -22,7 +22,7 @@ public static Pipeline Create(Catalog catalog) output: catalog.PreprocessedCompanies ); - pipeline.AddNode( + pipeline.AddStep( label: "PreprocessShuttles", description: "Parse and validate raw shuttle data (C#). Stores result in EFCore.", transform: PreprocessShuttlesNode.Create(), @@ -30,7 +30,7 @@ public static Pipeline Create(Catalog catalog) output: catalog.PreprocessedShuttles ); - pipeline.AddNode( + pipeline.AddStep( label: "CreateModelInputTable", description: "Join preprocessed shuttles, companies, and reviews into a model input table (C#). Stores result in EFCore for Python consumption.", transform: CreateModelInputTableNode.Create(), diff --git a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/DataSciencePipeline.cs b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/DataSciencePipeline.cs index cca1370f..9258f6bc 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/DataSciencePipeline.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/DataSciencePipeline.cs @@ -1,6 +1,6 @@ using Flowthru.Extensions.Python.Execution; -using Flowthru.Extensions.Python.Nodes; -using Flowthru.Pipelines; +using Flowthru.Extensions.Python.Steps; +using Flowthru.Flows; using SpaceflightsPythonEFCore.Data; using SpaceflightsPythonEFCore.Data._03_Primary.Schemas; using SpaceflightsPythonEFCore.Data._05_ModelInput.Schemas; @@ -17,11 +17,11 @@ namespace SpaceflightsPythonEFCore.Pipelines.DataScience; /// public static class DataSciencePipeline { - public static Pipeline Create(Catalog catalog, IPythonExecutor executor) + public static Flow Create(Catalog catalog, IPythonExecutor executor) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddPythonNode< + pipeline.AddPythonStep< IEnumerable, IEnumerable, IEnumerable, @@ -37,7 +37,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) executor: executor ); - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "TrainModel", description: "Train linear regression model on train split (Python).", module: "Pipelines.DataScience.Nodes.train_model", @@ -47,7 +47,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) executor: executor ); - pipeline.AddPythonNode< + pipeline.AddPythonStep< LinearRegressionModel, IEnumerable, IEnumerable, @@ -62,7 +62,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) executor: executor ); - pipeline.AddPythonNode< + pipeline.AddPythonStep< LinearRegressionModel, IEnumerable, IEnumerable, diff --git a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/evaluate_model.py b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/evaluate_model.py index 5d028495..6b077318 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/evaluate_model.py +++ b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/evaluate_model.py @@ -1,14 +1,14 @@ -"""Model evaluation node.""" +"""Model evaluation step.""" import logging import numpy as np import pandas as pd from sklearn.metrics import max_error, mean_absolute_error, r2_score -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["LinearRegressionModel", "XValues", "YValues"], outputs="ModelMetrics") +@step(inputs=["LinearRegressionModel", "XValues", "YValues"], outputs="ModelMetrics") def evaluate_model(regressor_params: dict, X_test: pd.DataFrame, y_test: pd.DataFrame) -> dict: """Calculates and logs the coefficient of determination. diff --git a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/generate_predictions.py b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/generate_predictions.py index b546f316..65a13a08 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/generate_predictions.py +++ b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/generate_predictions.py @@ -1,13 +1,13 @@ -"""Model prediction generation node.""" +"""Model prediction generation step.""" import logging import numpy as np import pandas as pd -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["LinearRegressionModel", "XValues", "YValues"], outputs="ModelPredictions") +@step(inputs=["LinearRegressionModel", "XValues", "YValues"], outputs="ModelPredictions") def generate_predictions(regressor_params: dict, X_test: pd.DataFrame, y_test: pd.DataFrame) -> pd.DataFrame: """Generate predictions from the trained model for visualization. diff --git a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/split_data.py b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/split_data.py index b19e8beb..14fcd976 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/split_data.py +++ b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/split_data.py @@ -1,8 +1,8 @@ -"""Data splitting node for train/test split.""" +"""Data splitting step for train/test split.""" import logging import pandas as pd from sklearn.model_selection import train_test_split -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) @@ -21,7 +21,7 @@ ] -@node( +@step( inputs=["ModelInputTableSchema"], outputs=["XValues", "XValues", "YValues", "YValues"], ) diff --git a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/train_model.py b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/train_model.py index d54564d3..0053a160 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/train_model.py +++ b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/DataScience/Nodes/train_model.py @@ -1,13 +1,13 @@ -"""Model training node using scikit-learn.""" +"""Model training step using scikit-learn.""" import logging import pandas as pd from sklearn.linear_model import LinearRegression -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["XValues", "YValues"], outputs="LinearRegressionModel") +@step(inputs=["XValues", "YValues"], outputs="LinearRegressionModel") def train_model(X_train: pd.DataFrame, y_train: pd.DataFrame) -> dict: """Trains the linear regression model. diff --git a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/__init__.py b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/__init__.py index 2e932c2f..40fc227d 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/__init__.py +++ b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/__init__.py @@ -1 +1 @@ -"""Reporting pipeline nodes.""" +"""Reporting pipeline steps.""" diff --git a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/compare_passenger_capacity.py b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/compare_passenger_capacity.py index 6006f909..2506a851 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/compare_passenger_capacity.py +++ b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/compare_passenger_capacity.py @@ -1,15 +1,15 @@ -"""Passenger capacity comparison visualization nodes.""" +"""Passenger capacity comparison visualization steps.""" import logging import pandas as pd import plotly.express as px import plotly.graph_objs as go import plotly.io as pio -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["PreprocessedShuttleSchema"], outputs="CapacityPlotExpress") +@step(inputs=["PreprocessedShuttleSchema"], outputs="CapacityPlotExpress") def compare_passenger_capacity_exp(preprocessed_shuttles: pd.DataFrame) -> str: """Create passenger capacity comparison using plotly.express. @@ -48,7 +48,7 @@ def compare_passenger_capacity_exp(preprocessed_shuttles: pd.DataFrame) -> str: return pio.to_json(fig, pretty=True) -@node(inputs=["PreprocessedShuttleSchema"], outputs="CapacityPlotGraphObj") +@step(inputs=["PreprocessedShuttleSchema"], outputs="CapacityPlotGraphObj") def compare_passenger_capacity_go(preprocessed_shuttles: pd.DataFrame) -> str: """Create passenger capacity comparison using plotly.graph_objects. diff --git a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/create_confusion_matrix.py b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/create_confusion_matrix.py index 210fbabb..54d94823 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/create_confusion_matrix.py +++ b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/Nodes/create_confusion_matrix.py @@ -1,4 +1,4 @@ -"""Confusion matrix visualization node.""" +"""Confusion matrix visualization step.""" import io import logging import matplotlib @@ -6,12 +6,12 @@ import numpy as np import pandas as pd import seaborn as sn -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["ModelPredictions"], outputs="ConfusionMatrix") +@step(inputs=["ModelPredictions"], outputs="ConfusionMatrix") def create_confusion_matrix(predictions: pd.DataFrame) -> bytes: """Create a confusion matrix from regression predictions binned into categories. diff --git a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/ReportingPipeline.cs b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/ReportingPipeline.cs index 590b590f..04f716a6 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/ReportingPipeline.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Pipelines/Reporting/ReportingPipeline.cs @@ -1,6 +1,6 @@ using Flowthru.Extensions.Python.Execution; -using Flowthru.Extensions.Python.Nodes; -using Flowthru.Pipelines; +using Flowthru.Extensions.Python.Steps; +using Flowthru.Flows; using SpaceflightsPythonEFCore.Data; using SpaceflightsPythonEFCore.Data._02_Intermediate.Schemas; using SpaceflightsPythonEFCore.Data._07_ModelOutput.Schemas; @@ -14,11 +14,11 @@ namespace SpaceflightsPythonEFCore.Pipelines.Reporting; /// public static class ReportingPipeline { - public static Pipeline Create(Catalog catalog, IPythonExecutor executor) + public static Flow Create(Catalog catalog, IPythonExecutor executor) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "ComparePassengerCapacityExpress", description: "Shuttle capacity bar chart via plotly.express (Python). Reads PreprocessedShuttles from EFCore.", module: "Pipelines.Reporting.Nodes.compare_passenger_capacity", @@ -28,7 +28,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) executor: executor ); - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "ComparePassengerCapacityGraphObj", description: "Shuttle capacity bar chart via plotly.graph_objects (Python). Reads PreprocessedShuttles from EFCore.", module: "Pipelines.Reporting.Nodes.compare_passenger_capacity", @@ -38,7 +38,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) executor: executor ); - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "CreateConfusionMatrix", description: "Confusion matrix heatmap from model predictions (Python). Reads ModelPredictions from EFCore.", module: "Pipelines.Reporting.Nodes.create_confusion_matrix", diff --git a/examples/advanced/SpaceflightsPythonEFCore/Program.cs b/examples/advanced/SpaceflightsPythonEFCore/Program.cs index 08dcab27..cb262541 100644 --- a/examples/advanced/SpaceflightsPythonEFCore/Program.cs +++ b/examples/advanced/SpaceflightsPythonEFCore/Program.cs @@ -43,17 +43,33 @@ public class Program public static Task Main(string[] args) => FlowthruCli.RunStandaloneAsync( args, - services => ConfigureServices(services, Directory.GetCurrentDirectory()) + services => + ConfigureServices( + services, + Directory.GetCurrentDirectory(), + AppDomain.CurrentDomain.BaseDirectory + ) ); - public static IServiceProvider ConfigureServices(string? basePath = null) + public static IServiceProvider ConfigureServices( + string? basePath = null, + string? outputPath = null + ) { var services = new ServiceCollection(); - ConfigureServices(services, basePath ?? Directory.GetCurrentDirectory()); + ConfigureServices( + services, + basePath ?? Directory.GetCurrentDirectory(), + outputPath ?? AppDomain.CurrentDomain.BaseDirectory + ); return services.BuildServiceProvider(); } - private static void ConfigureServices(IServiceCollection services, string basePath) + private static void ConfigureServices( + IServiceCollection services, + string basePath, + string outputPath + ) { services.AddLogging(logging => { @@ -70,7 +86,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa services.AddFlowthru(flowthru => { flowthru.UseConfiguration(opts => opts.ConfigurationPath = basePath); - flowthru.UseCatalog(sp => new Catalog( + flowthru.RegisterCatalog(sp => new Catalog( basePath: Path.Combine(basePath, "Data"), contextFactory: sp.GetRequiredService>() )); @@ -91,7 +107,8 @@ private static void ConfigureServices(IServiceCollection services, string basePa flowthru.UsePython(python => { python.ModuleSearchPaths.Add(basePath); - python.ModuleSearchPaths.Add(AppDomain.CurrentDomain.BaseDirectory); + python.ModuleSearchPaths.Add(outputPath); + python.VenvPath = outputPath; }); // Resolve the Python executor before pipeline registration (Phase 6 workaround). @@ -101,15 +118,15 @@ private static void ConfigureServices(IServiceCollection services, string basePa tempProvider.GetRequiredService(); flowthru - .RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create) + .RegisterFlow(label: "DataProcessing", flow: DataProcessingPipeline.Create) .WithDescription("Preprocesses companies and shuttles (C#), stores in EFCore"); flowthru - .RegisterPipeline(label: "DataScience", pipeline: DataSciencePipeline.Create) + .RegisterFlow(label: "DataScience", flow: DataSciencePipeline.Create) .WithDescription("Trains and evaluates regression model (Python); reads/writes EFCore"); flowthru - .RegisterPipeline(label: "Reporting", pipeline: ReportingPipeline.Create) + .RegisterFlow(label: "Reporting", flow: ReportingPipeline.Create) .WithDescription( "Generates visualizations (Python); reads PreprocessedShuttles and ModelPredictions from EFCore" ); diff --git a/examples/archived/UmapReferenceComparisons/Data/Catalog.cs b/examples/archived/UmapReferenceComparisons/Data/Catalog.cs index e77fa724..0e075386 100644 --- a/examples/archived/UmapReferenceComparisons/Data/Catalog.cs +++ b/examples/archived/UmapReferenceComparisons/Data/Catalog.cs @@ -9,7 +9,7 @@ namespace UmapReferenceComparisons.Data; /// Manages reference data from Python UMAP implementation and comparison results. /// The catalog is split across partial classes organized by data layer. /// -public partial class Catalog : DataCatalogBase +public partial class Catalog : CatalogAbstract { private readonly string _basePath; diff --git a/examples/archived/UmapReferenceComparisons/Data/_01_Raw/Catalog.Inputs.cs b/examples/archived/UmapReferenceComparisons/Data/_01_Raw/Catalog.Inputs.cs index 7be6af86..16982c5c 100644 --- a/examples/archived/UmapReferenceComparisons/Data/_01_Raw/Catalog.Inputs.cs +++ b/examples/archived/UmapReferenceComparisons/Data/_01_Raw/Catalog.Inputs.cs @@ -9,56 +9,56 @@ public partial class Catalog // Inputs - feature/label files moved from Reference into Inputs/ // ============================================================================ - public ICatalogEntry> IrisInput => - GetOrCreateEntry( + public IItem> IrisInput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "IrisInput", filePath: $"{_basePath}/_01_Raw/Datasets/Inputs/iris/input.parquet" ) ); - public ICatalogEntry> DigitsInput => - GetOrCreateEntry( + public IItem> DigitsInput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "DigitsInput", filePath: $"{_basePath}/_01_Raw/Datasets/Inputs/digits/input.parquet" ) ); - public ICatalogEntry> MnistInput => - GetOrCreateEntry( + public IItem> MnistInput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "MnistInput", filePath: $"{_basePath}/_01_Raw/Datasets/Inputs/mnist/input.parquet" ) ); - public ICatalogEntry> FashionMnistInput => - GetOrCreateEntry( + public IItem> FashionMnistInput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "FashionMnistInput", filePath: $"{_basePath}/_01_Raw/Datasets/Inputs/fashion-mnist/input.parquet" ) ); // MNIST-transform train/test inputs - public ICatalogEntry> MnistTransformTrainInput => - GetOrCreateEntry( + public IItem> MnistTransformTrainInput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "MnistTransformTrainInput", filePath: $"{_basePath}/_01_Raw/Datasets/Inputs/mnist-transform/train_input.parquet" ) ); - public ICatalogEntry> MnistTransformTestInput => - GetOrCreateEntry( + public IItem> MnistTransformTestInput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "MnistTransformTestInput", filePath: $"{_basePath}/_01_Raw/Datasets/Inputs/mnist-transform/test_input.parquet" ) diff --git a/examples/archived/UmapReferenceComparisons/Data/_01_Raw/Catalog.Reference.cs b/examples/archived/UmapReferenceComparisons/Data/_01_Raw/Catalog.Reference.cs index a1dfc1dc..d11fd141 100644 --- a/examples/archived/UmapReferenceComparisons/Data/_01_Raw/Catalog.Reference.cs +++ b/examples/archived/UmapReferenceComparisons/Data/_01_Raw/Catalog.Reference.cs @@ -9,56 +9,56 @@ public partial class Catalog // Reference - Python-produced outputs and any reference-only artifacts // ============================================================================ - public ICatalogEntry> IrisPythonOutput => - GetOrCreateEntry( + public IItem> IrisPythonOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "IrisPythonOutput", filePath: $"{_basePath}/_01_Raw/Datasets/Reference/iris/output.parquet" ) ); - public ICatalogEntry> DigitsPythonOutput => - GetOrCreateEntry( + public IItem> DigitsPythonOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "DigitsPythonOutput", filePath: $"{_basePath}/_01_Raw/Datasets/Reference/digits/output.parquet" ) ); - public ICatalogEntry> MnistPythonOutput => - GetOrCreateEntry( + public IItem> MnistPythonOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "MnistPythonOutput", filePath: $"{_basePath}/_01_Raw/Datasets/Reference/mnist/output.parquet" ) ); - public ICatalogEntry> FashionMnistPythonOutput => - GetOrCreateEntry( + public IItem> FashionMnistPythonOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "FashionMnistPythonOutput", filePath: $"{_basePath}/_01_Raw/Datasets/Reference/fashion-mnist/output.parquet" ) ); // MNIST-transform reference outputs (train/test) - public ICatalogEntry> MnistTransformTrainOutput => - GetOrCreateEntry( + public IItem> MnistTransformTrainOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "MnistTransformTrainOutput", filePath: $"{_basePath}/_01_Raw/Datasets/Reference/mnist-transform/train_output.parquet" ) ); - public ICatalogEntry> MnistTransformTestOutput => - GetOrCreateEntry( + public IItem> MnistTransformTestOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "MnistTransformTestOutput", filePath: $"{_basePath}/_01_Raw/Datasets/Reference/mnist-transform/test_output.parquet" ) diff --git a/examples/archived/UmapReferenceComparisons/Data/_02_Intermediate/Catalog.Intermediate.cs b/examples/archived/UmapReferenceComparisons/Data/_02_Intermediate/Catalog.Intermediate.cs index 419eea65..40a9c33b 100644 --- a/examples/archived/UmapReferenceComparisons/Data/_02_Intermediate/Catalog.Intermediate.cs +++ b/examples/archived/UmapReferenceComparisons/Data/_02_Intermediate/Catalog.Intermediate.cs @@ -12,26 +12,24 @@ public partial class Catalog /// /// Iris dataset in universal UMAP input format (in-memory). /// - public ICatalogEntry> IrisUmapInput => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "IrisUmapInput")); + public IItem> IrisUmapInput => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "IrisUmapInput")); /// /// Digits dataset in universal UMAP input format (in-memory). /// - public ICatalogEntry> DigitsUmapInput => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "DigitsUmapInput")); + public IItem> DigitsUmapInput => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "DigitsUmapInput")); /// /// MNIST dataset in universal UMAP input format (in-memory). /// - public ICatalogEntry> MnistUmapInput => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "MnistUmapInput")); + public IItem> MnistUmapInput => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "MnistUmapInput")); /// /// Fashion-MNIST dataset in universal UMAP input format (in-memory). /// - public ICatalogEntry> FashionMnistUmapInput => - GetOrCreateEntry( - () => CatalogEntries.Enumerable.Memory(label: "FashionMnistUmapInput") - ); + public IItem> FashionMnistUmapInput => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "FashionMnistUmapInput")); } diff --git a/examples/archived/UmapReferenceComparisons/Data/_02_ModelOutputs/Catalog.ModelOutputs.cs b/examples/archived/UmapReferenceComparisons/Data/_02_ModelOutputs/Catalog.ModelOutputs.cs index 3db4662d..2e430fde 100644 --- a/examples/archived/UmapReferenceComparisons/Data/_02_ModelOutputs/Catalog.ModelOutputs.cs +++ b/examples/archived/UmapReferenceComparisons/Data/_02_ModelOutputs/Catalog.ModelOutputs.cs @@ -9,55 +9,55 @@ public partial class Catalog // Model Outputs - C# UMAP embeddings generated by pipeline nodes // ============================================================================ - public ICatalogEntry> IrisCSharpOutput => - GetOrCreateEntry( + public IItem> IrisCSharpOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "IrisCSharpOutput", filePath: $"{_basePath}/_02_ModelOutputs/Datasets/iris_csharp_output.parquet" ) ); - public ICatalogEntry> DigitsCSharpOutput => - GetOrCreateEntry( + public IItem> DigitsCSharpOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "DigitsCSharpOutput", filePath: $"{_basePath}/_02_ModelOutputs/Datasets/digits_csharp_output.parquet" ) ); - public ICatalogEntry> MnistCSharpOutput => - GetOrCreateEntry( + public IItem> MnistCSharpOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "MnistCSharpOutput", filePath: $"{_basePath}/_02_ModelOutputs/Datasets/mnist_csharp_output.parquet" ) ); - public ICatalogEntry> FashionMnistCSharpOutput => - GetOrCreateEntry( + public IItem> FashionMnistCSharpOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "FashionMnistCSharpOutput", filePath: $"{_basePath}/_02_ModelOutputs/Datasets/fashion_mnist_csharp_output.parquet" ) ); - public ICatalogEntry> MnistTransformTrainCSharpOutput => - GetOrCreateEntry( + public IItem> MnistTransformTrainCSharpOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "MnistTransformTrainCSharpOutput", filePath: $"{_basePath}/_02_ModelOutputs/Datasets/mnist_transform_train_csharp_output.parquet" ) ); - public ICatalogEntry> MnistTransformTestCSharpOutput => - GetOrCreateEntry( + public IItem> MnistTransformTestCSharpOutput => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "MnistTransformTestCSharpOutput", filePath: $"{_basePath}/_02_ModelOutputs/Datasets/mnist_transform_test_csharp_output.parquet" ) diff --git a/examples/archived/UmapReferenceComparisons/Data/_03_Reports/Catalog.Reports.cs b/examples/archived/UmapReferenceComparisons/Data/_03_Reports/Catalog.Reports.cs index 38452826..973ea48a 100644 --- a/examples/archived/UmapReferenceComparisons/Data/_03_Reports/Catalog.Reports.cs +++ b/examples/archived/UmapReferenceComparisons/Data/_03_Reports/Catalog.Reports.cs @@ -11,113 +11,109 @@ public partial class Catalog // Reports - Comparison results and analysis outputs // ============================================================================ - public ICatalogEntry IrisComparison => - GetOrCreateEntry( + public IItem IrisComparison => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "IrisComparison", filePath: $"{_basePath}/_03_Reports/Datasets/iris_comparison.json" ) ); - public ICatalogEntry IrisRuntimeReport => - GetOrCreateEntry( + public IItem IrisRuntimeReport => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "IrisRuntimeReport", filePath: $"{_basePath}/_03_Reports/Datasets/iris_runtime_report.txt" ) ); - public ICatalogEntry IrisVisualization => - GetOrCreateEntry(() => CatalogEntries.Single.Memory(label: "IrisVisualization")); + public IItem IrisVisualization => + CreateItem(() => ItemFactory.Single.Memory(label: "IrisVisualization")); - public ICatalogEntry IrisVisualizationPng => - GetOrCreateEntry( + public IItem IrisVisualizationPng => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "IrisVisualizationPng", filePath: $"{_basePath}/_03_Reports/Datasets/iris_comparison.png" ) ); - public ICatalogEntry DigitsComparison => - GetOrCreateEntry( + public IItem DigitsComparison => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "DigitsComparison", filePath: $"{_basePath}/_03_Reports/Datasets/digits_comparison.json" ) ); - public ICatalogEntry DigitsRuntimeReport => - GetOrCreateEntry( + public IItem DigitsRuntimeReport => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "DigitsRuntimeReport", filePath: $"{_basePath}/_03_Reports/Datasets/digits_runtime_report.txt" ) ); - public ICatalogEntry DigitsVisualization => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "DigitsVisualization") - ); + public IItem DigitsVisualization => + CreateItem(() => ItemFactory.Single.Memory(label: "DigitsVisualization")); - public ICatalogEntry DigitsVisualizationPng => - GetOrCreateEntry( + public IItem DigitsVisualizationPng => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "DigitsVisualizationPng", filePath: $"{_basePath}/_03_Reports/Datasets/digits_comparison.png" ) ); - public ICatalogEntry MnistComparison => - GetOrCreateEntry( + public IItem MnistComparison => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "MnistComparison", filePath: $"{_basePath}/_03_Reports/Datasets/mnist_comparison.json" ) ); - public ICatalogEntry FashionMnistComparison => - GetOrCreateEntry( + public IItem FashionMnistComparison => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "FashionMnistComparison", filePath: $"{_basePath}/_03_Reports/Datasets/fashion_mnist_comparison.json" ) ); - public ICatalogEntry FashionMnistRuntimeReport => - GetOrCreateEntry( + public IItem FashionMnistRuntimeReport => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "FashionMnistRuntimeReport", filePath: $"{_basePath}/_03_Reports/Datasets/fashion_mnist_runtime_report.txt" ) ); - public ICatalogEntry MnistTransformComparison => - GetOrCreateEntry( + public IItem MnistTransformComparison => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "MnistTransformComparison", filePath: $"{_basePath}/_03_Reports/Datasets/mnist_transform_comparison.json" ) ); - public ICatalogEntry FashionMnistVisualization => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "FashionMnistVisualization") - ); + public IItem FashionMnistVisualization => + CreateItem(() => ItemFactory.Single.Memory(label: "FashionMnistVisualization")); - public ICatalogEntry FashionMnistVisualizationPng => - GetOrCreateEntry( + public IItem FashionMnistVisualizationPng => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "FashionMnistVisualizationPng", filePath: $"{_basePath}/_03_Reports/Datasets/fashion_mnist_comparison.png" ) diff --git a/examples/archived/UmapReferenceComparisons/Helpers/Nodes/PlotlyImageExportNode.cs b/examples/archived/UmapReferenceComparisons/Helpers/Nodes/PlotlyImageExportNode.cs index 8934f469..464a9f9f 100644 --- a/examples/archived/UmapReferenceComparisons/Helpers/Nodes/PlotlyImageExportNode.cs +++ b/examples/archived/UmapReferenceComparisons/Helpers/Nodes/PlotlyImageExportNode.cs @@ -11,13 +11,13 @@ namespace UmapReferenceComparisons.Helpers.Nodes; /// /// This node handles the PNG export concern separately from chart generation, /// enabling a clean separation of visualization logic from output format concerns. -/// The PNG binary data can be stored in a BinaryFileCatalogEntry<byte[]>. +/// The PNG binary data can be stored in a BinaryFileItem<byte[]>. /// /// /// Input: GenericChart object from memory catalog /// /// -/// Output: PNG binary data as byte[] (can be stored in BinaryFileCatalogEntry<byte[]>) +/// Output: PNG binary data as byte[] (can be stored in BinaryFileItem<byte[]>) /// /// /// Reusability: This node can be used for any Plotly chart type diff --git a/examples/archived/UmapReferenceComparisons/Pipelines/DigitsComparison/DigitsComparisonPipeline.cs b/examples/archived/UmapReferenceComparisons/Pipelines/DigitsComparison/DigitsComparisonPipeline.cs index 9850216d..4d1f3ed7 100644 --- a/examples/archived/UmapReferenceComparisons/Pipelines/DigitsComparison/DigitsComparisonPipeline.cs +++ b/examples/archived/UmapReferenceComparisons/Pipelines/DigitsComparison/DigitsComparisonPipeline.cs @@ -1,6 +1,6 @@ +using Flowthru.Flows; using Flowthru.Misc.ML.UMAP; using Flowthru.Misc.ML.UMAP.Core; -using Flowthru.Pipelines; using UmapReferenceComparisons.Data; using UmapReferenceComparisons.Helpers.Nodes; using UmapReferenceComparisons.Pipelines.DigitsComparison.Nodes; @@ -26,7 +26,7 @@ namespace UmapReferenceComparisons.Pipelines.DigitsComparison; /// public static class DigitsComparisonPipeline { - public static Pipeline Create(Catalog catalog) + public static Flow Create(Catalog catalog) { var umapParameters = new UmapParameters { @@ -40,9 +40,9 @@ public static Pipeline Create(Catalog catalog) Verbosity = 2, }; - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "ConvertDigitsToUmapInput", description: """ Converts Digits-specific schema to universal UmapInput format. @@ -53,7 +53,7 @@ Extracts all 64 pixel values (8x8 image) into a float array. output: catalog.DigitsUmapInput ); - pipeline.AddNode( + pipeline.AddStep( label: "TransformDigitsWithCSharpUmap", description: """ Applies C# UMAP to Digits input features using the same parameters @@ -72,7 +72,7 @@ Outputs both the embedding and runtime performance metrics. output: (catalog.DigitsCSharpOutput, catalog.DigitsRuntimeReport) ); - pipeline.AddNode( + pipeline.AddStep( label: "CompareDigitsOutputs", description: """ Compares C# UMAP output against Python reference output using neighborhood preservation validation. @@ -101,7 +101,7 @@ Outputs both the embedding and runtime performance metrics. output: catalog.DigitsComparison ); - pipeline.AddNode( + pipeline.AddStep( label: "VisualizeComparison", description: """ Creates a side-by-side scatter plot comparing Python and C# UMAP embeddings. @@ -124,7 +124,7 @@ Points are colored by digit class (0-9). ); // NOTE: Commented out due to performance issues with Plotly.NET - // pipeline.AddNode( + // pipeline.AddStep( // label: "ExportVisualizationToPng", // description: """ // Exports the side-by-side comparison chart to a PNG file. diff --git a/examples/archived/UmapReferenceComparisons/Pipelines/FashionComparison/FashionComparisonPipeline.cs b/examples/archived/UmapReferenceComparisons/Pipelines/FashionComparison/FashionComparisonPipeline.cs index 647b98d8..dc01cf5c 100644 --- a/examples/archived/UmapReferenceComparisons/Pipelines/FashionComparison/FashionComparisonPipeline.cs +++ b/examples/archived/UmapReferenceComparisons/Pipelines/FashionComparison/FashionComparisonPipeline.cs @@ -1,6 +1,6 @@ +using Flowthru.Flows; using Flowthru.Misc.ML.UMAP; using Flowthru.Misc.ML.UMAP.Core; -using Flowthru.Pipelines; using UmapReferenceComparisons.Data; using UmapReferenceComparisons.Helpers.Nodes; using UmapReferenceComparisons.Pipelines.FashionComparison.Nodes; @@ -25,7 +25,7 @@ namespace UmapReferenceComparisons.Pipelines.FashionComparison; /// public static class FashionComparisonPipeline { - public static Pipeline Create(Catalog catalog) + public static Flow Create(Catalog catalog) { var umapParameters = new UmapParameters { @@ -38,9 +38,9 @@ public static Pipeline Create(Catalog catalog) Verbosity = 2, }; - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "ConvertFashionMnistToUmapInput", description: "Converts Fashion-MNIST schema to universal UmapInput format (float[784] features, class label)", transform: ConvertFashionMnistToUmapInputNode.Create(), @@ -48,7 +48,7 @@ public static Pipeline Create(Catalog catalog) output: catalog.FashionMnistUmapInput ); - pipeline.AddNode( + pipeline.AddStep( label: "TransformFashionMnistWithCSharpUmap", description: "Applies C# UMAP to Fashion-MNIST input features using Python reference parameters. Outputs both the embedding and runtime performance metrics.", transform: TransformWithUmapNode.Create( @@ -64,7 +64,7 @@ public static Pipeline Create(Catalog catalog) // This node requires a brute force KNN, and is too slow to test currently. - // pipeline.AddNode( + // pipeline.AddStep( // label: "CompareFashionMnistOutputs", // description: "Compares C# UMAP output against Python reference output using neighborhood preservation validation.", // transform: CompareUmapImplementationsNode.Create( @@ -86,7 +86,7 @@ public static Pipeline Create(Catalog catalog) // output: catalog.FashionMnistComparison // ); - pipeline.AddNode( + pipeline.AddStep( label: "VisualizeFashionMnistComparison", description: "Creates a side-by-side scatter plot comparing Python and C# UMAP embeddings. Points colored by class.", transform: VisualizeUmapComparisonNode.Create( @@ -105,7 +105,7 @@ public static Pipeline Create(Catalog catalog) ); // NOTE: Commented out due to performance issues with Plotly.NET - // pipeline.AddNode( + // pipeline.AddStep( // label: "ExportFashionMnistVisualizationToPng", // description: "Exports the side-by-side comparison chart to a PNG file.", // transform: PlotlyImageExportNode.Create(), diff --git a/examples/archived/UmapReferenceComparisons/Pipelines/IrisComparison/IrisComparisonPipeline.cs b/examples/archived/UmapReferenceComparisons/Pipelines/IrisComparison/IrisComparisonPipeline.cs index e090adb8..36999be0 100644 --- a/examples/archived/UmapReferenceComparisons/Pipelines/IrisComparison/IrisComparisonPipeline.cs +++ b/examples/archived/UmapReferenceComparisons/Pipelines/IrisComparison/IrisComparisonPipeline.cs @@ -1,6 +1,6 @@ +using Flowthru.Flows; using Flowthru.Misc.ML.UMAP; using Flowthru.Misc.ML.UMAP.Core; -using Flowthru.Pipelines; using UmapReferenceComparisons.Data; using UmapReferenceComparisons.Data._01_Raw.Schemas; using UmapReferenceComparisons.Helpers.Nodes; @@ -23,7 +23,7 @@ namespace UmapReferenceComparisons.Pipelines.IrisComparison; /// public static class IrisComparisonPipeline { - public static Pipeline Create(Catalog catalog) + public static Flow Create(Catalog catalog) { var umapParameters = new UmapParameters { @@ -36,9 +36,9 @@ public static Pipeline Create(Catalog catalog) Verbosity = 2, }; - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "ConvertIrisToUmapInput", description: """ Converts Iris-specific schema to universal UmapInput format. @@ -49,7 +49,7 @@ Extracts the 4 Iris features (sepal/petal length/width) into a float array. output: catalog.IrisUmapInput ); - pipeline.AddNode( + pipeline.AddStep( label: "TransformIrisWithCSharpUmap", description: """ Applies C# UMAP to Iris input features using the same parameters @@ -75,7 +75,7 @@ Outputs both the embedding and runtime performance metrics. output: (catalog.IrisCSharpOutput, catalog.IrisRuntimeReport) ); - pipeline.AddNode( + pipeline.AddStep( label: "CompareIrisOutputs", description: """ Compares C# UMAP output against Python reference output using neighborhood preservation validation. @@ -102,7 +102,7 @@ Outputs both the embedding and runtime performance metrics. output: catalog.IrisComparison ); - pipeline.AddNode( + pipeline.AddStep( label: "VisualizeComparison", description: """ Creates a side-by-side scatter plot comparing Python and C# UMAP embeddings. @@ -132,7 +132,7 @@ Points are colored by iris species (setosa, versicolor, virginica). ); // NOTE: Commented out due to performance issues with Plotly.NET - // pipeline.AddNode( + // pipeline.AddStep( // label: "ExportVisualizationToPng", // description: """ // Exports the side-by-side comparison chart to a PNG file. diff --git a/examples/archived/UmapReferenceComparisons/Program.cs b/examples/archived/UmapReferenceComparisons/Program.cs index 00df9569..e76375de 100644 --- a/examples/archived/UmapReferenceComparisons/Program.cs +++ b/examples/archived/UmapReferenceComparisons/Program.cs @@ -47,20 +47,17 @@ private static void ConfigureServices(IServiceCollection services, string basePa { // Load configuration flowthru.UseConfiguration(opts => opts.ConfigurationPath = basePath); - flowthru.UseCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); + flowthru.RegisterCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); // Register comparison pipelines flowthru - .RegisterPipeline(label: "IrisComparisonPipeline", pipeline: IrisComparisonPipeline.Create) + .RegisterFlow(label: "IrisComparisonPipeline", flow: IrisComparisonPipeline.Create) .WithDescription( "Compare C# UMAP against Python reference for Iris dataset (150 samples, 4 features)" ); flowthru - .RegisterPipeline( - label: "DigitsComparisonPipeline", - pipeline: DigitsComparisonPipeline.Create - ) + .RegisterFlow(label: "DigitsComparisonPipeline", flow: DigitsComparisonPipeline.Create) .WithDescription( "Compare C# UMAP against Python reference for Digits dataset (1,797 samples, 64 features, 8x8 images)" ); diff --git a/examples/item-templates/README.md b/examples/item-templates/README.md index c85ff879..eec77644 100644 --- a/examples/item-templates/README.md +++ b/examples/item-templates/README.md @@ -46,7 +46,7 @@ DataProcessing/ **Before (generated):** ```csharp -pipeline.AddNode( +pipeline.AddStep( label: "DataProcessingDummy", description: "Placeholder node - replace with actual transformation logic.", transform: DataProcessingDummyNode.Create(), @@ -57,7 +57,7 @@ pipeline.AddNode( **After (with real schemas):** ```csharp -pipeline.AddNode( +pipeline.AddStep( label: "PreprocessCompanies", description: "Clean and validate company data.", transform: PreprocessCompaniesNode.Create(), diff --git a/examples/item-templates/pipeline/.template.config/template.json b/examples/item-templates/pipeline/.template.config/template.json index 16b69c3b..3003f7a0 100644 --- a/examples/item-templates/pipeline/.template.config/template.json +++ b/examples/item-templates/pipeline/.template.config/template.json @@ -13,7 +13,7 @@ "language": "C#", "type": "item" }, - "sourceName": "PipelineName", + "sourceName": "FlowName", "symbols": { "ProjectName": { "type": "parameter", @@ -44,10 +44,10 @@ ], "primaryOutputs": [ { - "path": "PipelineName/PipelineNamePipeline.cs" + "path": "FlowName/FlowNamePipeline.cs" }, { - "path": "PipelineName/Nodes/PipelineNameDummyNode.cs" + "path": "FlowName/Nodes/FlowNameDummyNode.cs" } ], "postActions": [ @@ -58,7 +58,7 @@ "text": "1. If ProjectName was not specified, replace 'ProjectName' with your actual project namespace in the generated files." }, { - "text": "2. Register the pipeline in Program.cs: flowthru.RegisterPipeline();" + "text": "2. Register the pipeline in Program.cs: flowthru.RegisterPipeline();" }, { "text": "3. Replace the dummy node with actual transformation logic and update input/output catalog entries." diff --git a/examples/item-templates/pipeline/PipelineName/Nodes/PipelineNameDummyNode.cs b/examples/item-templates/pipeline/PipelineName/Nodes/PipelineNameDummyNode.cs index b293bec5..87c4eebe 100644 --- a/examples/item-templates/pipeline/PipelineName/Nodes/PipelineNameDummyNode.cs +++ b/examples/item-templates/pipeline/PipelineName/Nodes/PipelineNameDummyNode.cs @@ -1,11 +1,11 @@ using Flowthru.Abstractions; -namespace ProjectName.Pipelines.PipelineName.Nodes; +namespace ProjectName.Pipelines.FlowName.Nodes; /// -/// Dummy node for PipelineName pipeline - replace with actual transformation logic. +/// Dummy node for FlowName pipeline - replace with actual transformation logic. /// -public static class PipelineNameDummyNode +public static class FlowNameDummyNode { /// /// Creates a dummy transformation function. Replace this with your actual processing logic. diff --git a/examples/item-templates/pipeline/PipelineName/PipelineNamePipeline.cs b/examples/item-templates/pipeline/PipelineName/PipelineNamePipeline.cs index bab0a5bd..211131e5 100644 --- a/examples/item-templates/pipeline/PipelineName/PipelineNamePipeline.cs +++ b/examples/item-templates/pipeline/PipelineName/PipelineNamePipeline.cs @@ -1,30 +1,30 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using ProjectName.Data; -using ProjectName.Pipelines.PipelineName.Nodes; +using ProjectName.Pipelines.FlowName.Nodes; -namespace ProjectName.Pipelines.PipelineName; +namespace ProjectName.Pipelines.FlowName; /// -/// Pipeline for PipelineName operations. +/// Pipeline for FlowName operations. /// -public static class PipelineNamePipeline +public static class FlowNamePipeline { /// - /// Creates the PipelineName pipeline. + /// Creates the FlowName pipeline. /// /// The data catalog containing input and output entries. /// - /// A configured pipeline for PipelineName processing. + /// A configured pipeline for FlowName processing. /// public static Pipeline Create(Catalog catalog) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // Start with a dummy node - replace with your actu - pipeline.AddNode( - label: "PipelineNameDummy", + pipeline.AddStep( + label: "FlowNameDummy", description: "Placeholder", - transform: PipelineNameDummyNode.Create(), + transform: FlowNameDummyNode.Create(), input: catalog.NoData, output: catalog.NoData ); diff --git a/examples/starter/KedroIris/.template.config/template.json b/examples/starter/KedroIris/.template.config/template.json index 641d6e91..dbfa060b 100644 --- a/examples/starter/KedroIris/.template.config/template.json +++ b/examples/starter/KedroIris/.template.config/template.json @@ -21,7 +21,7 @@ "FlowthruVersion": { "type": "parameter", "dataType": "string", - "defaultValue": "0.1.32", + "defaultValue": "0.1.33", "replaces": "FlowthruVersion", "description": "Flowthru package version (auto-injected at pack time)" } diff --git a/examples/starter/KedroIris/Data/Catalog.cs b/examples/starter/KedroIris/Data/Catalog.cs index ea120fef..f45cb340 100644 --- a/examples/starter/KedroIris/Data/Catalog.cs +++ b/examples/starter/KedroIris/Data/Catalog.cs @@ -5,7 +5,7 @@ namespace KedroIris.Data; /// /// Data catalog for the Iris classification pipeline, providing access to datasets across all data layers. /// -public partial class Catalog : DataCatalogBase +public partial class Catalog : CatalogAbstract { private readonly string _basePath; diff --git a/examples/starter/KedroIris/Data/_01_Raw/Catalog.Raw.cs b/examples/starter/KedroIris/Data/_01_Raw/Catalog.Raw.cs index 3c79d486..8e7b7f1a 100644 --- a/examples/starter/KedroIris/Data/_01_Raw/Catalog.Raw.cs +++ b/examples/starter/KedroIris/Data/_01_Raw/Catalog.Raw.cs @@ -11,10 +11,10 @@ public partial class Catalog /// /// Raw iris dataset with measurements and species labels. /// - public ICatalogEntry> IrisRaw => - GetOrCreateEntry( + public IItem> IrisRaw => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "IrisRaw", filePath: $"{_basePath}/_01_Raw/Datasets/iris.csv" ) diff --git a/examples/starter/KedroIris/Data/_04_Feature/Catalog.Feature.cs b/examples/starter/KedroIris/Data/_04_Feature/Catalog.Feature.cs index 0373ccc0..83767e99 100644 --- a/examples/starter/KedroIris/Data/_04_Feature/Catalog.Feature.cs +++ b/examples/starter/KedroIris/Data/_04_Feature/Catalog.Feature.cs @@ -12,10 +12,10 @@ public partial class Catalog /// Iris data with one-hot encoded species classifications. /// Created by the DataEngineering pipeline after splitting and encoding. /// - public ICatalogEntry> IrisFeatures => - GetOrCreateEntry( + public IItem> IrisFeatures => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "IrisFeatures", filePath: $"{_basePath}/_04_Feature/Datasets/iris_features.csv" ) diff --git a/examples/starter/KedroIris/Data/_05_ModelInput/Catalog.ModelInput.cs b/examples/starter/KedroIris/Data/_05_ModelInput/Catalog.ModelInput.cs index c74788bc..f09c124a 100644 --- a/examples/starter/KedroIris/Data/_05_ModelInput/Catalog.ModelInput.cs +++ b/examples/starter/KedroIris/Data/_05_ModelInput/Catalog.ModelInput.cs @@ -12,10 +12,10 @@ public partial class Catalog /// /// Training feature vectors (X). /// - public ICatalogEntry> TrainX => - GetOrCreateEntry( + public IItem> TrainX => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "TrainX", filePath: $"{_basePath}/_05_ModelInput/Datasets/train_x.csv" ) @@ -24,10 +24,10 @@ public partial class Catalog /// /// Training target labels (Y). /// - public ICatalogEntry> TrainY => - GetOrCreateEntry( + public IItem> TrainY => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "TrainY", filePath: $"{_basePath}/_05_ModelInput/Datasets/train_y.csv" ) @@ -36,10 +36,10 @@ public partial class Catalog /// /// Test feature vectors (X). /// - public ICatalogEntry> TestX => - GetOrCreateEntry( + public IItem> TestX => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "TestX", filePath: $"{_basePath}/_05_ModelInput/Datasets/test_x.csv" ) @@ -48,10 +48,10 @@ public partial class Catalog /// /// Test target labels (Y). /// - public ICatalogEntry> TestY => - GetOrCreateEntry( + public IItem> TestY => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "TestY", filePath: $"{_basePath}/_05_ModelInput/Datasets/test_y.csv" ) diff --git a/examples/starter/KedroIris/Data/_06_Models/Catalog.Models.cs b/examples/starter/KedroIris/Data/_06_Models/Catalog.Models.cs index d75c448b..c3e29c95 100644 --- a/examples/starter/KedroIris/Data/_06_Models/Catalog.Models.cs +++ b/examples/starter/KedroIris/Data/_06_Models/Catalog.Models.cs @@ -12,10 +12,10 @@ public partial class Catalog /// Trained multi-class logistic regression model. /// Contains weight matrix for all three species classifiers. /// - public ICatalogEntry IrisModel => - GetOrCreateEntry( + public IItem IrisModel => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "IrisModel", filePath: $"{_basePath}/_06_Models/Datasets/iris_model.json" ) diff --git a/examples/starter/KedroIris/Data/_07_ModelOutput/Catalog.ModelOutput.cs b/examples/starter/KedroIris/Data/_07_ModelOutput/Catalog.ModelOutput.cs index b35f02d6..b4aa1100 100644 --- a/examples/starter/KedroIris/Data/_07_ModelOutput/Catalog.ModelOutput.cs +++ b/examples/starter/KedroIris/Data/_07_ModelOutput/Catalog.ModelOutput.cs @@ -11,10 +11,10 @@ public partial class Catalog /// /// Predictions from the trained model on the test set. /// - public ICatalogEntry> Predictions => - GetOrCreateEntry( + public IItem> Predictions => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Predictions", filePath: $"{_basePath}/_07_ModelOutput/Datasets/predictions.csv" ) diff --git a/examples/starter/KedroIris/Data/_08_Reporting/Catalog.Reporting.cs b/examples/starter/KedroIris/Data/_08_Reporting/Catalog.Reporting.cs index 9bfbb8f5..52540367 100644 --- a/examples/starter/KedroIris/Data/_08_Reporting/Catalog.Reporting.cs +++ b/examples/starter/KedroIris/Data/_08_Reporting/Catalog.Reporting.cs @@ -11,10 +11,10 @@ public partial class Catalog /// /// Model evaluation metrics including accuracy and confusion statistics. /// - public ICatalogEntry Metrics => - GetOrCreateEntry( + public IItem Metrics => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "Metrics", filePath: $"{_basePath}/_08_Reporting/Datasets/metrics.json" ) diff --git a/examples/starter/KedroIris/Pipelines/DataEngineering/DataEngineeringPipeline.cs b/examples/starter/KedroIris/Pipelines/DataEngineering/DataEngineeringPipeline.cs index a3b56e00..7ff3f2af 100644 --- a/examples/starter/KedroIris/Pipelines/DataEngineering/DataEngineeringPipeline.cs +++ b/examples/starter/KedroIris/Pipelines/DataEngineering/DataEngineeringPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using KedroIris.Data; using KedroIris.Pipelines.DataEngineering.Nodes; @@ -26,11 +26,11 @@ public record Params /// The data catalog containing input and output entries. /// Configuration parameters for the pipeline. /// A configured pipeline that produces training and test splits with one-hot encoding. - public static Pipeline Create(Catalog catalog, Params parameters) + public static Flow Create(Catalog catalog, Params parameters) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "SplitAndEncode", description: """ Splits the Iris dataset into training and test sets. diff --git a/examples/starter/KedroIris/Pipelines/DataScience/DataSciencePipeline.cs b/examples/starter/KedroIris/Pipelines/DataScience/DataSciencePipeline.cs index 1001eea5..3ff5112a 100644 --- a/examples/starter/KedroIris/Pipelines/DataScience/DataSciencePipeline.cs +++ b/examples/starter/KedroIris/Pipelines/DataScience/DataSciencePipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using KedroIris.Data; using KedroIris.Pipelines.DataScience.Nodes; @@ -31,11 +31,11 @@ public record Params /// The data catalog containing input and output entries. /// Configuration parameters for the pipeline. /// A configured pipeline that produces a trained model, predictions, and metrics. - public static Pipeline Create(Catalog catalog, Params parameters) + public static Flow Create(Catalog catalog, Params parameters) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "TrainModel", description: "Trains a multi-class logistic regression model using gradient descent.", transform: TrainModelNode.Create(parameters.NumTrainIter, parameters.LearningRate), @@ -43,7 +43,7 @@ public static Pipeline Create(Catalog catalog, Params parameters) output: catalog.IrisModel ); - pipeline.AddNode( + pipeline.AddStep( label: "Predict", description: "Predicts species classifications for the test set using the trained model.", transform: PredictNode.Create(), @@ -51,7 +51,7 @@ public static Pipeline Create(Catalog catalog, Params parameters) output: catalog.Predictions ); - pipeline.AddNode( + pipeline.AddStep( label: "Evaluate", description: "Evaluates prediction accuracy and saves metrics to the reporting layer.", transform: EvaluateModelNode.Create(), diff --git a/examples/starter/KedroIris/Program.cs b/examples/starter/KedroIris/Program.cs index dea194d3..c1870339 100644 --- a/examples/starter/KedroIris/Program.cs +++ b/examples/starter/KedroIris/Program.cs @@ -44,7 +44,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa services.AddFlowthru(flowthru => { flowthru.UseConfiguration(opts => opts.ConfigurationPath = basePath); - flowthru.UseCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); + flowthru.RegisterCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); // Output pipeline metadata flowthru.ConfigureMetadata(meta => @@ -60,18 +60,18 @@ private static void ConfigureServices(IServiceCollection services, string basePa // Register data engineering pipeline with configuration parameters flowthru - .RegisterPipeline( + .RegisterFlow( label: "DataEngineering", - pipeline: DataEngineeringPipeline.Create, + flow: DataEngineeringPipeline.Create, configurationSection: "Flowthru:Pipelines:DataEngineering" ) .WithDescription("Splits iris data into training and test sets with one-hot encoding"); // Register data science pipeline with configuration parameters flowthru - .RegisterPipeline( + .RegisterFlow( label: "DataScience", - pipeline: DataSciencePipeline.Create, + flow: DataSciencePipeline.Create, configurationSection: "Flowthru:Pipelines:DataScience" ) .WithDescription("Trains multi-class logistic regression model for iris classification"); diff --git a/examples/starter/KedroIrisPython/.template.config/template.json b/examples/starter/KedroIrisPython/.template.config/template.json index 1bcaf9c8..6af0c61e 100644 --- a/examples/starter/KedroIrisPython/.template.config/template.json +++ b/examples/starter/KedroIrisPython/.template.config/template.json @@ -22,7 +22,7 @@ "FlowthruVersion": { "type": "parameter", "dataType": "string", - "defaultValue": "0.1.32", + "defaultValue": "0.1.33", "replaces": "FlowthruVersion", "description": "Flowthru package version (auto-injected at pack time)" } diff --git a/examples/starter/KedroIrisPython/Data/Catalog.cs b/examples/starter/KedroIrisPython/Data/Catalog.cs index 4562bfda..eaa5016a 100644 --- a/examples/starter/KedroIrisPython/Data/Catalog.cs +++ b/examples/starter/KedroIrisPython/Data/Catalog.cs @@ -5,7 +5,7 @@ namespace KedroIrisPython.Data; /// /// Data catalog for the Iris pipeline, providing access to datasets across all data layers. /// -public partial class Catalog : DataCatalogBase +public partial class Catalog : CatalogAbstract { private readonly string _basePath; diff --git a/examples/starter/KedroIrisPython/Data/_01_Raw/Catalog.Raw.cs b/examples/starter/KedroIrisPython/Data/_01_Raw/Catalog.Raw.cs index c8f260fd..061c3406 100644 --- a/examples/starter/KedroIrisPython/Data/_01_Raw/Catalog.Raw.cs +++ b/examples/starter/KedroIrisPython/Data/_01_Raw/Catalog.Raw.cs @@ -11,10 +11,10 @@ public partial class Catalog /// /// Raw iris dataset with measurements and species labels. /// - public ICatalogEntry> IrisRaw => - GetOrCreateEntry( + public IItem> IrisRaw => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "IrisRaw", filePath: $"{_basePath}/_01_Raw/Datasets/iris.csv" ) diff --git a/examples/starter/KedroIrisPython/Data/_05_ModelInput/Catalog.ModelInput.cs b/examples/starter/KedroIrisPython/Data/_05_ModelInput/Catalog.ModelInput.cs index bce9beb7..801ad069 100644 --- a/examples/starter/KedroIrisPython/Data/_05_ModelInput/Catalog.ModelInput.cs +++ b/examples/starter/KedroIrisPython/Data/_05_ModelInput/Catalog.ModelInput.cs @@ -12,10 +12,10 @@ public partial class Catalog /// /// Training feature vectors (X). /// - public ICatalogEntry> TrainX => - GetOrCreateEntry( + public IItem> TrainX => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "TrainX", filePath: $"{_basePath}/_05_ModelInput/Datasets/train_x.parquet" ) @@ -24,10 +24,10 @@ public partial class Catalog /// /// Training target labels (Y) with one-hot encoding. /// - public ICatalogEntry> TrainY => - GetOrCreateEntry( + public IItem> TrainY => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "TrainY", filePath: $"{_basePath}/_05_ModelInput/Datasets/train_y.parquet" ) @@ -36,10 +36,10 @@ public partial class Catalog /// /// Test feature vectors (X). /// - public ICatalogEntry> TestX => - GetOrCreateEntry( + public IItem> TestX => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "TestX", filePath: $"{_basePath}/_05_ModelInput/Datasets/test_x.parquet" ) @@ -48,10 +48,10 @@ public partial class Catalog /// /// Test target labels (Y) with one-hot encoding. /// - public ICatalogEntry> TestY => - GetOrCreateEntry( + public IItem> TestY => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "TestY", filePath: $"{_basePath}/_05_ModelInput/Datasets/test_y.parquet" ) diff --git a/examples/starter/KedroIrisPython/Data/_06_Models/Catalog.Models.cs b/examples/starter/KedroIrisPython/Data/_06_Models/Catalog.Models.cs index 9f04986c..43e953c5 100644 --- a/examples/starter/KedroIrisPython/Data/_06_Models/Catalog.Models.cs +++ b/examples/starter/KedroIrisPython/Data/_06_Models/Catalog.Models.cs @@ -11,10 +11,10 @@ public partial class Catalog /// Trained logistic regression model weights. /// Stored as a pickled binary file (Python pickle format). /// - public ICatalogEntry ModelWeights => - GetOrCreateEntry( + public IItem ModelWeights => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "ModelWeights", filePath: $"{_basePath}/_06_Models/Datasets/model.pkl" ) diff --git a/examples/starter/KedroIrisPython/Data/_07_ModelOutput/Catalog.ModelOutput.cs b/examples/starter/KedroIrisPython/Data/_07_ModelOutput/Catalog.ModelOutput.cs index 80aacbb5..f237cab3 100644 --- a/examples/starter/KedroIrisPython/Data/_07_ModelOutput/Catalog.ModelOutput.cs +++ b/examples/starter/KedroIrisPython/Data/_07_ModelOutput/Catalog.ModelOutput.cs @@ -11,10 +11,10 @@ public partial class Catalog /// /// Predicted class indices for test data. /// - public ICatalogEntry> Predictions => - GetOrCreateEntry( + public IItem> Predictions => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "Predictions", filePath: $"{_basePath}/_07_ModelOutput/Datasets/predictions.parquet" ) diff --git a/examples/starter/KedroIrisPython/Data/_08_Reporting/Catalog.Reporting.cs b/examples/starter/KedroIrisPython/Data/_08_Reporting/Catalog.Reporting.cs index f58b77ab..7f9462c5 100644 --- a/examples/starter/KedroIrisPython/Data/_08_Reporting/Catalog.Reporting.cs +++ b/examples/starter/KedroIrisPython/Data/_08_Reporting/Catalog.Reporting.cs @@ -11,10 +11,10 @@ public partial class Catalog /// /// Model accuracy report with detailed metrics. /// - public ICatalogEntry AccuracyReport => - GetOrCreateEntry( + public IItem AccuracyReport => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "AccuracyReport", filePath: $"{_basePath}/_08_Reporting/Datasets/accuracy_report.json" ) diff --git a/examples/starter/KedroIrisPython/Pipelines/DataEngineering/DataEngineeringPipeline.cs b/examples/starter/KedroIrisPython/Pipelines/DataEngineering/DataEngineeringPipeline.cs index 7430a1cf..870c8ee9 100644 --- a/examples/starter/KedroIrisPython/Pipelines/DataEngineering/DataEngineeringPipeline.cs +++ b/examples/starter/KedroIrisPython/Pipelines/DataEngineering/DataEngineeringPipeline.cs @@ -1,6 +1,6 @@ using Flowthru.Extensions.Python.Execution; -using Flowthru.Extensions.Python.Nodes; -using Flowthru.Pipelines; +using Flowthru.Extensions.Python.Steps; +using Flowthru.Flows; using KedroIrisPython.Data; using KedroIrisPython.Data._01_Raw.Schemas; using KedroIrisPython.Data._05_ModelInput.Schemas; @@ -15,11 +15,11 @@ public static class DataEngineeringPipeline /// /// Creates the data engineering pipeline. /// - public static Pipeline Create(Catalog catalog, IPythonExecutor executor) + public static Flow Create(Catalog catalog, IPythonExecutor executor) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddPythonNode< + pipeline.AddPythonStep< IEnumerable, IEnumerable, IEnumerable, diff --git a/examples/starter/KedroIrisPython/Pipelines/DataEngineering/Nodes/__init__.py b/examples/starter/KedroIrisPython/Pipelines/DataEngineering/Nodes/__init__.py index dbceda6e..d888d2ed 100644 --- a/examples/starter/KedroIrisPython/Pipelines/DataEngineering/Nodes/__init__.py +++ b/examples/starter/KedroIrisPython/Pipelines/DataEngineering/Nodes/__init__.py @@ -1 +1 @@ -"""Data engineering nodes package.""" +"""Data engineering steps package.""" diff --git a/examples/starter/KedroIrisPython/Pipelines/DataEngineering/Nodes/split_data.py b/examples/starter/KedroIrisPython/Pipelines/DataEngineering/Nodes/split_data.py index 718a379d..a412bad8 100644 --- a/examples/starter/KedroIrisPython/Pipelines/DataEngineering/Nodes/split_data.py +++ b/examples/starter/KedroIrisPython/Pipelines/DataEngineering/Nodes/split_data.py @@ -1,9 +1,9 @@ -"""Data engineering node for splitting iris data into train/test sets.""" +"""Data engineering step for splitting iris data into train/test sets.""" import pandas as pd -from flowthru import node +from flowthru import step -@node( +@step( inputs=["IrisRawSchema"], outputs=["FeatureVectorSchema", "TargetLabelSchema", "FeatureVectorSchema", "TargetLabelSchema"] ) diff --git a/examples/starter/KedroIrisPython/Pipelines/DataScience/DataSciencePipeline.cs b/examples/starter/KedroIrisPython/Pipelines/DataScience/DataSciencePipeline.cs index 5acd965d..03ec411d 100644 --- a/examples/starter/KedroIrisPython/Pipelines/DataScience/DataSciencePipeline.cs +++ b/examples/starter/KedroIrisPython/Pipelines/DataScience/DataSciencePipeline.cs @@ -1,6 +1,6 @@ using Flowthru.Extensions.Python.Execution; -using Flowthru.Extensions.Python.Nodes; -using Flowthru.Pipelines; +using Flowthru.Extensions.Python.Steps; +using Flowthru.Flows; using KedroIrisPython.Data; using KedroIrisPython.Data._05_ModelInput.Schemas; using KedroIrisPython.Data._07_ModelOutput.Schemas; @@ -16,12 +16,12 @@ public static class DataSciencePipeline /// /// Creates the data science pipeline. /// - public static Pipeline Create(Catalog catalog, IPythonExecutor executor) + public static Flow Create(Catalog catalog, IPythonExecutor executor) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // Train model using training data - pipeline.AddPythonNode< + pipeline.AddPythonStep< IEnumerable, IEnumerable, byte[] @@ -36,7 +36,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) ); // Generate predictions using trained model - pipeline.AddPythonNode< + pipeline.AddPythonStep< byte[], IEnumerable, IEnumerable @@ -51,7 +51,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) ); // Report accuracy metrics - pipeline.AddPythonNode< + pipeline.AddPythonStep< IEnumerable, IEnumerable, AccuracyReportSchema diff --git a/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/__init__.py b/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/__init__.py index 8c07d257..dc37d2bc 100644 --- a/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/__init__.py +++ b/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/__init__.py @@ -1 +1 @@ -"""Data science nodes package.""" +"""Data science steps package.""" diff --git a/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/predict.py b/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/predict.py index 3fb242a1..71b5f6ae 100644 --- a/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/predict.py +++ b/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/predict.py @@ -3,7 +3,7 @@ import pickle import numpy as np import pandas as pd -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) @@ -13,7 +13,7 @@ def _sigmoid(z): return 1 / (1 + np.exp(-z)) -@node(inputs=["bytes", "FeatureVectorSchema"], outputs=["PredictionSchema"]) +@step(inputs=["bytes", "FeatureVectorSchema"], outputs=["PredictionSchema"]) def predict(model_bytes: bytes, test_x: pd.DataFrame) -> pd.DataFrame: """Node for making predictions given a pre-trained model and a test set. diff --git a/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/report_accuracy.py b/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/report_accuracy.py index 523b1d62..37c3f931 100644 --- a/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/report_accuracy.py +++ b/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/report_accuracy.py @@ -2,12 +2,12 @@ import logging import numpy as np import pandas as pd -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["PredictionSchema", "TargetLabelSchema"], outputs="AccuracyReportSchema") +@step(inputs=["PredictionSchema", "TargetLabelSchema"], outputs="AccuracyReportSchema") def report_accuracy(predictions: pd.DataFrame, test_y: pd.DataFrame) -> dict: """Node for reporting the accuracy of the predictions. diff --git a/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/train_model.py b/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/train_model.py index a4190b99..b50086ab 100644 --- a/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/train_model.py +++ b/examples/starter/KedroIrisPython/Pipelines/DataScience/Nodes/train_model.py @@ -3,7 +3,7 @@ import pickle import numpy as np import pandas as pd -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) @@ -13,7 +13,7 @@ def _sigmoid(z): return 1 / (1 + np.exp(-z)) -@node(inputs=["FeatureVectorSchema", "TargetLabelSchema"], outputs=["bytes"]) +@step(inputs=["FeatureVectorSchema", "TargetLabelSchema"], outputs=["bytes"]) def train_model(train_x: pd.DataFrame, train_y: pd.DataFrame) -> bytes: """Node for training a simple multi-class logistic regression model. diff --git a/examples/starter/KedroIrisPython/Program.cs b/examples/starter/KedroIrisPython/Program.cs index 27906085..4e513624 100644 --- a/examples/starter/KedroIrisPython/Program.cs +++ b/examples/starter/KedroIrisPython/Program.cs @@ -39,24 +39,40 @@ public class Program public static Task Main(string[] args) => FlowthruCli.RunStandaloneAsync( args, - services => ConfigureServices(services, Directory.GetCurrentDirectory()) + services => + ConfigureServices( + services, + Directory.GetCurrentDirectory(), + AppDomain.CurrentDomain.BaseDirectory + ) ); /// /// Configures services for the application. Used by test infrastructure. /// /// Optional base path for data files (defaults to current directory) - public static IServiceProvider ConfigureServices(string? basePath = null) + public static IServiceProvider ConfigureServices( + string? basePath = null, + string? outputPath = null + ) { var services = new ServiceCollection(); - ConfigureServices(services, basePath ?? Directory.GetCurrentDirectory()); + ConfigureServices( + services, + basePath ?? Directory.GetCurrentDirectory(), + outputPath ?? AppDomain.CurrentDomain.BaseDirectory + ); return services.BuildServiceProvider(); } /// /// Shared service configuration logic. /// - private static void ConfigureServices(IServiceCollection services, string basePath) + private static void ConfigureServices( + IServiceCollection services, + string basePath, + string outputPath + ) { // Add logging first (required by PythonRuntime) services.AddLogging(logging => @@ -68,7 +84,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa services.AddFlowthru(flowthru => { flowthru.UseConfiguration(opts => opts.ConfigurationPath = basePath); - flowthru.UseCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); + flowthru.RegisterCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); flowthru.ConfigureMetadata(meta => { @@ -84,14 +100,12 @@ private static void ConfigureServices(IServiceCollection services, string basePa // Configure Python runtime flowthru.UsePython(python => { - // Add project root to sys.path (for importing from Pipelines/) + // Project root: makes Pipelines/ importable as a Python module tree python.ModuleSearchPaths.Add(basePath); - - // Add output directory for flowthru Python package (contains @node decorator) - var outputDir = AppDomain.CurrentDomain.BaseDirectory; - python.ModuleSearchPaths.Add(outputDir); - - // Note: PythonRuntime auto-discovers .venv in AppContext.BaseDirectory via uv sync + // Output directory: contains the flowthru package (@step decorator) + python.ModuleSearchPaths.Add(outputPath); + // Use this example's own output directory for venv isolation + python.VenvPath = outputPath; }); // Phase 6 workaround: Resolve Python dependencies for pipeline registration @@ -103,11 +117,11 @@ private static void ConfigureServices(IServiceCollection services, string basePa // Register pipelines with resolved executor flowthru - .RegisterPipeline(label: "DataEngineering", pipeline: DataEngineeringPipeline.Create) + .RegisterFlow(label: "DataEngineering", flow: DataEngineeringPipeline.Create) .WithDescription("Splits iris data into training and test sets using Python"); flowthru - .RegisterPipeline(label: "DataScience", pipeline: DataSciencePipeline.Create) + .RegisterFlow(label: "DataScience", flow: DataSciencePipeline.Create) .WithDescription( "Trains multi-class logistic regression model and evaluates predictions using Python" ); diff --git a/examples/starter/KedroSpaceflights/.template.config/template.json b/examples/starter/KedroSpaceflights/.template.config/template.json index eff73a10..003db0f2 100644 --- a/examples/starter/KedroSpaceflights/.template.config/template.json +++ b/examples/starter/KedroSpaceflights/.template.config/template.json @@ -21,7 +21,7 @@ "FlowthruVersion": { "type": "parameter", "dataType": "string", - "defaultValue": "0.1.32", + "defaultValue": "0.1.33", "replaces": "FlowthruVersion", "description": "Flowthru package version (auto-injected at pack time)" } diff --git a/examples/starter/KedroSpaceflights/Data/Catalog.cs b/examples/starter/KedroSpaceflights/Data/Catalog.cs index 1356793a..9b00214f 100644 --- a/examples/starter/KedroSpaceflights/Data/Catalog.cs +++ b/examples/starter/KedroSpaceflights/Data/Catalog.cs @@ -5,7 +5,7 @@ namespace KedroSpaceflights.Data; /// /// Data catalog for the Spaceflights pipeline, providing access to datasets across all data layers. /// -public partial class Catalog : DataCatalogBase +public partial class Catalog : CatalogAbstract { private readonly string _basePath; diff --git a/examples/starter/KedroSpaceflights/Data/_01_Raw/Catalog.Raw.cs b/examples/starter/KedroSpaceflights/Data/_01_Raw/Catalog.Raw.cs index 94e016e7..fb468f7c 100644 --- a/examples/starter/KedroSpaceflights/Data/_01_Raw/Catalog.Raw.cs +++ b/examples/starter/KedroSpaceflights/Data/_01_Raw/Catalog.Raw.cs @@ -8,10 +8,10 @@ public partial class Catalog /// /// Raw company data imported from external sources. /// - public ICatalogEntry> Companies => - GetOrCreateEntry( + public IItem> Companies => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Companies", filePath: $"{_basePath}/_01_Raw/Datasets/companies.csv" ) @@ -20,10 +20,10 @@ public partial class Catalog /// /// Raw review data imported from external sources. /// - public ICatalogEntry> Reviews => - GetOrCreateEntry( + public IItem> Reviews => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Reviews", filePath: $"{_basePath}/_01_Raw/Datasets/reviews.csv" ) @@ -32,10 +32,10 @@ public partial class Catalog /// /// Raw shuttle data imported from external sources. /// - public ICatalogEntry> Shuttles => - GetOrCreateEntry( + public IItem> Shuttles => + CreateItem( () => - CatalogEntries.Enumerable.Excel( + ItemFactory.Enumerable.Excel( label: "Shuttles", filePath: $"{_basePath}/_01_Raw/Datasets/shuttles.xlsx", sheetName: "Sheet1" diff --git a/examples/starter/KedroSpaceflights/Data/_02_Intermediate/Catalog.Intermediate.cs b/examples/starter/KedroSpaceflights/Data/_02_Intermediate/Catalog.Intermediate.cs index 7064d9fb..93ba0711 100644 --- a/examples/starter/KedroSpaceflights/Data/_02_Intermediate/Catalog.Intermediate.cs +++ b/examples/starter/KedroSpaceflights/Data/_02_Intermediate/Catalog.Intermediate.cs @@ -8,10 +8,10 @@ public partial class Catalog /// /// Preprocessed company data with validated and strongly-typed fields. /// - public ICatalogEntry> PreprocessedCompanies => - GetOrCreateEntry( + public IItem> PreprocessedCompanies => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "PreprocessedCompanies", filePath: $"{_basePath}/_02_Intermediate/Datasets/preprocessed_companies.parquet" ) @@ -20,10 +20,10 @@ public partial class Catalog /// /// Preprocessed shuttle data with validated and strongly-typed fields. /// - public ICatalogEntry> PreprocessedShuttles => - GetOrCreateEntry( + public IItem> PreprocessedShuttles => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "PreprocessedShuttles", filePath: $"{_basePath}/_02_Intermediate/Datasets/preprocessed_shuttles.parquet" ) diff --git a/examples/starter/KedroSpaceflights/Data/_03_Primary/Catalog.Primary.cs b/examples/starter/KedroSpaceflights/Data/_03_Primary/Catalog.Primary.cs index 122a44e1..172d2a44 100644 --- a/examples/starter/KedroSpaceflights/Data/_03_Primary/Catalog.Primary.cs +++ b/examples/starter/KedroSpaceflights/Data/_03_Primary/Catalog.Primary.cs @@ -12,10 +12,10 @@ public partial class Catalog /// /// Unified model input table combining shuttle, company, and review data. /// - public ICatalogEntry> ModelInputTable => - GetOrCreateEntry( + public IItem> ModelInputTable => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "ModelInputTable", filePath: $"{_basePath}/_03_Primary/Datasets/model_input_table.parquet" ) diff --git a/examples/starter/KedroSpaceflights/Data/_04_Feature/Catalog.Feature.cs b/examples/starter/KedroSpaceflights/Data/_04_Feature/Catalog.Feature.cs index bb95039b..40f27b9d 100644 --- a/examples/starter/KedroSpaceflights/Data/_04_Feature/Catalog.Feature.cs +++ b/examples/starter/KedroSpaceflights/Data/_04_Feature/Catalog.Feature.cs @@ -10,10 +10,10 @@ public partial class Catalog { // TODO: Add feature datasets here as they are created // Example: - // public ICatalogEntry> Features => - // GetOrCreateEntry( + // public IItem> Features => + // CreateEntry( // () => - // CatalogEntries.Enumerable.Csv( + // ItemFactory.Enumerable.Csv( // label: "Features", // filePath: $"{_basePath}/_04_Feature/Datasets/features.csv" // ) diff --git a/examples/starter/KedroSpaceflights/Data/_05_ModelInput/Catalog.ModelInput.cs b/examples/starter/KedroSpaceflights/Data/_05_ModelInput/Catalog.ModelInput.cs index 63891ea9..def5433a 100644 --- a/examples/starter/KedroSpaceflights/Data/_05_ModelInput/Catalog.ModelInput.cs +++ b/examples/starter/KedroSpaceflights/Data/_05_ModelInput/Catalog.ModelInput.cs @@ -12,12 +12,12 @@ public partial class Catalog /// /// Training dataset split from the model input table. Transient (memory only). /// - public ICatalogEntry> TrainSplit => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "XTrain")); + public IItem> TrainSplit => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "XTrain")); /// /// Test dataset split from the model input table. Transient (memory only). /// - public ICatalogEntry> TestSplit => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "XTest")); + public IItem> TestSplit => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "XTest")); } diff --git a/examples/starter/KedroSpaceflights/Data/_06_Models/Catalog.Models.cs b/examples/starter/KedroSpaceflights/Data/_06_Models/Catalog.Models.cs index 76109c39..a87a1402 100644 --- a/examples/starter/KedroSpaceflights/Data/_06_Models/Catalog.Models.cs +++ b/examples/starter/KedroSpaceflights/Data/_06_Models/Catalog.Models.cs @@ -12,10 +12,10 @@ public partial class Catalog /// /// Trained linear regression model with coefficients and feature names. /// - public ICatalogEntry Regressor => - GetOrCreateEntry( + public IItem Regressor => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "Regressor", filePath: $"{_basePath}/_06_Models/Datasets/regressor.json" ) diff --git a/examples/starter/KedroSpaceflights/Data/_07_ModelOutput/Catalog.ModelOutput.cs b/examples/starter/KedroSpaceflights/Data/_07_ModelOutput/Catalog.ModelOutput.cs index 64116ff8..f9108334 100644 --- a/examples/starter/KedroSpaceflights/Data/_07_ModelOutput/Catalog.ModelOutput.cs +++ b/examples/starter/KedroSpaceflights/Data/_07_ModelOutput/Catalog.ModelOutput.cs @@ -12,10 +12,10 @@ public partial class Catalog /// /// Evaluation metrics for the trained regression model. /// - public ICatalogEntry ModelMetrics => - GetOrCreateEntry( + public IItem ModelMetrics => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "ModelMetrics", filePath: $"{_basePath}/_07_ModelOutput/Datasets/model_metrics.json" ) @@ -25,10 +25,10 @@ public partial class Catalog /// Model predictions containing actual and predicted values from the test set. /// Used for generating confusion matrices and prediction accuracy visualizations. /// - public ICatalogEntry> ModelPredictions => - GetOrCreateEntry( + public IItem> ModelPredictions => + CreateItem( () => - CatalogEntries.Enumerable.Json( + ItemFactory.Enumerable.Json( label: "ModelPredictions", filePath: $"{_basePath}/_07_ModelOutput/Datasets/model_predictions.json" ) diff --git a/examples/starter/KedroSpaceflights/Data/_08_Reporting/Catalog.Reporting.cs b/examples/starter/KedroSpaceflights/Data/_08_Reporting/Catalog.Reporting.cs index c8d6bfa3..a28af713 100644 --- a/examples/starter/KedroSpaceflights/Data/_08_Reporting/Catalog.Reporting.cs +++ b/examples/starter/KedroSpaceflights/Data/_08_Reporting/Catalog.Reporting.cs @@ -13,10 +13,10 @@ public partial class Catalog /// /// Passenger capacity analysis report grouped by shuttle type. /// - public ICatalogEntry> ShuttleCapacityReport => - GetOrCreateEntry( + public IItem> ShuttleCapacityReport => + CreateItem( () => - CatalogEntries.Enumerable.Json( + ItemFactory.Enumerable.Json( label: "ShuttleCapacityReport", filePath: $"{_basePath}/_08_Reporting/Datasets/shuttle_capacity_report.json" ) @@ -26,9 +26,9 @@ public partial class Catalog /// Shuttle passenger capacity bar chart (in-memory GenericChart). /// Intermediate chart object stored in memory for downstream export to PNG. /// - public ICatalogEntry ShuttlePassengerCapacityChart => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "ShuttlePassengerCapacityChart") + public IItem ShuttlePassengerCapacityChart => + CreateItem( + () => ItemFactory.Single.Memory(label: "ShuttlePassengerCapacityChart") ); /// @@ -36,10 +36,10 @@ public partial class Catalog /// Static image representation of the passenger capacity visualization. /// Stored as binary PNG file. /// - public ICatalogEntry ShuttlePassengerCapacityPlotPng => - GetOrCreateEntry( + public IItem ShuttlePassengerCapacityPlotPng => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "ShuttlePassengerCapacityPlotPng", filePath: $"{_basePath}/_08_Reporting/Images/shuttle_passenger_capacity_plot.png" ) @@ -49,20 +49,18 @@ public partial class Catalog /// Confusion matrix heatmap (in-memory GenericChart). /// Intermediate chart object stored in memory for downstream export to PNG. /// - public ICatalogEntry ConfusionMatrixChart => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "ConfusionMatrixChart") - ); + public IItem ConfusionMatrixChart => + CreateItem(() => ItemFactory.Single.Memory(label: "ConfusionMatrixChart")); /// /// Confusion matrix heatmap (PNG image). /// Static image representation of the confusion matrix visualization. /// Stored as binary PNG file. /// - public ICatalogEntry ConfusionMatrixPlotPng => - GetOrCreateEntry( + public IItem ConfusionMatrixPlotPng => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "ConfusionMatrixPlotPng", filePath: $"{_basePath}/_08_Reporting/Images/confusion_matrix_plot.png" ) diff --git a/examples/starter/KedroSpaceflights/Pipelines/DataProcessing/DataProcessingPipeline.cs b/examples/starter/KedroSpaceflights/Pipelines/DataProcessing/DataProcessingPipeline.cs index 08ebe9cc..c0672578 100644 --- a/examples/starter/KedroSpaceflights/Pipelines/DataProcessing/DataProcessingPipeline.cs +++ b/examples/starter/KedroSpaceflights/Pipelines/DataProcessing/DataProcessingPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using KedroSpaceflights.Data; using KedroSpaceflights.Pipelines.DataProcessing.Nodes; @@ -15,11 +15,11 @@ public static class DataProcessingPipeline /// /// The data catalog containing input and output entries. /// A configured pipeline that produces a model input table from raw data sources. - public static Pipeline Create(Catalog catalog) + public static Flow Create(Catalog catalog) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "PreprocessCompanies", description: "Cleans and preprocesses raw company data.", transform: PreprocessCompaniesNode.Create(), @@ -27,7 +27,7 @@ public static Pipeline Create(Catalog catalog) output: catalog.PreprocessedCompanies ); - pipeline.AddNode( + pipeline.AddStep( label: "PreprocessShuttles", description: "Cleans and preprocesses raw shuttle data.", transform: PreprocessShuttlesNode.Create(), @@ -35,7 +35,7 @@ public static Pipeline Create(Catalog catalog) output: catalog.PreprocessedShuttles ); - pipeline.AddNode( + pipeline.AddStep( label: "CreateModelInputTable", description: """ Joins preprocessed shuttle and company data with review scores to create a diff --git a/examples/starter/KedroSpaceflights/Pipelines/DataScience/DataSciencePipeline.cs b/examples/starter/KedroSpaceflights/Pipelines/DataScience/DataSciencePipeline.cs index 464efd36..6561fe94 100644 --- a/examples/starter/KedroSpaceflights/Pipelines/DataScience/DataSciencePipeline.cs +++ b/examples/starter/KedroSpaceflights/Pipelines/DataScience/DataSciencePipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using KedroSpaceflights.Data; namespace KedroSpaceflights.Pipelines.DataScience; @@ -25,11 +25,11 @@ public record Params /// The data catalog containing input and output entries. /// Configuration parameters for the pipeline. /// A configured pipeline that produces a trained model and evaluation metrics. - public static Pipeline Create(Catalog catalog, Params parameters) + public static Flow Create(Catalog catalog, Params parameters) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "SplitData", description: "Splits model input data into training and test sets.", transform: Nodes.SplitDataNode.Create(parameters.ModelOptions), @@ -37,7 +37,7 @@ public static Pipeline Create(Catalog catalog, Params parameters) output: (catalog.TrainSplit, catalog.TestSplit) ); - pipeline.AddNode( + pipeline.AddStep( label: "TrainModel", description: "Trains a regression model to predict shuttle prices.", transform: Nodes.TrainModelNode.Create(), @@ -45,7 +45,7 @@ public static Pipeline Create(Catalog catalog, Params parameters) output: catalog.Regressor ); - pipeline.AddNode( + pipeline.AddStep( label: "EvaluateModel", description: "Evaluates the trained model on the test set and computes metrics and predictions.", transform: Nodes.EvaluateModelNode.Create(), diff --git a/examples/starter/KedroSpaceflights/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs b/examples/starter/KedroSpaceflights/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs index 2323e12c..dff5efcd 100644 --- a/examples/starter/KedroSpaceflights/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs +++ b/examples/starter/KedroSpaceflights/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs @@ -11,13 +11,13 @@ namespace KedroSpaceflights.Pipelines.Reporting.Nodes; /// /// This node handles the PNG export concern separately from chart generation, /// enabling a clean separation of visualization logic from output format concerns. -/// The PNG binary data can be stored in a BinaryFileCatalogEntry<byte[]>. +/// The PNG binary data can be stored in a BinaryFileItem<byte[]>. /// /// /// Input: GenericChart object from memory catalog /// /// -/// Output: PNG binary data as byte[] (can be stored in BinaryFileCatalogEntry<byte[]>) +/// Output: PNG binary data as byte[] (can be stored in BinaryFileItem<byte[]>) /// /// /// Reusability: This node can be used for any Plotly chart type diff --git a/examples/starter/KedroSpaceflights/Pipelines/Reporting/ReportingPipeline.cs b/examples/starter/KedroSpaceflights/Pipelines/Reporting/ReportingPipeline.cs index 929067fc..b879278f 100644 --- a/examples/starter/KedroSpaceflights/Pipelines/Reporting/ReportingPipeline.cs +++ b/examples/starter/KedroSpaceflights/Pipelines/Reporting/ReportingPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using KedroSpaceflights.Data; using KedroSpaceflights.Pipelines.Reporting.Nodes; @@ -41,15 +41,15 @@ public record Params /// The data catalog containing input and output entries. /// Configuration parameters for the pipeline (optional). /// A configured pipeline that produces visualizations and reports. - public static Pipeline Create(Catalog catalog, Params? parameters = null) + public static Flow Create(Catalog catalog, Params? parameters = null) { var p = parameters ?? new Params(); - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // ===== Shuttle Passenger Capacity Report (JSON) ===== - pipeline.AddNode( + pipeline.AddStep( label: "ComparePassengerCapacity", transform: ComparePassengerCapacityNode.Create(), input: catalog.PreprocessedShuttles, @@ -59,7 +59,7 @@ public static Pipeline Create(Catalog catalog, Params? parameters = null) // ===== Shuttle Passenger Capacity Visualization ===== // Step 1: Generate chart from preprocessed shuttle data - pipeline.AddNode( + pipeline.AddStep( label: "GeneratePassengerCapacityChart", transform: GeneratePassengerCapacityChartNode.Create(), input: catalog.PreprocessedShuttles, @@ -68,7 +68,7 @@ public static Pipeline Create(Catalog catalog, Params? parameters = null) // NOTE: Commented out due to performance issues with Plotly.NET // // Step 2: Export chart to PNG for static reports - // pipeline.AddNode( + // pipeline.AddStep( // label: "ExportPassengerCapacityPng", // transform: PlotlyImageExportNode.Create(), // input: catalog.ShuttlePassengerCapacityChart, @@ -78,7 +78,7 @@ public static Pipeline Create(Catalog catalog, Params? parameters = null) // ===== Confusion Matrix Visualization ===== // Step 1: Generate confusion matrix heatmap from model predictions - pipeline.AddNode( + pipeline.AddStep( label: "GenerateConfusionMatrixChart", transform: CreateConfusionMatrixNode.Create(p.ConfusionMatrixOptions), input: catalog.ModelPredictions, @@ -87,7 +87,7 @@ public static Pipeline Create(Catalog catalog, Params? parameters = null) // NOTE: Commented out due to performance issues with Plotly.NET // // Step 2: Export chart to PNG for static reports - // pipeline.AddNode( + // pipeline.AddStep( // label: "ExportConfusionMatrixPng", // transform: PlotlyImageExportNode.Create(), // input: catalog.ConfusionMatrixChart, diff --git a/examples/starter/KedroSpaceflights/Program.cs b/examples/starter/KedroSpaceflights/Program.cs index 9b9d7dba..aebd607f 100644 --- a/examples/starter/KedroSpaceflights/Program.cs +++ b/examples/starter/KedroSpaceflights/Program.cs @@ -45,7 +45,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa services.AddFlowthru(flowthru => { flowthru.UseConfiguration(opts => opts.ConfigurationPath = basePath); - flowthru.UseCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); + flowthru.RegisterCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); // Output pipeline metadata flowthru.ConfigureMetadata(meta => @@ -61,23 +61,23 @@ private static void ConfigureServices(IServiceCollection services, string basePa // Register data processing pipeline flowthru - .RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create) + .RegisterFlow(label: "DataProcessing", flow: DataProcessingPipeline.Create) .WithDescription("Preprocesses companies and shuttles data"); // Register data science pipeline with configuration parameters flowthru - .RegisterPipeline( + .RegisterFlow( label: "DataScience", - pipeline: DataSciencePipeline.Create, + flow: DataSciencePipeline.Create, configurationSection: "Flowthru:Pipelines:DataScience" ) .WithDescription("Trains linear regression model for price prediction"); // Register reporting pipeline with configuration parameters flowthru - .RegisterPipeline( + .RegisterFlow( label: "Reporting", - pipeline: ReportingPipeline.Create, + flow: ReportingPipeline.Create, configurationSection: "Flowthru:Pipelines:Reporting" ) .WithDescription("Generates passenger capacity reports and visualizations"); diff --git a/examples/starter/KedroSpaceflightsPython/.template.config/template.json b/examples/starter/KedroSpaceflightsPython/.template.config/template.json index ee904dff..8433f435 100644 --- a/examples/starter/KedroSpaceflightsPython/.template.config/template.json +++ b/examples/starter/KedroSpaceflightsPython/.template.config/template.json @@ -22,7 +22,7 @@ "FlowthruVersion": { "type": "parameter", "dataType": "string", - "defaultValue": "0.1.32", + "defaultValue": "0.1.33", "replaces": "FlowthruVersion", "description": "Flowthru package version (auto-injected at pack time)" } diff --git a/examples/starter/KedroSpaceflightsPython/Data/Catalog.cs b/examples/starter/KedroSpaceflightsPython/Data/Catalog.cs index 4ddc0bd5..239ff39d 100644 --- a/examples/starter/KedroSpaceflightsPython/Data/Catalog.cs +++ b/examples/starter/KedroSpaceflightsPython/Data/Catalog.cs @@ -5,7 +5,7 @@ namespace KedroSpaceflightsPython.Data; /// /// Data catalog for the Spaceflights pipeline, providing access to datasets across all data layers. /// -public partial class Catalog : DataCatalogBase +public partial class Catalog : CatalogAbstract { private readonly string _basePath; diff --git a/examples/starter/KedroSpaceflightsPython/Data/_01_Raw/Catalog.Raw.cs b/examples/starter/KedroSpaceflightsPython/Data/_01_Raw/Catalog.Raw.cs index 602dbf70..4642239b 100644 --- a/examples/starter/KedroSpaceflightsPython/Data/_01_Raw/Catalog.Raw.cs +++ b/examples/starter/KedroSpaceflightsPython/Data/_01_Raw/Catalog.Raw.cs @@ -8,10 +8,10 @@ public partial class Catalog /// /// Raw company data imported from external sources. /// - public ICatalogEntry> Companies => - GetOrCreateEntry( + public IItem> Companies => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Companies", filePath: $"{_basePath}/_01_Raw/Datasets/companies.csv" ) @@ -20,10 +20,10 @@ public partial class Catalog /// /// Raw review data imported from external sources. /// - public ICatalogEntry> Reviews => - GetOrCreateEntry( + public IItem> Reviews => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Reviews", filePath: $"{_basePath}/_01_Raw/Datasets/reviews.csv" ) @@ -32,10 +32,10 @@ public partial class Catalog /// /// Raw shuttle data imported from external sources. /// - public ICatalogEntry> Shuttles => - GetOrCreateEntry( + public IItem> Shuttles => + CreateItem( () => - CatalogEntries.Enumerable.Excel( + ItemFactory.Enumerable.Excel( label: "Shuttles", filePath: $"{_basePath}/_01_Raw/Datasets/shuttles.xlsx", sheetName: "Sheet1" diff --git a/examples/starter/KedroSpaceflightsPython/Data/_02_Intermediate/Catalog.Intermediate.cs b/examples/starter/KedroSpaceflightsPython/Data/_02_Intermediate/Catalog.Intermediate.cs index 5ee3e8c8..9e2d4d8c 100644 --- a/examples/starter/KedroSpaceflightsPython/Data/_02_Intermediate/Catalog.Intermediate.cs +++ b/examples/starter/KedroSpaceflightsPython/Data/_02_Intermediate/Catalog.Intermediate.cs @@ -8,10 +8,10 @@ public partial class Catalog /// /// Preprocessed company data with validated and strongly-typed fields. /// - public ICatalogEntry> PreprocessedCompanies => - GetOrCreateEntry( + public IItem> PreprocessedCompanies => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "PreprocessedCompanies", filePath: $"{_basePath}/_02_Intermediate/Datasets/preprocessed_companies.parquet" ) @@ -20,10 +20,10 @@ public partial class Catalog /// /// Preprocessed shuttle data with validated and strongly-typed fields. /// - public ICatalogEntry> PreprocessedShuttles => - GetOrCreateEntry( + public IItem> PreprocessedShuttles => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "PreprocessedShuttles", filePath: $"{_basePath}/_02_Intermediate/Datasets/preprocessed_shuttles.parquet" ) diff --git a/examples/starter/KedroSpaceflightsPython/Data/_03_Primary/Catalog.Primary.cs b/examples/starter/KedroSpaceflightsPython/Data/_03_Primary/Catalog.Primary.cs index 28fa5400..d2416d70 100644 --- a/examples/starter/KedroSpaceflightsPython/Data/_03_Primary/Catalog.Primary.cs +++ b/examples/starter/KedroSpaceflightsPython/Data/_03_Primary/Catalog.Primary.cs @@ -12,10 +12,10 @@ public partial class Catalog /// /// Unified model input table combining shuttle, company, and review data. /// - public ICatalogEntry> ModelInputTable => - GetOrCreateEntry( + public IItem> ModelInputTable => + CreateItem( () => - CatalogEntries.Enumerable.Parquet( + ItemFactory.Enumerable.Parquet( label: "ModelInputTable", filePath: $"{_basePath}/_03_Primary/Datasets/model_input_table.parquet" ) diff --git a/examples/starter/KedroSpaceflightsPython/Data/_04_Feature/Catalog.Feature.cs b/examples/starter/KedroSpaceflightsPython/Data/_04_Feature/Catalog.Feature.cs index f391ef91..24726250 100644 --- a/examples/starter/KedroSpaceflightsPython/Data/_04_Feature/Catalog.Feature.cs +++ b/examples/starter/KedroSpaceflightsPython/Data/_04_Feature/Catalog.Feature.cs @@ -10,10 +10,10 @@ public partial class Catalog { // TODO: Add feature datasets here as they are created // Example: - // public ICatalogEntry> Features => - // GetOrCreateEntry( + // public IItem> Features => + // CreateEntry( // () => - // CatalogEntries.Enumerable.Csv( + // ItemFactory.Enumerable.Csv( // label: "Features", // filePath: $"{_basePath}/_04_Feature/Datasets/features.csv" // ) diff --git a/examples/starter/KedroSpaceflightsPython/Data/_05_ModelInput/Catalog.ModelInput.cs b/examples/starter/KedroSpaceflightsPython/Data/_05_ModelInput/Catalog.ModelInput.cs index 0811e442..a182c860 100644 --- a/examples/starter/KedroSpaceflightsPython/Data/_05_ModelInput/Catalog.ModelInput.cs +++ b/examples/starter/KedroSpaceflightsPython/Data/_05_ModelInput/Catalog.ModelInput.cs @@ -12,24 +12,24 @@ public partial class Catalog /// /// Training features split from the model input table. Transient (memory only). /// - public ICatalogEntry> XTest => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "XTest")); + public IItem> XTest => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "XTest")); /// /// Test features split from the model input table. Transient (memory only). /// - public ICatalogEntry> XTrain => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "XTrain")); + public IItem> XTrain => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "XTrain")); /// /// Test targets split from the model input table. Transient (memory only). /// - public ICatalogEntry> YTest => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "YTest")); + public IItem> YTest => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "YTest")); /// /// Training targets split from the model input table. Transient (memory only). /// - public ICatalogEntry> YTrain => - GetOrCreateEntry(() => CatalogEntries.Enumerable.Memory(label: "YTrain")); + public IItem> YTrain => + CreateItem(() => ItemFactory.Enumerable.Memory(label: "YTrain")); } diff --git a/examples/starter/KedroSpaceflightsPython/Data/_06_Models/Catalog.Models.cs b/examples/starter/KedroSpaceflightsPython/Data/_06_Models/Catalog.Models.cs index 0589b54e..90f4a178 100644 --- a/examples/starter/KedroSpaceflightsPython/Data/_06_Models/Catalog.Models.cs +++ b/examples/starter/KedroSpaceflightsPython/Data/_06_Models/Catalog.Models.cs @@ -12,10 +12,10 @@ public partial class Catalog /// /// Trained linear regression model with coefficients and feature names. /// - public ICatalogEntry Regressor => - GetOrCreateEntry( + public IItem Regressor => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "Regressor", filePath: $"{_basePath}/_06_Models/Datasets/regressor.json" ) diff --git a/examples/starter/KedroSpaceflightsPython/Data/_07_ModelOutput/Catalog.ModelOutput.cs b/examples/starter/KedroSpaceflightsPython/Data/_07_ModelOutput/Catalog.ModelOutput.cs index a2b7d4a3..3cf48e11 100644 --- a/examples/starter/KedroSpaceflightsPython/Data/_07_ModelOutput/Catalog.ModelOutput.cs +++ b/examples/starter/KedroSpaceflightsPython/Data/_07_ModelOutput/Catalog.ModelOutput.cs @@ -12,10 +12,10 @@ public partial class Catalog /// /// Evaluation metrics for the trained regression model. /// - public ICatalogEntry ModelMetrics => - GetOrCreateEntry( + public IItem ModelMetrics => + CreateItem( () => - CatalogEntries.Single.Json( + ItemFactory.Single.Json( label: "ModelMetrics", filePath: $"{_basePath}/_07_ModelOutput/Datasets/model_metrics.json" ) @@ -25,10 +25,10 @@ public partial class Catalog /// Model predictions containing actual and predicted values from the test set. /// Used for generating confusion matrices and prediction accuracy visualizations. /// - public ICatalogEntry> ModelPredictions => - GetOrCreateEntry( + public IItem> ModelPredictions => + CreateItem( () => - CatalogEntries.Enumerable.Json( + ItemFactory.Enumerable.Json( label: "ModelPredictions", filePath: $"{_basePath}/_07_ModelOutput/Datasets/model_predictions.json" ) diff --git a/examples/starter/KedroSpaceflightsPython/Data/_08_Reporting/Catalog.Reporting.cs b/examples/starter/KedroSpaceflightsPython/Data/_08_Reporting/Catalog.Reporting.cs index a9c34ae2..b9618010 100644 --- a/examples/starter/KedroSpaceflightsPython/Data/_08_Reporting/Catalog.Reporting.cs +++ b/examples/starter/KedroSpaceflightsPython/Data/_08_Reporting/Catalog.Reporting.cs @@ -12,10 +12,10 @@ public partial class Catalog /// Passenger capacity comparison visualization (plotly.express format). /// Stores JSON representation of plotly figure for shuttle capacity by type. /// - public ICatalogEntry CapacityPlotExpress => - GetOrCreateEntry( + public IItem CapacityPlotExpress => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "CapacityPlotExpress", filePath: $"{_basePath}/_08_Reporting/Datasets/shuttle_passenger_capacity_plot_exp.json" ) @@ -25,10 +25,10 @@ public partial class Catalog /// Passenger capacity comparison visualization (plotly.graph_objects format). /// Stores JSON representation of plotly figure for shuttle capacity by type. /// - public ICatalogEntry CapacityPlotGraphObj => - GetOrCreateEntry( + public IItem CapacityPlotGraphObj => + CreateItem( () => - CatalogEntries.Single.Text( + ItemFactory.Single.Text( label: "CapacityPlotGraphObj", filePath: $"{_basePath}/_08_Reporting/Datasets/shuttle_passenger_capacity_plot_go.json" ) @@ -39,10 +39,10 @@ public partial class Catalog /// Shows actual vs predicted price categories (Low/Medium/High) as a heatmap. /// Stores PNG image as binary data. /// - public ICatalogEntry ConfusionMatrix => - GetOrCreateEntry( + public IItem ConfusionMatrix => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "ConfusionMatrix", filePath: $"{_basePath}/_08_Reporting/Images/confusion_matrix.png" ) diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/DataProcessingPipeline.cs b/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/DataProcessingPipeline.cs index ed8e623d..ea599397 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/DataProcessingPipeline.cs +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/DataProcessingPipeline.cs @@ -1,6 +1,6 @@ using Flowthru.Extensions.Python.Execution; -using Flowthru.Extensions.Python.Nodes; -using Flowthru.Pipelines; +using Flowthru.Extensions.Python.Steps; +using Flowthru.Flows; using KedroSpaceflightsPython.Data; using KedroSpaceflightsPython.Data._01_Raw.Schemas; using KedroSpaceflightsPython.Data._02_Intermediate.Schemas; @@ -16,11 +16,11 @@ public static class DataProcessingPipeline /// /// Creates the data processing pipeline. /// - public static Pipeline Create(Catalog catalog, IPythonExecutor executor) + public static Flow Create(Catalog catalog, IPythonExecutor executor) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "PreprocessCompanies", description: "Clean and parse company data (Python)", module: "Pipelines.DataProcessing.Nodes.preprocess_companies", @@ -30,7 +30,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) executor: executor ); - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "PreprocessShuttles", description: "Clean and parse shuttle data (Python)", module: "Pipelines.DataProcessing.Nodes.preprocess_shuttles", @@ -40,7 +40,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) executor: executor ); - pipeline.AddPythonNode< + pipeline.AddPythonStep< IEnumerable, IEnumerable, IEnumerable, diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/create_model_input_table.py b/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/create_model_input_table.py index f3cbec32..7d9e1fa0 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/create_model_input_table.py +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/create_model_input_table.py @@ -1,12 +1,12 @@ -"""Data joining node to create model input table.""" +"""Data joining step to create model input table.""" import logging import pandas as pd -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node( +@step( inputs=["ShuttlePreprocessedSchema", "CompanyPreprocessedSchema", "ReviewSchema"], outputs=["ModelInputTableSchema"], ) diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/preprocess_companies.py b/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/preprocess_companies.py index 22e07994..ddf84c78 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/preprocess_companies.py +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/preprocess_companies.py @@ -1,8 +1,8 @@ -"""Data preprocessing nodes for companies data.""" +"""Data preprocessing steps for companies data.""" import logging import numpy as np import pandas as pd -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) @@ -21,7 +21,7 @@ def _parse_percentage(x: pd.Series) -> pd.Series: return x -@node(inputs=["CompanyRawSchema"], outputs=["CompanyPreprocessedSchema"]) +@step(inputs=["CompanyRawSchema"], outputs=["CompanyPreprocessedSchema"]) def preprocess_companies(companies: pd.DataFrame) -> pd.DataFrame: """Preprocesses the data for companies. diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/preprocess_shuttles.py b/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/preprocess_shuttles.py index 0dd24610..47c6c1cf 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/preprocess_shuttles.py +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/DataProcessing/Nodes/preprocess_shuttles.py @@ -1,8 +1,8 @@ -"""Data preprocessing nodes for shuttle data.""" +"""Data preprocessing steps for shuttle data.""" import logging import numpy as np import pandas as pd -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ def _parse_money(x: pd.Series) -> pd.Series: return x -@node(inputs=["ShuttleRawSchema"], outputs=["ShuttlePreprocessedSchema"]) +@step(inputs=["ShuttleRawSchema"], outputs=["ShuttlePreprocessedSchema"]) def preprocess_shuttles(shuttles: pd.DataFrame) -> pd.DataFrame: """Preprocesses the data for shuttles. diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/DataSciencePipeline.cs b/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/DataSciencePipeline.cs index bfae8716..b9e70a2e 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/DataSciencePipeline.cs +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/DataSciencePipeline.cs @@ -1,6 +1,6 @@ using Flowthru.Extensions.Python.Execution; -using Flowthru.Extensions.Python.Nodes; -using Flowthru.Pipelines; +using Flowthru.Extensions.Python.Steps; +using Flowthru.Flows; using KedroSpaceflightsPython.Data; using KedroSpaceflightsPython.Data._03_Primary.Schemas; using KedroSpaceflightsPython.Data._05_ModelInput.Schemas; @@ -17,12 +17,12 @@ public static class DataSciencePipeline /// /// Creates the data science pipeline. /// - public static Pipeline Create(Catalog catalog, IPythonExecutor executor) + public static Flow Create(Catalog catalog, IPythonExecutor executor) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // Split data into train/test sets (1×4 output node) - pipeline.AddPythonNode< + pipeline.AddPythonStep< IEnumerable, IEnumerable, IEnumerable, @@ -39,7 +39,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) ); // Train model (2×1 input node, returns sklearn model object) - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "TrainModel", description: "Train linear regression model (Python 2×1 node)", module: "Pipelines.DataScience.Nodes.train_model", @@ -50,7 +50,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) ); // Evaluate model (3×1 input node with sklearn model object) - pipeline.AddPythonNode< + pipeline.AddPythonStep< LinearRegressionModel, IEnumerable, IEnumerable, @@ -66,7 +66,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) ); // Generate predictions for visualization (3×1 input node) - pipeline.AddPythonNode< + pipeline.AddPythonStep< LinearRegressionModel, IEnumerable, IEnumerable, diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/evaluate_model.py b/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/evaluate_model.py index 5d028495..6b077318 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/evaluate_model.py +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/evaluate_model.py @@ -1,14 +1,14 @@ -"""Model evaluation node.""" +"""Model evaluation step.""" import logging import numpy as np import pandas as pd from sklearn.metrics import max_error, mean_absolute_error, r2_score -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["LinearRegressionModel", "XValues", "YValues"], outputs="ModelMetrics") +@step(inputs=["LinearRegressionModel", "XValues", "YValues"], outputs="ModelMetrics") def evaluate_model(regressor_params: dict, X_test: pd.DataFrame, y_test: pd.DataFrame) -> dict: """Calculates and logs the coefficient of determination. diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/generate_predictions.py b/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/generate_predictions.py index b546f316..65a13a08 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/generate_predictions.py +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/generate_predictions.py @@ -1,13 +1,13 @@ -"""Model prediction generation node.""" +"""Model prediction generation step.""" import logging import numpy as np import pandas as pd -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["LinearRegressionModel", "XValues", "YValues"], outputs="ModelPredictions") +@step(inputs=["LinearRegressionModel", "XValues", "YValues"], outputs="ModelPredictions") def generate_predictions(regressor_params: dict, X_test: pd.DataFrame, y_test: pd.DataFrame) -> pd.DataFrame: """Generate predictions from the trained model for visualization. diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/split_data.py b/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/split_data.py index b19e8beb..14fcd976 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/split_data.py +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/split_data.py @@ -1,8 +1,8 @@ -"""Data splitting node for train/test split.""" +"""Data splitting step for train/test split.""" import logging import pandas as pd from sklearn.model_selection import train_test_split -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) @@ -21,7 +21,7 @@ ] -@node( +@step( inputs=["ModelInputTableSchema"], outputs=["XValues", "XValues", "YValues", "YValues"], ) diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/train_model.py b/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/train_model.py index d54564d3..0053a160 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/train_model.py +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/DataScience/Nodes/train_model.py @@ -1,13 +1,13 @@ -"""Model training node using scikit-learn.""" +"""Model training step using scikit-learn.""" import logging import pandas as pd from sklearn.linear_model import LinearRegression -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["XValues", "YValues"], outputs="LinearRegressionModel") +@step(inputs=["XValues", "YValues"], outputs="LinearRegressionModel") def train_model(X_train: pd.DataFrame, y_train: pd.DataFrame) -> dict: """Trains the linear regression model. diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/__init__.py b/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/__init__.py index 2e932c2f..40fc227d 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/__init__.py +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/__init__.py @@ -1 +1 @@ -"""Reporting pipeline nodes.""" +"""Reporting pipeline steps.""" diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/compare_passenger_capacity.py b/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/compare_passenger_capacity.py index 6006f909..2506a851 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/compare_passenger_capacity.py +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/compare_passenger_capacity.py @@ -1,15 +1,15 @@ -"""Passenger capacity comparison visualization nodes.""" +"""Passenger capacity comparison visualization steps.""" import logging import pandas as pd import plotly.express as px import plotly.graph_objs as go import plotly.io as pio -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["PreprocessedShuttleSchema"], outputs="CapacityPlotExpress") +@step(inputs=["PreprocessedShuttleSchema"], outputs="CapacityPlotExpress") def compare_passenger_capacity_exp(preprocessed_shuttles: pd.DataFrame) -> str: """Create passenger capacity comparison using plotly.express. @@ -48,7 +48,7 @@ def compare_passenger_capacity_exp(preprocessed_shuttles: pd.DataFrame) -> str: return pio.to_json(fig, pretty=True) -@node(inputs=["PreprocessedShuttleSchema"], outputs="CapacityPlotGraphObj") +@step(inputs=["PreprocessedShuttleSchema"], outputs="CapacityPlotGraphObj") def compare_passenger_capacity_go(preprocessed_shuttles: pd.DataFrame) -> str: """Create passenger capacity comparison using plotly.graph_objects. diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/create_confusion_matrix.py b/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/create_confusion_matrix.py index 210fbabb..54d94823 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/create_confusion_matrix.py +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/Nodes/create_confusion_matrix.py @@ -1,4 +1,4 @@ -"""Confusion matrix visualization node.""" +"""Confusion matrix visualization step.""" import io import logging import matplotlib @@ -6,12 +6,12 @@ import numpy as np import pandas as pd import seaborn as sn -from flowthru import node +from flowthru import step logger = logging.getLogger(__name__) -@node(inputs=["ModelPredictions"], outputs="ConfusionMatrix") +@step(inputs=["ModelPredictions"], outputs="ConfusionMatrix") def create_confusion_matrix(predictions: pd.DataFrame) -> bytes: """Create a confusion matrix from regression predictions binned into categories. diff --git a/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/ReportingPipeline.cs b/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/ReportingPipeline.cs index e44250ef..7983ad30 100644 --- a/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/ReportingPipeline.cs +++ b/examples/starter/KedroSpaceflightsPython/Pipelines/Reporting/ReportingPipeline.cs @@ -1,6 +1,6 @@ using Flowthru.Extensions.Python.Execution; -using Flowthru.Extensions.Python.Nodes; -using Flowthru.Pipelines; +using Flowthru.Extensions.Python.Steps; +using Flowthru.Flows; using KedroSpaceflightsPython.Data; using KedroSpaceflightsPython.Data._02_Intermediate.Schemas; using KedroSpaceflightsPython.Data._07_ModelOutput.Schemas; @@ -16,12 +16,12 @@ public static class ReportingPipeline /// /// Creates the reporting pipeline. /// - public static Pipeline Create(Catalog catalog, IPythonExecutor executor) + public static Flow Create(Catalog catalog, IPythonExecutor executor) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // Compare passenger capacity using plotly.express - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "ComparePassengerCapacityExpress", description: "Generate passenger capacity bar chart using plotly.express", module: "Pipelines.Reporting.Nodes.compare_passenger_capacity", @@ -32,7 +32,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) ); // Compare passenger capacity using plotly.graph_objects - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "ComparePassengerCapacityGraphObj", description: "Generate passenger capacity bar chart using plotly.graph_objects", module: "Pipelines.Reporting.Nodes.compare_passenger_capacity", @@ -43,7 +43,7 @@ public static Pipeline Create(Catalog catalog, IPythonExecutor executor) ); // Create confusion matrix from model predictions - pipeline.AddPythonNode( + pipeline.AddPythonStep( label: "CreateConfusionMatrix", description: "Generate confusion matrix heatmap from model predictions (binned into categories)", module: "Pipelines.Reporting.Nodes.create_confusion_matrix", diff --git a/examples/starter/KedroSpaceflightsPython/Program.cs b/examples/starter/KedroSpaceflightsPython/Program.cs index e0acbd9a..030e231a 100644 --- a/examples/starter/KedroSpaceflightsPython/Program.cs +++ b/examples/starter/KedroSpaceflightsPython/Program.cs @@ -40,24 +40,40 @@ public class Program public static Task Main(string[] args) => FlowthruCli.RunStandaloneAsync( args, - services => ConfigureServices(services, Directory.GetCurrentDirectory()) + services => + ConfigureServices( + services, + Directory.GetCurrentDirectory(), + AppDomain.CurrentDomain.BaseDirectory + ) ); /// /// Configures services for the application. Used by test infrastructure. /// /// Optional base path for data files (defaults to current directory) - public static IServiceProvider ConfigureServices(string? basePath = null) + public static IServiceProvider ConfigureServices( + string? basePath = null, + string? outputPath = null + ) { var services = new ServiceCollection(); - ConfigureServices(services, basePath ?? Directory.GetCurrentDirectory()); + ConfigureServices( + services, + basePath ?? Directory.GetCurrentDirectory(), + outputPath ?? AppDomain.CurrentDomain.BaseDirectory + ); return services.BuildServiceProvider(); } /// /// Shared service configuration logic. /// - private static void ConfigureServices(IServiceCollection services, string basePath) + private static void ConfigureServices( + IServiceCollection services, + string basePath, + string outputPath + ) { // Add logging first (required by PythonRuntime) services.AddLogging(logging => @@ -69,7 +85,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa services.AddFlowthru(flowthru => { flowthru.UseConfiguration(opts => opts.ConfigurationPath = basePath); - flowthru.UseCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); + flowthru.RegisterCatalog(_ => new Catalog(Path.Combine(basePath, "Data"))); // Output pipeline metadata flowthru.ConfigureMetadata(meta => @@ -86,15 +102,12 @@ private static void ConfigureServices(IServiceCollection services, string basePa // Configure Python runtime flowthru.UsePython(python => { - // Add project root to sys.path (for importing from Pipelines/) + // Project root: makes Pipelines/ importable as a Python module tree python.ModuleSearchPaths.Add(basePath); - - // Add output directory for flowthru Python package (contains @node decorator) - var outputDir = AppDomain.CurrentDomain.BaseDirectory; - python.ModuleSearchPaths.Add(outputDir); - - // Note: PythonRuntime auto-discovers .venv in AppContext.BaseDirectory via uv sync - // No need to manually add site-packages path here + // Output directory: contains the flowthru package (@step decorator) + python.ModuleSearchPaths.Add(outputPath); + // Use this example's own output directory for venv isolation + python.VenvPath = outputPath; }); // Phase 6 workaround: Resolve Python dependencies for pipeline registration @@ -106,17 +119,17 @@ private static void ConfigureServices(IServiceCollection services, string basePa // Register pipelines with resolved executor flowthru - .RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create) + .RegisterFlow(label: "DataProcessing", flow: DataProcessingPipeline.Create) .WithDescription("Preprocesses companies, shuttles, and reviews data using Python"); flowthru - .RegisterPipeline(label: "DataScience", pipeline: DataSciencePipeline.Create) + .RegisterFlow(label: "DataScience", flow: DataSciencePipeline.Create) .WithDescription( "Trains linear regression model for price prediction using Python/scikit-learn" ); flowthru - .RegisterPipeline(label: "Reporting", pipeline: ReportingPipeline.Create) + .RegisterFlow(label: "Reporting", flow: ReportingPipeline.Create) .WithDescription( "Generates visualization outputs including passenger capacity plots and confusion matrix" ); diff --git a/examples/starter/Minimal/.template.config/template.json b/examples/starter/Minimal/.template.config/template.json index e5140153..d5b765f5 100644 --- a/examples/starter/Minimal/.template.config/template.json +++ b/examples/starter/Minimal/.template.config/template.json @@ -21,7 +21,7 @@ "FlowthruVersion": { "type": "parameter", "dataType": "string", - "defaultValue": "0.1.32", + "defaultValue": "0.1.33", "replaces": "FlowthruVersion", "description": "Flowthru package version (auto-injected at pack time)" } diff --git a/examples/starter/Minimal/Data/Catalog.cs b/examples/starter/Minimal/Data/Catalog.cs index 0c13bd5b..5b431f22 100644 --- a/examples/starter/Minimal/Data/Catalog.cs +++ b/examples/starter/Minimal/Data/Catalog.cs @@ -5,7 +5,7 @@ namespace Minimal.Data; /// /// Data catalog providing access to all datasets in the pipeline. /// -public partial class Catalog : DataCatalogBase +public partial class Catalog : CatalogAbstract { private readonly string _basePath; diff --git a/examples/starter/Minimal/Data/_01_Raw/Catalog.Raw.cs b/examples/starter/Minimal/Data/_01_Raw/Catalog.Raw.cs index b6b43312..156d1f5b 100644 --- a/examples/starter/Minimal/Data/_01_Raw/Catalog.Raw.cs +++ b/examples/starter/Minimal/Data/_01_Raw/Catalog.Raw.cs @@ -8,10 +8,10 @@ public partial class Catalog /// /// Raw name data imported from CSV file. /// - public ICatalogEntry> Names => - GetOrCreateEntry( + public IItem> Names => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Names", filePath: $"{_basePath}/Data/_01_Raw/Datasets/names.csv" ) diff --git a/examples/starter/Minimal/Data/_02_Intermediate/Catalog.Intermediate.cs b/examples/starter/Minimal/Data/_02_Intermediate/Catalog.Intermediate.cs index 758d8d5e..51df9750 100644 --- a/examples/starter/Minimal/Data/_02_Intermediate/Catalog.Intermediate.cs +++ b/examples/starter/Minimal/Data/_02_Intermediate/Catalog.Intermediate.cs @@ -8,10 +8,10 @@ public partial class Catalog /// /// Greetings with "Hello" prefix. /// - public ICatalogEntry> HelloGreetings => - GetOrCreateEntry( + public IItem> HelloGreetings => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "HelloGreetings", filePath: $"{_basePath}/Data/_02_Intermediate/Datasets/hello_greetings.csv" ) diff --git a/examples/starter/Minimal/Data/_03_Primary/Catalog.Primary.cs b/examples/starter/Minimal/Data/_03_Primary/Catalog.Primary.cs index 5873c683..022da9ea 100644 --- a/examples/starter/Minimal/Data/_03_Primary/Catalog.Primary.cs +++ b/examples/starter/Minimal/Data/_03_Primary/Catalog.Primary.cs @@ -8,10 +8,10 @@ public partial class Catalog /// /// Farewell greetings with "Goodbye" prefix. /// - public ICatalogEntry> Goodbyes => - GetOrCreateEntry( + public IItem> Goodbyes => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Goodbyes", filePath: $"{_basePath}/Data/_03_Primary/Datasets/goodbyes.csv" ) @@ -20,10 +20,10 @@ public partial class Catalog /// /// Farewell greetings with "So long" prefix. /// - public ICatalogEntry> SoLongs => - GetOrCreateEntry( + public IItem> SoLongs => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "SoLongs", filePath: $"{_basePath}/Data/_03_Primary/Datasets/solongs.csv" ) diff --git a/examples/starter/Minimal/Pipelines/Greetings/GreetingsPipeline.cs b/examples/starter/Minimal/Pipelines/Greetings/GreetingsPipeline.cs index e655694b..189eac9e 100644 --- a/examples/starter/Minimal/Pipelines/Greetings/GreetingsPipeline.cs +++ b/examples/starter/Minimal/Pipelines/Greetings/GreetingsPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using Minimal.Data; using Minimal.Pipelines.Greetings.Nodes; @@ -16,12 +16,12 @@ public static class GreetingsPipeline /// /// A configured pipeline that transforms names into various greeting formats. /// - public static Pipeline Create(Catalog catalog) + public static Flow Create(Catalog catalog) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // Node 1: Transform names to "Hello, {name}!" - pipeline.AddNode( + pipeline.AddStep( label: "CreateHello", description: "Transform names into 'Hello' greetings.", transform: CreateHelloNode.Create(), @@ -30,7 +30,7 @@ public static Pipeline Create(Catalog catalog) ); // Node 2: Transform "Hello" greetings into "Goodbye" and "So long" variants - pipeline.AddNode( + pipeline.AddStep( label: "TransformGreetings", description: "Convert 'Hello' greetings into 'Goodbye' and 'So long' variants.", transform: TransformGreetingsNode.Create(), diff --git a/examples/starter/Minimal/Program.cs b/examples/starter/Minimal/Program.cs index cff12315..546dd473 100644 --- a/examples/starter/Minimal/Program.cs +++ b/examples/starter/Minimal/Program.cs @@ -43,7 +43,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa services.AddFlowthru(flowthru => { flowthru.UseConfiguration(opts => opts.ConfigurationPath = basePath); - flowthru.UseCatalog(_ => new Catalog(basePath)); + flowthru.RegisterCatalog(_ => new Catalog(basePath)); flowthru.ConfigureMetadata(meta => { var metadataPath = Path.Combine(basePath, "Metadata"); @@ -57,7 +57,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa // Register the greetings pipeline flowthru - .RegisterPipeline(label: "Greetings", pipeline: GreetingsPipeline.Create) + .RegisterFlow(label: "Greetings", flow: GreetingsPipeline.Create) .WithDescription( "A minimal pipeline demonstrating name transformation into multiple greeting formats" ); diff --git a/examples/starter/Minimal/README.md b/examples/starter/Minimal/README.md index b917a05a..70f04d34 100644 --- a/examples/starter/Minimal/README.md +++ b/examples/starter/Minimal/README.md @@ -63,4 +63,4 @@ View generated greetings in: - **Catalog entries** connecting schemas to files - **Simple node functions** with pure transformations - **Multiple outputs** from a single node (1→2) -- **Pipeline construction** with `PipelineBuilder` +- **Pipeline construction** with `FlowBuilder` diff --git a/examples/starter/SpaceflightsEFCore/.template.config/template.json b/examples/starter/SpaceflightsEFCore/.template.config/template.json index 714403a7..9f82b2ea 100644 --- a/examples/starter/SpaceflightsEFCore/.template.config/template.json +++ b/examples/starter/SpaceflightsEFCore/.template.config/template.json @@ -22,7 +22,7 @@ "FlowthruVersion": { "type": "parameter", "dataType": "string", - "defaultValue": "0.1.32", + "defaultValue": "0.1.33", "replaces": "FlowthruVersion", "description": "Flowthru package version (auto-injected at pack time)" } diff --git a/examples/starter/SpaceflightsEFCore/Data/Catalog.cs b/examples/starter/SpaceflightsEFCore/Data/Catalog.cs index 971b226b..634a95af 100644 --- a/examples/starter/SpaceflightsEFCore/Data/Catalog.cs +++ b/examples/starter/SpaceflightsEFCore/Data/Catalog.cs @@ -7,7 +7,7 @@ namespace SpaceflightsEFCore.Data; /// /// Data catalog for the Spaceflights pipeline, providing access to datasets across all data layers. /// -public partial class Catalog : DataCatalogBase +public partial class Catalog : CatalogAbstract { private readonly string _basePath; private readonly IDbContextFactory _contextFactory; diff --git a/examples/starter/SpaceflightsEFCore/Data/_01_Raw/Catalog.Raw.cs b/examples/starter/SpaceflightsEFCore/Data/_01_Raw/Catalog.Raw.cs index 59c88542..51c1832b 100644 --- a/examples/starter/SpaceflightsEFCore/Data/_01_Raw/Catalog.Raw.cs +++ b/examples/starter/SpaceflightsEFCore/Data/_01_Raw/Catalog.Raw.cs @@ -8,10 +8,10 @@ public partial class Catalog /// /// Raw company data imported from external sources. /// - public ICatalogEntry> Companies => - GetOrCreateEntry( + public IItem> Companies => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Companies", filePath: $"{_basePath}/_01_Raw/Datasets/companies.csv" ) @@ -20,10 +20,10 @@ public partial class Catalog /// /// Raw review data imported from external sources. /// - public ICatalogEntry> Reviews => - GetOrCreateEntry( + public IItem> Reviews => + CreateItem( () => - CatalogEntries.Enumerable.Csv( + ItemFactory.Enumerable.Csv( label: "Reviews", filePath: $"{_basePath}/_01_Raw/Datasets/reviews.csv" ) @@ -32,10 +32,10 @@ public partial class Catalog /// /// Raw shuttle data imported from external sources. /// - public ICatalogEntry> Shuttles => - GetOrCreateEntry( + public IItem> Shuttles => + CreateItem( () => - CatalogEntries.Enumerable.Excel( + ItemFactory.Enumerable.Excel( label: "Shuttles", filePath: $"{_basePath}/_01_Raw/Datasets/shuttles.xlsx", sheetName: "Sheet1" diff --git a/examples/starter/SpaceflightsEFCore/Data/_02_Intermediate/Catalog.Intermediate.cs b/examples/starter/SpaceflightsEFCore/Data/_02_Intermediate/Catalog.Intermediate.cs index 44154adf..5dd4ddbf 100644 --- a/examples/starter/SpaceflightsEFCore/Data/_02_Intermediate/Catalog.Intermediate.cs +++ b/examples/starter/SpaceflightsEFCore/Data/_02_Intermediate/Catalog.Intermediate.cs @@ -10,10 +10,10 @@ public partial class Catalog /// /// Preprocessed company data with validated and strongly-typed fields. /// - public ICatalogEntry> PreprocessedCompanies => - GetOrCreateEntry( + public IItem> PreprocessedCompanies => + CreateItem( () => - EFCoreCatalogEntries + EFCoreItemFactory .Enumerable.EFCore( label: "PreprocessedCompanies", contextFactory: _contextFactory @@ -24,10 +24,10 @@ public partial class Catalog /// /// Preprocessed shuttle data with validated and strongly-typed fields. /// - public ICatalogEntry> PreprocessedShuttles => - GetOrCreateEntry( + public IItem> PreprocessedShuttles => + CreateItem( () => - EFCoreCatalogEntries + EFCoreItemFactory .Enumerable.EFCore( label: "PreprocessedShuttles", contextFactory: _contextFactory diff --git a/examples/starter/SpaceflightsEFCore/Data/_03_Primary/Catalog.Primary.cs b/examples/starter/SpaceflightsEFCore/Data/_03_Primary/Catalog.Primary.cs index 9540dfa0..9cd8c0c2 100644 --- a/examples/starter/SpaceflightsEFCore/Data/_03_Primary/Catalog.Primary.cs +++ b/examples/starter/SpaceflightsEFCore/Data/_03_Primary/Catalog.Primary.cs @@ -14,10 +14,10 @@ public partial class Catalog /// Master table of features and labels prepared for model training. /// Combines preprocessed company and shuttle data into a single feature matrix. /// - public ICatalogEntry> ModelInputTable => - GetOrCreateEntry( + public IItem> ModelInputTable => + CreateItem( () => - EFCoreCatalogEntries.Enumerable.EFCore( + EFCoreItemFactory.Enumerable.EFCore( label: "ModelInputTable", contextFactory: _contextFactory, // Demonstrates query customization: order by shuttle ID for deterministic output. diff --git a/examples/starter/SpaceflightsEFCore/Data/_04_Feature/Catalog.Feature.cs b/examples/starter/SpaceflightsEFCore/Data/_04_Feature/Catalog.Feature.cs index b61d4f6b..429f460a 100644 --- a/examples/starter/SpaceflightsEFCore/Data/_04_Feature/Catalog.Feature.cs +++ b/examples/starter/SpaceflightsEFCore/Data/_04_Feature/Catalog.Feature.cs @@ -10,10 +10,10 @@ public partial class Catalog { // TODO: Add feature datasets here as they are created // Example: - // public ICatalogEntry> Features => - // GetOrCreateEntry( + // public IItem> Features => + // CreateEntry( // () => - // CatalogEntries.Enumerable.Csv( + // ItemFactory.Enumerable.Csv( // label: "Features", // filePath: $"{_basePath}/_04_Feature/Datasets/features.csv" // ) diff --git a/examples/starter/SpaceflightsEFCore/Data/_05_ModelInput/Catalog.ModelInput.cs b/examples/starter/SpaceflightsEFCore/Data/_05_ModelInput/Catalog.ModelInput.cs index 9515bc0f..84889930 100644 --- a/examples/starter/SpaceflightsEFCore/Data/_05_ModelInput/Catalog.ModelInput.cs +++ b/examples/starter/SpaceflightsEFCore/Data/_05_ModelInput/Catalog.ModelInput.cs @@ -13,10 +13,10 @@ public partial class Catalog /// /// Training dataset with features and labels. /// - public ICatalogEntry> TrainSplit => - GetOrCreateEntry( + public IItem> TrainSplit => + CreateItem( () => - EFCoreCatalogEntries.Enumerable.EFCore( + EFCoreItemFactory.Enumerable.EFCore( label: "XTrain", contextFactory: _contextFactory ) @@ -25,10 +25,10 @@ public partial class Catalog /// /// Test dataset with features and labels for model evaluation. /// - public ICatalogEntry> TestSplit => - GetOrCreateEntry( + public IItem> TestSplit => + CreateItem( () => - EFCoreCatalogEntries.Enumerable.EFCore( + EFCoreItemFactory.Enumerable.EFCore( label: "XTest", contextFactory: _contextFactory ) diff --git a/examples/starter/SpaceflightsEFCore/Data/_06_Models/Catalog.Models.cs b/examples/starter/SpaceflightsEFCore/Data/_06_Models/Catalog.Models.cs index b7213033..883adbc3 100644 --- a/examples/starter/SpaceflightsEFCore/Data/_06_Models/Catalog.Models.cs +++ b/examples/starter/SpaceflightsEFCore/Data/_06_Models/Catalog.Models.cs @@ -13,10 +13,10 @@ public partial class Catalog /// /// Trained linear regression model for price prediction. /// - public ICatalogEntry Regressor => - GetOrCreateEntry( + public IItem Regressor => + CreateItem( () => - EFCoreCatalogEntries.Single.EFCore( + EFCoreItemFactory.Single.EFCore( label: "Regressor", contextFactory: _contextFactory ) diff --git a/examples/starter/SpaceflightsEFCore/Data/_07_ModelOutput/Catalog.ModelOutput.cs b/examples/starter/SpaceflightsEFCore/Data/_07_ModelOutput/Catalog.ModelOutput.cs index cf84c3a1..c617704a 100644 --- a/examples/starter/SpaceflightsEFCore/Data/_07_ModelOutput/Catalog.ModelOutput.cs +++ b/examples/starter/SpaceflightsEFCore/Data/_07_ModelOutput/Catalog.ModelOutput.cs @@ -13,10 +13,10 @@ public partial class Catalog /// /// Evaluation metrics for the trained model (R², MAE, etc). /// - public ICatalogEntry ModelMetrics => - GetOrCreateEntry( + public IItem ModelMetrics => + CreateItem( () => - EFCoreCatalogEntries.Single.EFCore( + EFCoreItemFactory.Single.EFCore( label: "ModelMetrics", contextFactory: _contextFactory ) @@ -25,10 +25,10 @@ public partial class Catalog /// /// Model predictions on the test set with actual vs predicted prices. /// - public ICatalogEntry> ModelPredictions => - GetOrCreateEntry( + public IItem> ModelPredictions => + CreateItem( () => - EFCoreCatalogEntries.Enumerable.EFCore( + EFCoreItemFactory.Enumerable.EFCore( label: "ModelPredictions", contextFactory: _contextFactory ) diff --git a/examples/starter/SpaceflightsEFCore/Data/_08_Reporting/Catalog.Reporting.cs b/examples/starter/SpaceflightsEFCore/Data/_08_Reporting/Catalog.Reporting.cs index 3d62ab25..4c339220 100644 --- a/examples/starter/SpaceflightsEFCore/Data/_08_Reporting/Catalog.Reporting.cs +++ b/examples/starter/SpaceflightsEFCore/Data/_08_Reporting/Catalog.Reporting.cs @@ -13,10 +13,10 @@ public partial class Catalog /// /// Passenger capacity analysis report grouped by shuttle type. /// - public ICatalogEntry> ShuttleCapacityReport => - GetOrCreateEntry( + public IItem> ShuttleCapacityReport => + CreateItem( () => - CatalogEntries.Enumerable.Json( + ItemFactory.Enumerable.Json( label: "ShuttleCapacityReport", filePath: $"{_basePath}/_08_Reporting/Datasets/shuttle_capacity_report.json" ) @@ -26,9 +26,9 @@ public partial class Catalog /// Shuttle passenger capacity bar chart (in-memory GenericChart). /// Intermediate chart object stored in memory for downstream export to PNG. /// - public ICatalogEntry ShuttlePassengerCapacityChart => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "ShuttlePassengerCapacityChart") + public IItem ShuttlePassengerCapacityChart => + CreateItem( + () => ItemFactory.Single.Memory(label: "ShuttlePassengerCapacityChart") ); /// @@ -36,10 +36,10 @@ public partial class Catalog /// Static image representation of the passenger capacity visualization. /// Stored as binary PNG file. /// - public ICatalogEntry ShuttlePassengerCapacityPlotPng => - GetOrCreateEntry( + public IItem ShuttlePassengerCapacityPlotPng => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "ShuttlePassengerCapacityPlotPng", filePath: $"{_basePath}/_08_Reporting/Images/shuttle_passenger_capacity_plot.png" ) @@ -49,20 +49,18 @@ public partial class Catalog /// Confusion matrix heatmap (in-memory GenericChart). /// Intermediate chart object stored in memory for downstream export to PNG. /// - public ICatalogEntry ConfusionMatrixChart => - GetOrCreateEntry( - () => CatalogEntries.Single.Memory(label: "ConfusionMatrixChart") - ); + public IItem ConfusionMatrixChart => + CreateItem(() => ItemFactory.Single.Memory(label: "ConfusionMatrixChart")); /// /// Confusion matrix heatmap (PNG image). /// Static image representation of the confusion matrix visualization. /// Stored as binary PNG file. /// - public ICatalogEntry ConfusionMatrixPlotPng => - GetOrCreateEntry( + public IItem ConfusionMatrixPlotPng => + CreateItem( () => - CatalogEntries.Single.Binary( + ItemFactory.Single.Binary( label: "ConfusionMatrixPlotPng", filePath: $"{_basePath}/_08_Reporting/Images/confusion_matrix_plot.png" ) diff --git a/examples/starter/SpaceflightsEFCore/Pipelines/DataProcessing/DataProcessingPipeline.cs b/examples/starter/SpaceflightsEFCore/Pipelines/DataProcessing/DataProcessingPipeline.cs index c7f1a81a..97a4b0c6 100644 --- a/examples/starter/SpaceflightsEFCore/Pipelines/DataProcessing/DataProcessingPipeline.cs +++ b/examples/starter/SpaceflightsEFCore/Pipelines/DataProcessing/DataProcessingPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using SpaceflightsEFCore.Data; using SpaceflightsEFCore.Pipelines.DataProcessing.Nodes; @@ -15,11 +15,11 @@ public static class DataProcessingPipeline /// /// The data catalog containing input and output entries. /// A configured pipeline that produces a model input table from raw data sources. - public static Pipeline Create(Catalog catalog) + public static Flow Create(Catalog catalog) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "PreprocessCompanies", description: "Cleans and preprocesses raw company data.", transform: PreprocessCompaniesNode.Create(), @@ -27,7 +27,7 @@ public static Pipeline Create(Catalog catalog) output: catalog.PreprocessedCompanies ); - pipeline.AddNode( + pipeline.AddStep( label: "PreprocessShuttles", description: "Cleans and preprocesses raw shuttle data.", transform: PreprocessShuttlesNode.Create(), @@ -35,7 +35,7 @@ public static Pipeline Create(Catalog catalog) output: catalog.PreprocessedShuttles ); - pipeline.AddNode( + pipeline.AddStep( label: "CreateModelInputTable", description: """ Joins preprocessed shuttle and company data with review scores to create a diff --git a/examples/starter/SpaceflightsEFCore/Pipelines/DataScience/DataSciencePipeline.cs b/examples/starter/SpaceflightsEFCore/Pipelines/DataScience/DataSciencePipeline.cs index 609fcad3..379f4957 100644 --- a/examples/starter/SpaceflightsEFCore/Pipelines/DataScience/DataSciencePipeline.cs +++ b/examples/starter/SpaceflightsEFCore/Pipelines/DataScience/DataSciencePipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using SpaceflightsEFCore.Data; namespace SpaceflightsEFCore.Pipelines.DataScience; @@ -25,11 +25,11 @@ public record Params /// The data catalog containing input and output entries. /// Configuration parameters for the pipeline. /// A configured pipeline that produces a trained model and evaluation metrics. - public static Pipeline Create(Catalog catalog, Params parameters) + public static Flow Create(Catalog catalog, Params parameters) { - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { - pipeline.AddNode( + pipeline.AddStep( label: "SplitData", description: "Splits model input data into training and test sets.", transform: Nodes.SplitDataNode.Create(parameters.ModelOptions), @@ -37,7 +37,7 @@ public static Pipeline Create(Catalog catalog, Params parameters) output: (catalog.TrainSplit, catalog.TestSplit) ); - pipeline.AddNode( + pipeline.AddStep( label: "TrainModel", description: "Trains a regression model to predict shuttle prices.", transform: Nodes.TrainModelNode.Create(), @@ -45,7 +45,7 @@ public static Pipeline Create(Catalog catalog, Params parameters) output: catalog.Regressor ); - pipeline.AddNode( + pipeline.AddStep( label: "EvaluateModel", description: "Evaluates the trained model on the test set and computes metrics and predictions.", transform: Nodes.EvaluateModelNode.Create(), diff --git a/examples/starter/SpaceflightsEFCore/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs b/examples/starter/SpaceflightsEFCore/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs index 3bc36ec3..2099e9cb 100644 --- a/examples/starter/SpaceflightsEFCore/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs +++ b/examples/starter/SpaceflightsEFCore/Pipelines/Reporting/Nodes/PlotlyImageExportNode.cs @@ -11,13 +11,13 @@ namespace SpaceflightsEFCore.Pipelines.Reporting.Nodes; /// /// This node handles the PNG export concern separately from chart generation, /// enabling a clean separation of visualization logic from output format concerns. -/// The PNG binary data can be stored in a BinaryFileCatalogEntry<byte[]>. +/// The PNG binary data can be stored in a BinaryFileItem<byte[]>. /// /// /// Input: GenericChart object from memory catalog /// /// -/// Output: PNG binary data as byte[] (can be stored in BinaryFileCatalogEntry<byte[]>) +/// Output: PNG binary data as byte[] (can be stored in BinaryFileItem<byte[]>) /// /// /// Reusability: This node can be used for any Plotly chart type diff --git a/examples/starter/SpaceflightsEFCore/Pipelines/Reporting/ReportingPipeline.cs b/examples/starter/SpaceflightsEFCore/Pipelines/Reporting/ReportingPipeline.cs index ccb8f591..012d340b 100644 --- a/examples/starter/SpaceflightsEFCore/Pipelines/Reporting/ReportingPipeline.cs +++ b/examples/starter/SpaceflightsEFCore/Pipelines/Reporting/ReportingPipeline.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using SpaceflightsEFCore.Data; using SpaceflightsEFCore.Pipelines.Reporting.Nodes; @@ -41,15 +41,15 @@ public record Params /// The data catalog containing input and output entries. /// Configuration parameters for the pipeline (optional). /// A configured pipeline that produces visualizations and reports. - public static Pipeline Create(Catalog catalog, Params? parameters = null) + public static Flow Create(Catalog catalog, Params? parameters = null) { var p = parameters ?? new Params(); - return PipelineBuilder.CreatePipeline(pipeline => + return FlowBuilder.CreateFlow(pipeline => { // ===== Shuttle Passenger Capacity Report (JSON) ===== - pipeline.AddNode( + pipeline.AddStep( label: "ComparePassengerCapacity", transform: ComparePassengerCapacityNode.Create(), input: catalog.PreprocessedShuttles, @@ -59,7 +59,7 @@ public static Pipeline Create(Catalog catalog, Params? parameters = null) // ===== Shuttle Passenger Capacity Visualization ===== // Step 1: Generate chart from preprocessed shuttle data - pipeline.AddNode( + pipeline.AddStep( label: "GeneratePassengerCapacityChart", transform: GeneratePassengerCapacityChartNode.Create(), input: catalog.PreprocessedShuttles, @@ -68,7 +68,7 @@ public static Pipeline Create(Catalog catalog, Params? parameters = null) // NOTE: Commented out due to performance issues with Plotly.NET // // Step 2: Export chart to PNG for static reports - // pipeline.AddNode( + // pipeline.AddStep( // label: "ExportPassengerCapacityPng", // transform: PlotlyImageExportNode.Create(), // input: catalog.ShuttlePassengerCapacityChart, @@ -78,7 +78,7 @@ public static Pipeline Create(Catalog catalog, Params? parameters = null) // ===== Confusion Matrix Visualization ===== // Step 1: Generate confusion matrix heatmap from model predictions - pipeline.AddNode( + pipeline.AddStep( label: "GenerateConfusionMatrixChart", transform: CreateConfusionMatrixNode.Create(p.ConfusionMatrixOptions), input: catalog.ModelPredictions, @@ -87,7 +87,7 @@ public static Pipeline Create(Catalog catalog, Params? parameters = null) // NOTE: Commented out due to performance issues with Plotly.NET // // Step 2: Export chart to PNG for static reports - // pipeline.AddNode( + // pipeline.AddStep( // label: "ExportConfusionMatrixPng", // transform: PlotlyImageExportNode.Create(), // input: catalog.ConfusionMatrixChart, diff --git a/examples/starter/SpaceflightsEFCore/Program.cs b/examples/starter/SpaceflightsEFCore/Program.cs index f548617b..94adf1b9 100644 --- a/examples/starter/SpaceflightsEFCore/Program.cs +++ b/examples/starter/SpaceflightsEFCore/Program.cs @@ -54,30 +54,30 @@ private static void ConfigureServices(IServiceCollection services, string basePa services.AddFlowthru(flowthru => { flowthru.UseConfiguration(opts => opts.ConfigurationPath = basePath); - flowthru.UseCatalog(sp => new Catalog( + flowthru.RegisterCatalog(sp => new Catalog( basePath: Path.Combine(basePath, "Data"), contextFactory: sp.GetRequiredService>() )); // Register data processing pipeline flowthru - .RegisterPipeline(label: "DataProcessing", pipeline: DataProcessingPipeline.Create) + .RegisterFlow(label: "DataProcessing", flow: DataProcessingPipeline.Create) .WithDescription("Preprocesses companies and shuttles data"); // Register data science pipeline with configuration parameters flowthru - .RegisterPipeline( + .RegisterFlow( label: "DataScience", - pipeline: DataSciencePipeline.Create, + flow: DataSciencePipeline.Create, configurationSection: "Flowthru:Pipelines:DataScience" ) .WithDescription("Trains linear regression model for price prediction"); // Register reporting pipeline with configuration parameters flowthru - .RegisterPipeline( + .RegisterFlow( label: "Reporting", - pipeline: ReportingPipeline.Create, + flow: ReportingPipeline.Create, configurationSection: "Flowthru:Pipelines:Reporting" ) .WithDescription("Generates passenger capacity reports and visualizations"); diff --git a/examples/starter/SpaceflightsEFCore/README.md b/examples/starter/SpaceflightsEFCore/README.md index bd14b926..c7e7be26 100644 --- a/examples/starter/SpaceflightsEFCore/README.md +++ b/examples/starter/SpaceflightsEFCore/README.md @@ -8,7 +8,7 @@ This example demonstrates using `Flowthru.Extensions.EFCore` to read and write d - ✅ Database catalog entries as pipeline seeds (Layer 0 inputs) - ✅ Injected DbContext lifecycle management - ✅ Reading and writing entities via EFCore adapter -- ✅ Partial class pattern for extending `CatalogEntries` from external package +- ✅ Partial class pattern for extending `Items` from external package ## Project Structure @@ -72,12 +72,12 @@ public class AppDbContext : DbContext public static partial class DataCatalog { // Database source (seed) - public static ICatalogEntry> SourceCompanies(DbContext db) => - CatalogEntries.Enumerable.EFCore("source_companies", db, readOnly: true); + public static IItem> SourceCompanies(DbContext db) => + Items.Enumerable.EFCore("source_companies", db, readOnly: true); // Database destination (output) - public static ICatalogEntry> ProcessedCompanies(DbContext db) => - CatalogEntries.Enumerable.EFCore("processed_companies", db); + public static IItem> ProcessedCompanies(DbContext db) => + Items.Enumerable.EFCore("processed_companies", db); } ``` @@ -88,12 +88,12 @@ public static partial class DataCatalog using var db = new AppDbContext(); await db.Database.MigrateAsync(); -var pipeline = new PipelineBuilder("CompanyETL") - .AddNode("extract", catalog => new ExtractCompaniesNode( +var pipeline = new FlowBuilder("CompanyETL") + .AddStep("extract", catalog => new ExtractCompaniesNode( inputs: catalog.SourceCompanies(db), outputs: catalog.RawCompanies() )) - .AddNode("transform", catalog => new TransformCompaniesNode( + .AddStep("transform", catalog => new TransformCompaniesNode( inputs: catalog.RawCompanies(), outputs: catalog.ProcessedCompanies(db) )) diff --git a/scripts/docfx-metadata.sh b/scripts/docfx-metadata.sh new file mode 100755 index 00000000..914d567c --- /dev/null +++ b/scripts/docfx-metadata.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Generates markdown API reference docs into docs/reference/{src,examples}/ mirroring +# the repo structure. Discovers .csproj files in src/ and/or examples/, writes a +# temporary per-project docfx config, runs `docfx metadata`, and removes the temp config. +# +# Output layout example: +# docs/reference/src/core/Flowthru.Core/ +# docs/reference/src/extensions/Flowthru.Extensions.Csv/ +# +# Usage: ./scripts/docfx-metadata.sh [--projects src|examples|all] +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SCOPE="${1:---projects}" +FILTER="${2:-src}" + +# Parse arguments +if [[ "$SCOPE" == "--projects" ]]; then + SCOPE="$FILTER" +elif [[ "$SCOPE" != "src" && "$SCOPE" != "examples" && "$SCOPE" != "all" ]]; then + SCOPE="src" +fi + +# Build find paths based on scope +FIND_PATHS=() +case "$SCOPE" in + src) FIND_PATHS=("$REPO_ROOT/src") ;; + examples) FIND_PATHS=("$REPO_ROOT/examples") ;; + all) FIND_PATHS=("$REPO_ROOT/src" "$REPO_ROOT/examples") ;; +esac + +# Discover projects, excluding source generators, archived, and template projects +PROJECTS=() +while IFS= read -r line; do + PROJECTS+=("$line") +done < <( + find "${FIND_PATHS[@]}" \ + -name '*.csproj' \ + -not -path '*/archived/*' \ + -not -path '*/item-templates/*' \ + -not -path '*/bin/*' \ + -not -path '*/obj/*' \ + -not -path '*/dist/*' \ + -not -name '*SourceGenerators*' \ + | sort +) + +if [[ ${#PROJECTS[@]} -eq 0 ]]; then + echo "No projects found for scope: $SCOPE" + exit 0 +fi + +echo "Generating API reference docs for ${#PROJECTS[@]} projects (scope: $SCOPE)" +echo "---" + +FAILURES=() + +for csproj in "${PROJECTS[@]}"; do + project_dir="$(dirname "$csproj")" + csproj_name="$(basename "$csproj")" + project_name="${csproj_name%.csproj}" + relative_dir="${project_dir#"$REPO_ROOT"/}" + config_path="$project_dir/.docfx.metadata.json" + + # Output goes to docs/reference/{relative_dir}/, computed as a path relative + # to the config file location (which is in the project directory). + output_dir="$REPO_ROOT/docs/reference/$relative_dir" + # Make dest relative to config file (project_dir) + dest_rel="$(python3 -c "import os; print(os.path.relpath('$output_dir', '$project_dir'))")" + + echo "[$project_name] Generating metadata..." + + # Write temporary docfx config in the project directory so .csproj paths resolve naturally. + # shouldSkipMarkup: workaround for Markdig crash in DocFX 2.78.5 (ArgumentOutOfRangeException + # in XmlComment.cs). HTML tags from XML docs (

, , etc.) pass through as valid + # CommonMark inline HTML, which renders correctly on GitHub and VS Code. + cat > "$config_path" << EOF +{ + "metadata": [{ + "src": [{ "files": ["$csproj_name"] }], + "dest": "$dest_rel", + "outputFormat": "markdown", + "disableGitFeatures": true, + "shouldSkipMarkup": true + }] +} +EOF + + if dotnet docfx metadata "$config_path" --noRestore 2>&1 | sed 's/^/ /'; then + echo " ✓ docs/reference/$relative_dir/" + else + echo " ✗ Failed for $project_name" + FAILURES+=("$project_name") + fi + + rm -f "$config_path" +done + +echo "---" +if [[ ${#FAILURES[@]} -gt 0 ]]; then + echo "Completed with ${#FAILURES[@]} failure(s): ${FAILURES[*]}" + exit 1 +else + echo "✓ All ${#PROJECTS[@]} projects processed successfully" +fi diff --git a/src/core/Flowthru.Core/Cli/ArgumentParser.cs b/src/core/Flowthru.Core/Cli/ArgumentParser.cs index b4a5e926..c7654422 100644 --- a/src/core/Flowthru.Core/Cli/ArgumentParser.cs +++ b/src/core/Flowthru.Core/Cli/ArgumentParser.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using Flowthru.Services.Models; namespace Flowthru.Cli; @@ -151,7 +151,7 @@ public static ParsedArguments Parse(string[] args, IEnumerable available } // Build slice strategy if any slicing flags were provided - PipelineSliceStrategy? sliceStrategy = null; + FlowSliceStrategy? sliceStrategy = null; if ( pipelines != null || fromNodes != null @@ -161,9 +161,9 @@ public static ParsedArguments Parse(string[] args, IEnumerable available || onlyNodes != null ) { - sliceStrategy = new PipelineSliceStrategy + sliceStrategy = new FlowSliceStrategy { - Pipelines = pipelines, + Flows = pipelines, FromNodes = fromNodes, ToNodes = toNodes, FromData = fromData, diff --git a/src/core/Flowthru.Core/Cli/FlowthruCli.cs b/src/core/Flowthru.Core/Cli/FlowthruCli.cs index dd05c144..53332fd5 100644 --- a/src/core/Flowthru.Core/Cli/FlowthruCli.cs +++ b/src/core/Flowthru.Core/Cli/FlowthruCli.cs @@ -1,5 +1,5 @@ using System.Reflection; -using Flowthru.Pipelines; +using Flowthru.Flows; using Flowthru.Services; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -46,7 +46,8 @@ public FlowthruCli( _output = output ?? Console.Out; } - /// + /// Creates and runs a standalone Flowthru CLI application with automatic service provider lifecycle management. /// /// /// @@ -107,7 +108,7 @@ public async Task RunAsync(string[] args, CancellationToken cancellationTok try { // Parse arguments - var parsed = ArgumentParser.Parse(args, _service.PipelineNames); + var parsed = ArgumentParser.Parse(args, _service.FlowNames); // Handle special commands if (parsed.ShowHelp) @@ -131,7 +132,7 @@ public async Task RunAsync(string[] args, CancellationToken cancellationTok } // Execute unified pipeline (with optional slicing) - var result = await _service.ExecutePipelineAsync( + var result = await _service.ExecuteFlowAsync( parsed.Options, parsed.ExportMetadata, parsed.MetadataOutputDirectory, @@ -194,9 +195,9 @@ private void ShowHelp() ); _output.WriteLine(); _output.WriteLine("Available Pipelines:"); - foreach (var name in _service.PipelineNames.OrderBy(n => n)) + foreach (var name in _service.FlowNames.OrderBy(n => n)) { - var metadata = _service.GetPipelineMetadata(name); + var metadata = _service.GetFlowMetadata(name); _output.WriteLine($" {name, -20} {metadata.Description ?? "(no description)"}"); } } @@ -229,20 +230,20 @@ private void ShowVersion() /// Formats pipeline execution results. /// /// Pipeline execution result - private void FormatResult(PipelineResult result) + private void FormatResult(FlowResult result) { _output.WriteLine(); _output.WriteLine("═══════════════════════════════════════════════════════════"); - _output.WriteLine($"Pipeline: {result.PipelineName ?? "(merged)"}"); + _output.WriteLine($"Pipeline: {result.FlowName ?? "(merged)"}"); _output.WriteLine($"Status: {(result.Success ? "✓ SUCCESS" : "✗ FAILED")}"); _output.WriteLine($"Duration: {result.ExecutionTime:hh\\:mm\\:ss\\.fff}"); - _output.WriteLine($"Nodes: {result.NodeResults.Count} executed"); + _output.WriteLine($"Nodes: {result.StepResults.Count} executed"); if (!result.Success) { _output.WriteLine(); _output.WriteLine("Failed Nodes:"); - foreach (var (label, nodeResult) in result.NodeResults.Where(n => !n.Value.Success)) + foreach (var (label, nodeResult) in result.StepResults.Where(n => !n.Value.Success)) { _output.WriteLine($" - {label}: {nodeResult.Exception?.Message ?? "Unknown error"}"); } diff --git a/src/core/Flowthru.Core/Configuration/FlowthruOptions.cs b/src/core/Flowthru.Core/Configuration/FlowthruOptions.cs index be72461d..a2854871 100644 --- a/src/core/Flowthru.Core/Configuration/FlowthruOptions.cs +++ b/src/core/Flowthru.Core/Configuration/FlowthruOptions.cs @@ -27,9 +27,9 @@ public class FlowthruOptions public CatalogOptions Catalog { get; set; } = new(); ///

- /// Pipeline registration and configuration. + /// Flow registration and configuration. /// - public Dictionary Pipelines { get; set; } = new(); + public Dictionary Flows { get; set; } = new(); /// /// Logging configuration (extends standard .NET logging configuration). @@ -65,7 +65,7 @@ public class MetadataOptions /// Supports dynamic tokens that are replaced during export: /// /// - /// {PipelineName} - Sanitized pipeline name + /// {FlowName} - Sanitized Flow name /// {Timestamp} - Formatted timestamp (empty if disabled in Timestamp.IncludeTimestamp) /// {SliceType} - "FromNodes", "Tags", "Mixed", or empty if unsliced /// {FromNodes} - Comma-separated list of from-nodes @@ -79,7 +79,7 @@ public class MetadataOptions /// File extensions are added by individual providers (.json, .md, etc.). /// /// - /// Default: "dag-{PipelineName}-{Timestamp}-{SliceType}" + /// Default: "dag-{FlowName}-{Timestamp}-{SliceType}" /// /// /// Examples: @@ -89,7 +89,7 @@ public class MetadataOptions /// Sliced: dag-DataProcessing-20260304-153045-FromNodes.json /// /// - public string FilenameTemplate { get; set; } = "dag-{PipelineName}-{Timestamp}-{SliceType}"; + public string FilenameTemplate { get; set; } = "dag-{FlowName}-{Timestamp}-{SliceType}"; /// /// Configuration for timestamp generation in metadata filenames. @@ -140,7 +140,7 @@ public class MermaidMetadataOptions /// Color applied to nodes that are in the execution slice. /// Default: #2E7D32 (Material Design green-800). /// - public string ActiveNodeColor { get; set; } = "#2E7D32"; + public string ActiveStepColor { get; set; } = "#2E7D32"; /// /// Hex color code for active (sliced) catalog entries. @@ -194,12 +194,12 @@ public class CatalogOptions } /// -/// Configuration options for a single pipeline. +/// Configuration options for a single flow. /// -public class PipelineOptions +public class FlowOptions { /// - /// The fully-qualified type name of the pipeline factory class. + /// The fully-qualified type name of the Flow factory class. /// Must have a static Create method that accepts (catalog, parameters?). /// public string? Type { get; set; } @@ -210,26 +210,26 @@ public class PipelineOptions public string FactoryMethod { get; set; } = "Create"; /// - /// Human-readable description of the pipeline. + /// Human-readable description of the Flow. /// public string? Description { get; set; } /// - /// Pipeline-specific parameters (nested configuration section). - /// The structure must match the pipeline's parameter type. + /// Flow-specific parameters (nested configuration section). + /// The structure must match the Flow's parameter type. /// public Dictionary? Parameters { get; set; } /// - /// Validation configuration for this pipeline. + /// Validation configuration for this flow. /// - public PipelineValidationOptions? Validation { get; set; } + public FlowValidationOptions? Validation { get; set; } } /// -/// Configuration options for pipeline validation behavior. +/// Configuration options for Flow validation behavior. /// -public class PipelineValidationOptions +public class FlowValidationOptions { /// /// Default inspection level for all Layer 0 inputs. diff --git a/src/core/Flowthru.Core/Configuration/ICatalogFactory.cs b/src/core/Flowthru.Core/Configuration/ICatalogFactory.cs index 8ee901b1..0e23eea4 100644 --- a/src/core/Flowthru.Core/Configuration/ICatalogFactory.cs +++ b/src/core/Flowthru.Core/Configuration/ICatalogFactory.cs @@ -18,7 +18,7 @@ public interface ICatalogFactory /// Catalog configuration options /// Service provider for dependency injection /// The configured catalog instance - DataCatalogBase CreateCatalog(CatalogOptions options, IServiceProvider serviceProvider); + CatalogAbstract CreateCatalog(CatalogOptions options, IServiceProvider serviceProvider); } /// @@ -32,7 +32,7 @@ public interface ICatalogFactory /// internal class ReflectionCatalogFactory : ICatalogFactory { - public DataCatalogBase CreateCatalog(CatalogOptions options, IServiceProvider serviceProvider) + public CatalogAbstract CreateCatalog(CatalogOptions options, IServiceProvider serviceProvider) { if (string.IsNullOrWhiteSpace(options.Type)) { @@ -69,7 +69,7 @@ public DataCatalogBase CreateCatalog(CatalogOptions options, IServiceProvider se ); } - if (!typeof(DataCatalogBase).IsAssignableFrom(catalogType)) + if (!typeof(CatalogAbstract).IsAssignableFrom(catalogType)) { throw new InvalidOperationException( $"Type '{options.Type}' does not inherit from DataCatalogBase." @@ -140,7 +140,7 @@ public DataCatalogBase CreateCatalog(CatalogOptions options, IServiceProvider se if (allMatched) { - var catalog = (DataCatalogBase)constructor.Invoke(constructorArgs); + var catalog = (CatalogAbstract)constructor.Invoke(constructorArgs); return catalog; } } diff --git a/src/core/Flowthru.Core/Configuration/PipelineDiscoveryService.cs b/src/core/Flowthru.Core/Configuration/PipelineDiscoveryService.cs index 4796437c..d88e057b 100644 --- a/src/core/Flowthru.Core/Configuration/PipelineDiscoveryService.cs +++ b/src/core/Flowthru.Core/Configuration/PipelineDiscoveryService.cs @@ -1,65 +1,65 @@ using System.Reflection; using Flowthru.Data; -using Flowthru.Pipelines; +using Flowthru.Flows; using Microsoft.Extensions.Configuration; namespace Flowthru.Configuration; /// -/// Discovers and registers pipelines from configuration. +/// Discovers and registers flows from configuration. /// -internal class PipelineDiscoveryService +internal class FlowDiscoveryService { /// - /// Discovers pipelines from configuration and returns factory functions. + /// Discovers flows from configuration and returns factory functions. /// /// Configuration instance /// The catalog type to use - /// Dictionary of pipeline label to factory function - public static Dictionary DiscoverPipelines( + /// Dictionary of Flow label to factory function + public static Dictionary DiscoverFlows( IConfiguration configuration, Type catalogType ) { var flowthruConfig = configuration.GetSection(FlowthruOptions.SectionName); - var pipelinesSection = flowthruConfig.GetSection("Pipelines"); + var flowsSection = flowthruConfig.GetSection("Flows"); - if (!pipelinesSection.Exists()) + if (!flowsSection.Exists()) { - return new Dictionary(); + return new Dictionary(); } - var pipelines = new Dictionary(); + var flows = new Dictionary(); - foreach (var pipelineSection in pipelinesSection.GetChildren()) + foreach (var flowSection in flowsSection.GetChildren()) { - var label = pipelineSection.Key; - var options = new PipelineOptions(); - pipelineSection.Bind(options); + var label = flowSection.Key; + var options = new FlowOptions(); + flowSection.Bind(options); if (string.IsNullOrWhiteSpace(options.Type)) { throw new InvalidOperationException( - $"Pipeline '{label}' is missing required 'Type' configuration. " - + $"Example: \"Pipelines\": {{ \"{label}\": {{ \"Type\": \"MyApp.Pipelines.MyPipeline\" }} }}" + $"Flow '{label}' is missing required 'Type' configuration. " + + $"Example: \"Flows\": {{ \"{label}\": {{ \"Type\": \"MyApp.Flows.MyFlow\" }} }}" ); } - var factoryInfo = CreateFactoryInfo(label, options, catalogType, pipelineSection); - pipelines[label] = factoryInfo; + var factoryInfo = CreateFactoryInfo(label, options, catalogType, flowSection); + flows[label] = factoryInfo; } - return pipelines; + return flows; } - private static PipelineFactoryInfo CreateFactoryInfo( + private static FlowFactoryInfo CreateFactoryInfo( string label, - PipelineOptions options, + FlowOptions options, Type catalogType, - IConfigurationSection pipelineSection + IConfigurationSection flowSection ) { - // Find the pipeline factory type + // Find the Flow factory type Type? factoryType = null; // Try Type.GetType first (supports fully qualified names with assembly) @@ -81,8 +81,8 @@ IConfigurationSection pipelineSection if (factoryType == null) { throw new InvalidOperationException( - $"Could not find pipeline type '{options.Type}' for pipeline '{label}'. " - + $"Ensure the type name is fully qualified (e.g., 'MyApp.Pipelines.MyPipeline')." + $"Could not find flow type '{options.Type}' for flow '{label}'. " + + $"Ensure the type name is fully qualified (e.g., 'MyApp.Flows.MyFlow')." ); } @@ -95,9 +95,9 @@ IConfigurationSection pipelineSection if (factoryMethod == null) { throw new InvalidOperationException( - $"Could not find static method '{options.FactoryMethod}' on type '{options.Type}' for pipeline '{label}'. " - + $"Expected signature: public static Pipeline Create({catalogType.Name} catalog) or " - + $"public static Pipeline Create({catalogType.Name} catalog, TParams parameters)" + $"Could not find static method '{options.FactoryMethod}' on type '{options.Type}' for Flow '{label}'. " + + $"Expected signature: public static Flow Create({catalogType.Name} catalog) or " + + $"public static Flow Create({catalogType.Name} catalog, TParams parameters)" ); } @@ -110,33 +110,33 @@ IConfigurationSection pipelineSection ); } - // Check if this is a parameterless or parameterized pipeline + // Check if this is a parameterless or parameterized flow Type? parameterType = null; object? parameterInstance = null; if (parameters.Length > 1) { - // Parameterized pipeline + // Parameterized flow parameterType = parameters[1].ParameterType; // Load and validate parameters from configuration - var parametersSection = pipelineSection.GetSection("Parameters"); + var parametersSection = flowSection.GetSection("Parameters"); if (!parametersSection.Exists() && options.Parameters == null) { throw new InvalidOperationException( - $"Pipeline '{label}' requires parameters of type '{parameterType.Name}', " + $"Flow '{label}' requires parameters of type '{parameterType.Name}', " + $"but no 'Parameters' section was found in configuration." ); } parameterInstance = ConfigurationExtensions.GetValidated( - pipelineSection, + flowSection, "Parameters", parameterType ); } - return new PipelineFactoryInfo + return new FlowFactoryInfo { Label = label, FactoryType = factoryType, @@ -150,9 +150,9 @@ IConfigurationSection pipelineSection } /// -/// Information about a discovered pipeline factory. +/// Information about a discovered Flow factory. /// -internal class PipelineFactoryInfo +internal class FlowFactoryInfo { public required string Label { get; init; } public required Type FactoryType { get; init; } @@ -160,25 +160,25 @@ internal class PipelineFactoryInfo public Type? ParameterType { get; init; } public object? ParameterInstance { get; init; } public string? Description { get; init; } - public PipelineValidationOptions? ValidationOptions { get; init; } + public FlowValidationOptions? ValidationOptions { get; init; } /// - /// Invokes the factory method to create a pipeline instance. + /// Invokes the factory method to create a Flow instance. /// - public Pipeline CreatePipeline(DataCatalogBase catalog) + public Flow CreateFlow(CatalogAbstract catalog) { var args = ParameterInstance != null ? new object[] { catalog, ParameterInstance } : new object[] { catalog }; - if (FactoryMethod.Invoke(null, args) is not Pipeline pipeline) + if (FactoryMethod.Invoke(null, args) is not Flow flow) { throw new InvalidOperationException( - $"Factory method '{FactoryMethod.Name}' on type '{FactoryType.Name}' returned null or non-Pipeline value." + $"Factory method '{FactoryMethod.Name}' on type '{FactoryType.Name}' returned null or non-Flow value." ); } - return pipeline; + return flow; } } diff --git a/src/core/Flowthru.Core/Data/Capabilities/StorageTraits.cs b/src/core/Flowthru.Core/Data/Capabilities/StorageTraits.cs index 98c8d627..a78ca1c3 100644 --- a/src/core/Flowthru.Core/Data/Capabilities/StorageTraits.cs +++ b/src/core/Flowthru.Core/Data/Capabilities/StorageTraits.cs @@ -40,7 +40,7 @@ namespace Flowthru.Data.Capabilities; /// /// Adapter level: The adapter author declares what the storage medium intrinsically supports. /// These are structural truths (e.g., an HTTP GET endpoint cannot write). -/// Catalog level: The pipeline author can further constrain an adapter using CatalogEntry.Constrain(). +/// Catalog level: The pipeline author can further constrain an adapter using Item.Constrain(). /// Constraints can only tighten, never loosen (one-way ratchet). /// /// @@ -67,8 +67,8 @@ namespace Flowthru.Data.Capabilities; /// Usage in Catalogs: /// /// -/// public ICatalogEntry<IEnumerable<Company>> ReferenceData => -/// GetOrCreateEntry(() => CatalogEntries.Enumerable.Csv<Company>( +/// public IItem<IEnumerable<Company>> ReferenceData => +/// CreateEntry(() => ItemFactory.Enumerable.Csv<Company>( /// "ref_data", $"{_basePath}/reference.csv") /// .Constrain(t => t with { CanWrite = false })); /// diff --git a/src/core/Flowthru.Core/Data/CatalogEntries.Single.cs b/src/core/Flowthru.Core/Data/CatalogEntries.Single.cs index 1ad1af0f..2a2399ee 100644 --- a/src/core/Flowthru.Core/Data/CatalogEntries.Single.cs +++ b/src/core/Flowthru.Core/Data/CatalogEntries.Single.cs @@ -3,14 +3,14 @@ namespace Flowthru.Data; -public static partial class CatalogEntries +public static partial class ItemFactory { /// /// Factory methods for single (non-collection) values. /// /// /// - /// These methods create catalog entries for single objects rather than collections. + /// These methods create catalog items for single objects rather than collections. /// /// /// Use Cases: @@ -20,18 +20,18 @@ public static partial class CatalogEntries /// Metrics and evaluation results (single JSON objects) /// Text reports (Markdown, plain text) /// Binary files (images, PDFs) - /// Side-effect-only nodes (null/void semantics) + /// Side-effect-only steps (null/void semantics) /// /// public static partial class Single { /// - /// Creates a JSON file catalog entry for a single object (non-collection). + /// Creates a JSON file catalog item for a single object (non-collection). /// /// Object type (must be structured-serializable) /// Unique catalog label for DAG resolution /// Path to JSON file - /// Catalog entry for singleton JSON object + /// Catalog item for singleton JSON object /// /// /// Use Case: Model files, configuration objects, metrics, single records @@ -44,19 +44,19 @@ public static partial class Single /// format/container composition for direct object serialization. /// /// - public static CatalogEntry Json(string label, string filePath) + public static Item Json(string label, string filePath) where T : IStructuredSerializable { var storage = new SingletonJsonStorageAdapter(filePath); - return new CatalogEntry(label, storage); + return new Item(label, storage); } /// - /// Creates a memory catalog entry for a single object (non-collection). + /// Creates a memory catalog item for a single object (non-collection). /// /// Object type /// Unique catalog label for DAG resolution - /// Catalog entry for in-memory singleton + /// Catalog item for in-memory singleton /// /// /// Use Case: Models, charts, computed metrics that stay in memory @@ -71,18 +71,18 @@ public static CatalogEntry Json(string label, string filePath) /// Any singleton data that doesn't need persistence /// /// - public static CatalogEntry Memory(string label) + public static Item Memory(string label) { var storage = new MemoryStorageAdapter(); - return new CatalogEntry(label, storage); + return new Item(label, storage); } /// - /// Creates a plain text file catalog entry. + /// Creates a plain text file catalog item . /// /// Unique catalog label for DAG resolution /// Path to text file (.txt, .md, etc.) - /// Catalog entry for text file with string content + /// Catalog item for text file with string content /// /// /// Use Case: Markdown reports, plain text logs, configuration files @@ -94,18 +94,18 @@ public static CatalogEntry Memory(string label) /// Storage Traits: All traits use filesystem baseline defaults /// /// - public static CatalogEntry Text(string label, string filePath) + public static Item Text(string label, string filePath) { var storage = new TextFileStorageAdapter(filePath); - return new CatalogEntry(label, storage); + return new Item(label, storage); } /// - /// Creates a binary file catalog entry. + /// Creates a binary file catalog item. /// /// Unique catalog label for DAG resolution /// Path to binary file (.png, .jpg, .pdf, etc.) - /// Catalog entry for binary file with byte array content + /// Catalog item for binary file with byte array content /// /// /// Use Case: Images (PNG, JPG), PDFs, any binary data @@ -117,21 +117,21 @@ public static CatalogEntry Text(string label, string filePath) /// Storage Traits: All traits use filesystem baseline defaults /// /// - public static CatalogEntry Binary(string label, string filePath) + public static Item Binary(string label, string filePath) { var storage = new BinaryFileStorageAdapter(filePath); - return new CatalogEntry(label, storage); + return new Item(label, storage); } /// - /// Creates a null catalog entry for side-effect-only nodes. + /// Creates a null catalog item for side-effect-only steps. /// /// The data type (typically NoData) /// Unique catalog label for DAG resolution - /// Catalog entry for void/no-data semantics + /// Catalog item for void/no-data semantics /// /// - /// Use Case: Nodes that perform side effects (logging, visualization) without producing meaningful data + /// Use Case: Steps that perform side effects (logging, visualization) without producing meaningful data /// /// /// Implementation: Uses NullStorageAdapter which performs no I/O operations. @@ -144,10 +144,10 @@ public static CatalogEntry Binary(string label, string filePath) /// CanRead: false (Load throws NotSupportedException) /// /// - public static CatalogEntry Null(string label) + public static Item Null(string label) { var storage = new NullStorageAdapter(); - return new CatalogEntry(label, storage); + return new Item(label, storage); } } } diff --git a/src/core/Flowthru.Core/Data/CatalogEntries.cs b/src/core/Flowthru.Core/Data/CatalogEntries.cs index 9979e01f..7def9b83 100644 --- a/src/core/Flowthru.Core/Data/CatalogEntries.cs +++ b/src/core/Flowthru.Core/Data/CatalogEntries.cs @@ -32,7 +32,7 @@ namespace Flowthru.Data; /// string Name // Looks for "Name" in CSV/Excel/JSON /// ) : IFlatSchema, ITextSerializable; /// -/// var simple = CatalogEntries.Enumerable.Csv<SimpleSchema>("data", "data.csv"); +/// var simple = ItemFactory.Enumerable.Csv<SimpleSchema>("data", "data.csv"); /// /// // Tier 2: Explicit annotations - handle naming mismatches /// public record ShuttleSchema( @@ -49,23 +49,23 @@ namespace Flowthru.Data; /// int CompanyId /// ) : IFlatSchema, ITextSerializable; /// -/// var shuttles = CatalogEntries.Enumerable.Excel<ShuttleSchema>( +/// var shuttles = ItemFactory.Enumerable.Excel<ShuttleSchema>( /// "shuttles", /// "data/shuttles.xlsx", /// "Sheet1" /// ); /// /// // Same schema works across all formats -/// var csv = CatalogEntries.Enumerable.Csv<ShuttleSchema>("shuttles", "data/shuttles.csv"); -/// var json = CatalogEntries.Enumerable.Json<ShuttleSchema>("shuttles", "data/shuttles.json"); +/// var csv = ItemFactory.Enumerable.Csv<ShuttleSchema>("shuttles", "data/shuttles.csv"); +/// var json = ItemFactory.Enumerable.Json<ShuttleSchema>("shuttles", "data/shuttles.json"); /// /// -public static partial class CatalogEntries +public static partial class ItemFactory { /// /// Factory methods for catalog entries. /// - public static EnumerableCatalogEntries Enumerable { get; } = new EnumerableCatalogEntries(); + public static EnumerableItemFactory Enumerable { get; } = new EnumerableItemFactory(); /// /// Creates a null catalog entry for side-effect-only nodes. @@ -75,7 +75,7 @@ public static partial class CatalogEntries /// Catalog entry for void/no-data semantics /// /// - /// Use Case: Nodes that perform side effects (logging, visualization) without producing meaningful data + /// Use Case: Steps that perform side effects (logging, visualization) without producing meaningful data /// /// /// Implementation: Uses NullStorageAdapter which performs no I/O operations. @@ -88,9 +88,9 @@ public static partial class CatalogEntries /// CanRead: false (Load throws NotSupportedException) /// /// - public static CatalogEntry Null(string label) + public static Item Null(string label) { var storage = new NullStorageAdapter(); - return new CatalogEntry(label, storage); + return new Item(label, storage); } } diff --git a/src/core/Flowthru.Core/Data/CatalogEntry.cs b/src/core/Flowthru.Core/Data/CatalogEntry.cs index 45682049..883ac49b 100644 --- a/src/core/Flowthru.Core/Data/CatalogEntry.cs +++ b/src/core/Flowthru.Core/Data/CatalogEntry.cs @@ -6,7 +6,7 @@ namespace Flowthru.Data; /// -/// Standard catalog entry implementation that delegates to a storage adapter. +/// Standard catalog item implementation that delegates to a storage adapter. /// /// The data type (container with rows) /// @@ -16,25 +16,25 @@ namespace Flowthru.Data; /// /// This class is a thin wrapper that delegates all operations to an . /// The storage adapter handles the actual I/O logic, while this class provides: -/// - ICatalogEntry interface implementation +/// - ICatalogItem interface implementation /// - Identity for DAG dependency resolution (via Key) -/// - Type erasure for pipeline heterogeneous collections +/// - Type erasure for Flow heterogeneous collections /// /// /// Construction: /// /// -/// Typically created via static factory methods in : +/// Typically created via static factory methods in : /// /// -/// var entry = CatalogEntries.Csv<CompanySchema>("companies", "data.csv"); -/// // Returns: ICatalogEntry<IEnumerable<CompanySchema>> +/// var item = CatalogItemFactory.Csv<CompanySchema>("companies", "data.csv"); +/// // Returns: ICatalogItem<IEnumerable<CompanySchema>> /// /// /// Composition vs Inheritance: /// /// -/// Previous design: Inheritance hierarchy (CsvCatalogEntry, JsonCatalogEntry, etc.) +/// Previous design: Inheritance hierarchy (CsvCatalogItem, JsonCatalogItem, etc.) /// New design: Single class + composed storage adapter /// /// @@ -48,10 +48,10 @@ namespace Flowthru.Data; /// /// /// The underlying storage adapter provides inspection methods, which this catalog -/// entry automatically forwards. All storage adapters are required to implement inspection. +/// item automatically forwards. All storage adapters are required to implement inspection. /// /// -public sealed class CatalogEntry : ICatalogEntry +public sealed class Item : IItem { private readonly IStorageAdapter _storage; private InspectionLevel? _preferredInspectionLevel; @@ -59,11 +59,11 @@ public sealed class CatalogEntry : ICatalogEntry private string? _owningCatalogLabel; /// - /// Creates a new catalog entry with the specified key and storage adapter. + /// Creates a new catalog item with the specified key and storage adapter. /// - /// Unique identifier for this catalog entry + /// Unique identifier for this catalog item /// Storage adapter that handles I/O operations - public CatalogEntry(string label, IStorageAdapter storage) + public Item(string label, IStorageAdapter storage) { this.Label = label ?? throw new ArgumentNullException(nameof(label)); _storage = storage ?? throw new ArgumentNullException(nameof(storage)); @@ -85,7 +85,7 @@ public CatalogEntry(string label, IStorageAdapter storage) internal void SetOwningCatalog(string label) => _owningCatalogLabel ??= label; /// - /// Gets the effective storage traits for this catalog entry. + /// Gets the effective storage traits for this catalog item. /// /// /// @@ -93,7 +93,7 @@ public CatalogEntry(string label, IStorageAdapter storage) /// applied via . /// /// - /// Used by pipeline validation to enforce read-only constraints, network requirements, etc. + /// Used by Flow validation to enforce read-only constraints, network requirements, etc. /// /// public StorageTraits Traits => _effectiveTraits ?? _storage.Traits; @@ -123,7 +123,7 @@ public FlowIO SaveUntyped(object data) return FlowIO.Fail( new Exception( $"Type mismatch: Cannot save data of type '{data?.GetType().Name ?? "null"}' " - + $"to catalog entry '{Label}' expecting type '{typeof(T).Name}'" + + $"to catalog item '{Label}' expecting type '{typeof(T).Name}'" ) ); } @@ -171,23 +171,23 @@ private static bool IsCollectionType(Type type) } /// - /// Sets the preferred inspection level for this catalog entry. + /// Sets the preferred inspection level for this catalog item. /// /// The inspection level to use - /// This catalog entry for method chaining + /// This catalog item for method chaining /// /// - /// Used to configure how this entry should be validated before pipeline execution. + /// Used to configure how this item should be validated before Flow execution. /// /// /// Example: /// /// - /// var entry = CatalogEntries.Csv<Company>("companies", "data.csv") + /// var item = CatalogItemFactory.Csv<Company>("companies", "data.csv") /// .WithInspectionLevel(InspectionLevel.Deep); /// /// - public CatalogEntry WithInspectionLevel(InspectionLevel level) + public Item WithInspectionLevel(InspectionLevel level) { _preferredInspectionLevel = level; return this; @@ -197,7 +197,7 @@ public CatalogEntry WithInspectionLevel(InspectionLevel level) /// Applies user-specified constraints to the storage traits. /// /// Function that modifies traits to apply constraints - /// This catalog entry for method chaining + /// This catalog item for method chaining /// /// /// One-Way Ratchet: Constraints can only tighten, never loosen. @@ -221,8 +221,8 @@ public CatalogEntry WithInspectionLevel(InspectionLevel level) /// Example: /// /// - /// public ICatalogEntry<IEnumerable<Company>> ReferenceData => - /// GetOrCreateEntry(() => CatalogEntries.Enumerable.Csv<Company>( + /// public ICatalogItem<IEnumerable<Company>> ReferenceData => + /// CreateItem(() => CatalogItemFactory.Enumerable.Csv<Company>( /// "ref_data", $"{_basePath}/reference.csv") /// .Constrain(t => t with { CanWrite = false })); /// @@ -230,7 +230,7 @@ public CatalogEntry WithInspectionLevel(InspectionLevel level) /// /// Thrown when attempting to loosen a constraint (grant a capability the adapter doesn't support) /// - public CatalogEntry Constrain(Func constraintFn) + public Item Constrain(Func constraintFn) { if (constraintFn == null) throw new ArgumentNullException(nameof(constraintFn)); diff --git a/src/core/Flowthru.Core/Data/DataCatalogBase.cs b/src/core/Flowthru.Core/Data/DataCatalogBase.cs index 7b3b22f0..b2a249b3 100644 --- a/src/core/Flowthru.Core/Data/DataCatalogBase.cs +++ b/src/core/Flowthru.Core/Data/DataCatalogBase.cs @@ -10,13 +10,13 @@ namespace Flowthru.Data; /// /// /// Problem Solved: -/// Expression-bodied properties (Property => new Entry()) create new instances on each access, +/// Expression-bodied properties (Property => new Item()) create new instances on each access, /// breaking DAG dependency resolution which relies on object identity. /// /// /// Solution: /// Uses reflection to: -/// 1. Discover all ICatalogEntry properties on derived classes +/// 1. Discover all IItem properties on derived classes /// 2. Create backing fields to cache instances /// 3. Intercept property getters to return cached instances /// @@ -34,35 +34,35 @@ namespace Flowthru.Data; /// protected string BasePath { get; } /// /// // Declare once - automatically cached! -/// public ICatalogEntry<IEnumerable<MyData>> MyData => -/// GetOrCreateEntry(() => new CsvCatalogEntry<MyData>("my_data", $"{BasePath}/data.csv")); +/// public IItem<IEnumerable<MyData>> MyData => +/// CreateItem(() => new CsvCatalogItem<MyData>("my_data", $"{BasePath}/data.csv")); /// } /// /// /// /// Key Benefits: -/// - Declare catalog entries ONCE (no redundant constructor code) +/// - Declare catalog items ONCE (no redundant constructor code) /// - Automatic instance caching (object identity preserved) /// - Type-safe (compile-time checks) /// - Zero runtime overhead after first access (cached delegates) /// /// -public abstract class DataCatalogBase +public abstract class CatalogAbstract { /// /// Cache of property values to ensure object identity for DAG resolution. - /// Key: Property name, Value: Cached ICatalogEntry instance + /// Key: Property name, Value: Cached IItem instance /// - private readonly ConcurrentDictionary _propertyCache = new(); + private readonly ConcurrentDictionary _propertyCache = new(); /// - /// The display label used to identify this catalog instance in pipeline metadata. + /// The display label used to identify this catalog instance in Flow metadata. /// Defaults to the concrete class name when not specified. /// /// /// Pass an explicit label when constructing multiple instances of the same catalog type - /// in a single pipeline (e.g., per-partition or per-shard catalogs) so their entries - /// receive distinct qualified identifiers in the DAG: CatalogLabel.EntryLabel. + /// in a single Flow (e.g., per-partition or per-shard catalogs) so their items + /// receive distinct qualified identifiers in the DAG: CatalogLabel.ItemLabel. /// public string CatalogLabel { get; } @@ -70,85 +70,85 @@ public abstract class DataCatalogBase /// Optional display label for this catalog instance. When omitted, defaults to the /// concrete class name via GetType().Name. /// - protected DataCatalogBase(string? catalogLabel = null) + protected CatalogAbstract(string? catalogLabel = null) { CatalogLabel = catalogLabel ?? GetType().Name; } /// - /// Optional service provider for dependency injection into catalog entries. + /// Optional service provider for dependency injection into catalog items. /// /// - /// Set by the service layer before pipeline execution to enable catalog - /// entries to resolve services (e.g., database connections, HTTP clients). + /// Set by the service layer before Flow execution to enable catalog + /// items to resolve services (e.g., database connections, HTTP clients). /// public IServiceProvider? Services { get; set; } /// - /// Gets or creates a unified catalog entry, caching it for subsequent accesses. + /// Gets or creates a unified catalog item, caching it for subsequent accesses. /// /// - /// The data type stored in this catalog entry. + /// The data type stored in this catalog item. /// For singletons: Use T directly (e.g., LinearRegressionModel) /// For collections: Use IEnumerable<T> (e.g., IEnumerable<FeatureRow>) /// - /// Factory function to create the entry on first access + /// Factory function to create the item on first access /// Auto-populated by compiler with calling property name - /// Cached catalog entry instance + /// Cached catalog item instance /// /// - /// Unified API (v0.5.0): This single method replaces GetOrCreateObject - /// and GetOrCreateDataset. Cardinality is determined by the type parameter T. + /// Unified API (v0.5.0): This single method replaces CreateObject + /// and CreateDataset. Cardinality is determined by the type parameter T. /// /// /// Usage Examples: /// /// // Singleton object - /// public ICatalogEntry<LinearRegressionModel> Model => - /// GetOrCreateEntry(() => CatalogEntries.Single.Memory<LinearRegressionModel>("model")); + /// public IItem<LinearRegressionModel> Model => + /// CreateItem(() => ItemFactory.Single.Memory<LinearRegressionModel>("model")); /// /// // Collection - /// public ICatalogEntry<IEnumerable<FeatureRow>> Features => - /// GetOrCreateEntry(() => CatalogEntries.Enumerable.Csv<FeatureRow>("features", "data.csv")); + /// public IItem<IEnumerable<FeatureRow>> Features => + /// CreateItem(() => ItemFactory.Enumerable.Csv<FeatureRow>("features", "data.csv")); /// /// /// - protected ICatalogEntry GetOrCreateEntry( - Func> factory, + protected IItem CreateItem( + Func> factory, [System.Runtime.CompilerServices.CallerMemberName] string propertyName = "" ) { - var entry = _propertyCache.GetOrAdd(propertyName, _ => factory()); - if (entry is CatalogEntry concrete) + var item = _propertyCache.GetOrAdd(propertyName, _ => factory()); + if (item is Item concrete) concrete.SetOwningCatalog(CatalogLabel); - return (ICatalogEntry)entry; + return (IItem)item; } /// - /// Gets or creates a unified catalog entry with service provider access. + /// Gets or creates a unified catalog item with service provider access. /// /// The data type (singleton or collection) /// Factory function that receives service provider /// Auto-populated by compiler with calling property name - /// Cached catalog entry instance - protected ICatalogEntry GetOrCreateEntry( - Func> factory, + /// Cached catalog item instance + protected IItem CreateItem( + Func> factory, [System.Runtime.CompilerServices.CallerMemberName] string propertyName = "" ) { - var entry = _propertyCache.GetOrAdd(propertyName, _ => factory(Services)); - if (entry is CatalogEntry concrete) + var item = _propertyCache.GetOrAdd(propertyName, _ => factory(Services)); + if (item is Item concrete) concrete.SetOwningCatalog(CatalogLabel); - return (ICatalogEntry)entry; + return (IItem)item; } /// - /// Initializes all catalog entry properties by invoking their getters once. + /// Initializes all catalog item properties by invoking their getters once. /// /// /// - /// Purpose: Eager initialization ensures all entries are cached - /// before pipeline construction begins, preventing any potential race conditions + /// Purpose: Eager initialization ensures all items are cached + /// before Flow construction begins, preventing any potential race conditions /// or unexpected lazy initialization behavior. /// /// @@ -157,7 +157,7 @@ protected ICatalogEntry GetOrCreateEntry( /// /// /// How It Works: - /// Uses reflection to find all public instance properties that return ICatalogEntry, + /// Uses reflection to find all public instance properties that return IItem, /// then invokes each getter once to populate the cache. /// /// @@ -165,7 +165,7 @@ protected void InitializeCatalogProperties() { var catalogProperties = GetType() .GetProperties(BindingFlags.Public | BindingFlags.Instance) - .Where(p => typeof(ICatalogEntry).IsAssignableFrom(p.PropertyType)); + .Where(p => typeof(IItem).IsAssignableFrom(p.PropertyType)); foreach (var property in catalogProperties) { @@ -175,14 +175,14 @@ protected void InitializeCatalogProperties() } /// - /// Gets all cached catalog entries. + /// Gets all cached catalog items. /// - /// Enumerable of all initialized catalog entries + /// Enumerable of all initialized catalog items /// - /// Useful for diagnostic purposes or when you need to iterate over all entries + /// Useful for diagnostic purposes or when you need to iterate over all items /// (e.g., for validation, cleanup, or reporting). /// - protected IEnumerable GetAllEntries() + protected IEnumerable GetAllItemFactory() { return _propertyCache.Values; } @@ -192,7 +192,7 @@ protected IEnumerable GetAllEntries() /// /// /// - /// Warning: Clearing the cache after pipeline construction will break + /// Warning: Clearing the cache after Flow construction will break /// DAG dependencies since new instances will be created on next access. /// /// diff --git a/src/core/Flowthru.Core/Data/EnumerableCatalogEntries.cs b/src/core/Flowthru.Core/Data/EnumerableCatalogEntries.cs index 71bdc343..1ff292fc 100644 --- a/src/core/Flowthru.Core/Data/EnumerableCatalogEntries.cs +++ b/src/core/Flowthru.Core/Data/EnumerableCatalogEntries.cs @@ -7,7 +7,7 @@ namespace Flowthru.Data; /// -/// Extension point for factory methods. +/// Extension point for factory methods. /// /// /// @@ -26,7 +26,7 @@ namespace Flowthru.Data; /// Use Cases: /// /// -/// Standard data processing pipelines (90% of cases) +/// Standard data processing flows (90% of cases) /// Interop with .NET libraries expecting IEnumerable /// LINQ query composition /// Large datasets where you'll enumerate only once @@ -37,17 +37,17 @@ namespace Flowthru.Data; /// register new formats. /// /// -public sealed class EnumerableCatalogEntries +public sealed class EnumerableItemFactory { - internal EnumerableCatalogEntries() { } + internal EnumerableItemFactory() { } /// - /// Creates a JSON file catalog entry with IEnumerable container for collections. + /// Creates a JSON file catalog item with IEnumerable container for collections. /// /// Row schema type (must be structured-serializable) /// Unique catalog label for DAG resolution /// Path to JSON file - /// Catalog entry with file + JSON + IEnumerable composition + /// Catalog item with file + JSON + IEnumerable composition /// /// /// Requirements: @@ -68,7 +68,7 @@ internal EnumerableCatalogEntries() { } /// Serialization: JSON array format for collections /// /// - public CatalogEntry> Json(string label, string filePath) + public Item> Json(string label, string filePath) where TRow : notnull, IStructuredSerializable { var medium = new FileStorageMedium(filePath); @@ -76,18 +76,18 @@ public CatalogEntry> Json(string label, string filePath) var container = new EnumerableContainerAdapter(); var storage = new ComposedStorageAdapter, TRow>(medium, format, container); - return new CatalogEntry>(label, storage); + return new Item>(label, storage); } /// - /// Creates an in-memory transient catalog entry with IEnumerable container. + /// Creates an in-memory transient catalog item with IEnumerable container. /// /// Row schema type /// Unique catalog label for DAG resolution - /// Catalog entry with memory storage (no serialization) + /// Catalog item with memory storage (no serialization) /// /// - /// Use Case: Intermediate pipeline data that doesn't need persistence + /// Use Case: Intermediate Flow data that doesn't need persistence /// /// /// Storage Traits: @@ -96,9 +96,9 @@ public CatalogEntry> Json(string label, string filePath) /// IsPersistent: false (data lost when process ends) /// /// - public CatalogEntry> Memory(string label) + public Item> Memory(string label) { var storage = new MemoryStorageAdapter>(); - return new CatalogEntry>(label, storage); + return new Item>(label, storage); } } diff --git a/src/core/Flowthru.Core/Data/ICatalogEntry.cs b/src/core/Flowthru.Core/Data/ICatalogEntry.cs index d9b93fde..91eacccf 100644 --- a/src/core/Flowthru.Core/Data/ICatalogEntry.cs +++ b/src/core/Flowthru.Core/Data/ICatalogEntry.cs @@ -4,90 +4,90 @@ namespace Flowthru.Data; /// -/// Non-generic base interface for catalog entries. -/// Provides untyped operations for internal use by the pipeline executor and mapping layer. +/// Non-generic base interface for catalog items. +/// Provides untyped operations for internal use by the Flow executor and mapping layer. /// /// -/// This interface enables the pipeline to work with catalog entries +/// This interface enables the Flow to work with catalog items /// without knowing their specific type parameter at compile-time. /// -public interface ICatalogEntry +public interface IItem { /// - /// Unique label identifying this catalog entry within the data catalog. + /// Unique label identifying this catalog item within the data catalog. /// string Label { get; } /// - /// The runtime type of data stored in this catalog entry. + /// The runtime type of data stored in this catalog item. /// For singletons: Returns typeof(T). /// For collections: Returns typeof(IEnumerable<T>). /// Type DataType { get; } /// - /// Gets the preferred inspection level for this catalog entry. + /// Gets the preferred inspection level for this catalog item. /// InspectionLevel? PreferredInspectionLevel { get; } /// - /// The label of the -derived class that created - /// this entry. Set automatically by GetOrCreateEntry; null for entries created outside - /// a catalog or by custom implementations. + /// The label of the -derived class that created + /// this item. Set automatically by CreateItem; null for items created outside + /// a catalog or by custom implementations. /// /// - /// Used by the metadata layer to produce fully-qualified entry identifiers in the form - /// CatalogLabel.EntryLabel. First-write-wins: cross-catalog shared entries retain + /// Used by the metadata layer to produce fully-qualified item identifiers in the form + /// CatalogLabel.ItemLabel. First-write-wins: cross-catalog shared items retain /// the label of the catalog that originally created them. /// string? OwningCatalogLabel => null; /// - /// Loads data from the catalog entry as an untyped object. + /// Loads data from the catalog item as an untyped object. /// Returns an effect that can fail. /// The returned type matches the DataType property. /// FlowIO LoadUntyped(); /// - /// Saves untyped data to the catalog entry. + /// Saves untyped data to the catalog item. /// Returns an effect that can fail. /// The data type must be compatible with the DataType property. /// FlowIO SaveUntyped(object data); /// - /// Checks if data exists at this catalog entry location. + /// Checks if data exists at this catalog item location. /// Returns an effect that can fail. /// FlowIO Exists(); /// - /// Gets the count of items in this catalog entry. + /// Gets the count of items in this catalog item. /// For collections (IEnumerable<T>), returns the enumerable count. /// For singletons, returns 1 if exists, 0 otherwise. /// FlowIO GetCountAsync(); /// - /// Performs shallow validation of this catalog entry. + /// Performs shallow validation of this catalog item. /// /// Number of rows/records to sample for validation /// Effect producing validation result FlowIO InspectShallow(int sampleSize = 100); /// - /// Performs deep validation of this catalog entry. + /// Performs deep validation of this catalog item. /// /// Effect producing validation result FlowIO InspectDeep(); } /// -/// Unified catalog entry with cardinality encoded in the type parameter. +/// Unified catalog item with cardinality encoded in the type parameter. /// /// -/// The data type stored in this catalog entry. +/// The data type stored in this catalog item. /// Cardinality is determined by T itself: /// - For singletons: Use T directly (e.g., LinearRegressionModel, ModelMetrics) /// - For collections: Use IEnumerable<T> (e.g., IEnumerable<FeatureRow>) @@ -98,7 +98,7 @@ public interface ICatalogEntry /// system (ICatalogObject/ICatalogDataset). Cardinality is now purely a type-level concern. /// /// -/// Type Alignment: Node TInput/TOutput types should directly match catalog entry +/// Type Alignment: Step TInput/TOutput types should directly match catalog item /// T types, eliminating the need for wrapping/unwrapping ceremony. /// /// @@ -109,7 +109,7 @@ public interface ICatalogEntry /// - Functional composition /// /// -public interface ICatalogEntry : ICatalogEntry +public interface IItem : IItem { /// /// Load data as an effect (can fail, is async, can be cancelled). diff --git a/src/core/Flowthru.Core/Data/Storage/Format/JsonFormatSerializer.cs b/src/core/Flowthru.Core/Data/Storage/Format/JsonFormatSerializer.cs index a5e41fdc..44a17798 100644 --- a/src/core/Flowthru.Core/Data/Storage/Format/JsonFormatSerializer.cs +++ b/src/core/Flowthru.Core/Data/Storage/Format/JsonFormatSerializer.cs @@ -411,7 +411,7 @@ internal sealed class SerializedEnumJsonConverter : JsonConverter public SerializedEnumJsonConverter() { - _metadata = Serialization.EnumMetadataRegistry.GetOrCreate(); + _metadata = Serialization.EnumMetadataRegistry.Create(); } public override TEnum Read( diff --git a/src/core/Flowthru.Core/Data/Storage/IStorageAdapter.cs b/src/core/Flowthru.Core/Data/Storage/IStorageAdapter.cs index 897e3d90..0acd29e0 100644 --- a/src/core/Flowthru.Core/Data/Storage/IStorageAdapter.cs +++ b/src/core/Flowthru.Core/Data/Storage/IStorageAdapter.cs @@ -41,10 +41,10 @@ namespace Flowthru.Data.Storage; /// Usage in Catalog Entries: /// /// -/// delegates to this interface: +/// delegates to this interface: /// /// -/// public class CatalogEntry<T> : ICatalogEntry<T> +/// public class Item<T> : IItem<T> /// { /// private readonly IStorageAdapter<T> _storage; /// diff --git a/src/core/Flowthru.Core/Data/Storage/IStorageMedium.cs b/src/core/Flowthru.Core/Data/Storage/IStorageMedium.cs index 2b50e3e6..c353d3e6 100644 --- a/src/core/Flowthru.Core/Data/Storage/IStorageMedium.cs +++ b/src/core/Flowthru.Core/Data/Storage/IStorageMedium.cs @@ -118,7 +118,7 @@ public interface IStorageMedium /// /// /// This is used to determine if a catalog entry is a "seed" (Layer 0 input) - /// or if it's produced by a node in the pipeline. + /// or if it's produced by a step in the pipeline. /// /// FlowIO Exists(); diff --git a/src/core/Flowthru.Core/Data/Storage/MemoryStorageAdapter.cs b/src/core/Flowthru.Core/Data/Storage/MemoryStorageAdapter.cs index 901920b7..be5c2515 100644 --- a/src/core/Flowthru.Core/Data/Storage/MemoryStorageAdapter.cs +++ b/src/core/Flowthru.Core/Data/Storage/MemoryStorageAdapter.cs @@ -40,11 +40,11 @@ namespace Flowthru.Data.Storage; /// /// // Singleton usage /// var modelStorage = new MemoryStorageAdapter<LinearRegressionModel>(); -/// var modelEntry = new CatalogEntry<LinearRegressionModel>("model", modelStorage); +/// var modelEntry = new Item<LinearRegressionModel>("model", modelStorage); /// /// // Collection usage /// var dataStorage = new MemoryStorageAdapter<IEnumerable<FeatureRow>>(); -/// var dataEntry = new CatalogEntry<IEnumerable<FeatureRow>>("features", dataStorage); +/// var dataEntry = new Item<IEnumerable<FeatureRow>>("features", dataStorage); /// /// public sealed class MemoryStorageAdapter : IStorageAdapter diff --git a/src/core/Flowthru.Core/Data/Storage/NullStorageAdapter.cs b/src/core/Flowthru.Core/Data/Storage/NullStorageAdapter.cs index 3a2c4085..2a011b2c 100644 --- a/src/core/Flowthru.Core/Data/Storage/NullStorageAdapter.cs +++ b/src/core/Flowthru.Core/Data/Storage/NullStorageAdapter.cs @@ -9,7 +9,7 @@ namespace Flowthru.Data.Storage; /// The data type (typically NoData) /// /// -/// Use Case: Nodes that perform side effects (logging, visualization, alerts) +/// Use Case: Steps that perform side effects (logging, visualization, alerts) /// but don't produce data that downstream nodes need. /// /// diff --git a/src/core/Flowthru.Core/Data/Storage/SchemaActivator.cs b/src/core/Flowthru.Core/Data/Storage/SchemaActivator.cs index eb105c45..c61ff7e6 100644 --- a/src/core/Flowthru.Core/Data/Storage/SchemaActivator.cs +++ b/src/core/Flowthru.Core/Data/Storage/SchemaActivator.cs @@ -13,16 +13,16 @@ namespace Flowthru.Data.Storage; /// Design Philosophy: /// /// -/// With Flowthru's strong node contracts, schemas with required members are guaranteed to contain +/// With Flowthru's strong step contracts, schemas with required members are guaranteed to contain /// valid data because: /// /// /// Layer 0 (Seeds): Validation phase checks required fields exist before execution -/// Layers 1+ (Node outputs): C# compiler enforces required members when nodes construct output +/// Layers 1+ (Step outputs): C# compiler enforces required members when steps construct output /// /// /// This activator's role is to enable deserialization by creating instances that will be populated -/// via property reflection. No validation is performed here - that happens at the pipeline boundaries. +/// via property reflection. No validation is performed here - that happens at the Flow boundaries. /// /// /// Instantiation Strategy: @@ -82,7 +82,7 @@ public static class SchemaActivator /// /// This is safe because: /// - Layer 0: Validation ensures required fields exist in data - /// - Layers 1+: Data came from valid node output + /// - Layers 1+: Data came from valid step output /// /// public static T CreateInstance() @@ -171,7 +171,7 @@ private static Func CompileFactory(Type type) /// This is safe in Flowthru because properties will be populated immediately /// via reflection from deserialized data, which is guaranteed to contain all /// required fields (via validation phase for Layer 0, or compiler enforcement - /// for node outputs). + /// for step outputs). /// /// private static T CreateUninitializedObject() diff --git a/src/core/Flowthru.Core/Data/Storage/SingletonJsonStorageAdapter.cs b/src/core/Flowthru.Core/Data/Storage/SingletonJsonStorageAdapter.cs index 1f6b5aa4..c0cea28b 100644 --- a/src/core/Flowthru.Core/Data/Storage/SingletonJsonStorageAdapter.cs +++ b/src/core/Flowthru.Core/Data/Storage/SingletonJsonStorageAdapter.cs @@ -34,7 +34,7 @@ namespace Flowthru.Data.Storage; /// /// /// var storage = new SingletonJsonStorageAdapter<LinearRegressionModel>("model.json"); -/// var entry = new CatalogEntry<LinearRegressionModel>("model", storage); +/// var entry = new Item<LinearRegressionModel>("model", storage); /// /// // Save /// await entry.Save(model).RunAsync(); diff --git a/src/core/Flowthru.Core/Data/Storage/Strategies/DatabaseStorageEntryFactory.cs b/src/core/Flowthru.Core/Data/Storage/Strategies/DatabaseStorageEntryFactory.cs index 4627a00b..bfe21ce9 100644 --- a/src/core/Flowthru.Core/Data/Storage/Strategies/DatabaseStorageEntryFactory.cs +++ b/src/core/Flowthru.Core/Data/Storage/Strategies/DatabaseStorageEntryFactory.cs @@ -29,7 +29,7 @@ namespace Flowthru.Data.Storage.Strategies; /// /// services.AddFlowthru(flowthru => /// { -/// flowthru.UseCatalog<MyCatalog>(); +/// flowthru.RegisterCatalog<MyCatalog>(); /// /// if (env.IsProduction()) /// { @@ -76,10 +76,7 @@ public DatabaseStorageEntryFactory(string connectionString, string schema = "dbo /// /// Phase 2 stub - database support not yet implemented /// - public ICatalogEntry> CreateEnumerable( - string label, - StorageOptions? options = null - ) + public IItem> CreateEnumerable(string label, StorageOptions? options = null) where T : notnull, IFlatSchema, ITextSerializable { throw new NotImplementedException( @@ -93,7 +90,7 @@ public ICatalogEntry> CreateEnumerable( // var medium = new DatabaseStorageMedium(_connectionString, tableName); // var format = new SqlFormatSerializer(); // var container = new EnumerableContainerAdapter(); - // return new CatalogEntry>( + // return new Item>( // label, // new ComposedStorageAdapter, T>(medium, format, container) // ); @@ -103,7 +100,7 @@ public ICatalogEntry> CreateEnumerable( /// /// Phase 2 stub - database support not yet implemented /// - public ICatalogEntry CreateSingle(string label, StorageOptions? options = null) + public IItem CreateSingle(string label, StorageOptions? options = null) where T : IStructuredSerializable { throw new NotImplementedException( diff --git a/src/core/Flowthru.Core/Data/Storage/Strategies/IStorageEntryFactory.cs b/src/core/Flowthru.Core/Data/Storage/Strategies/IStorageEntryFactory.cs index e1472556..32c71724 100644 --- a/src/core/Flowthru.Core/Data/Storage/Strategies/IStorageEntryFactory.cs +++ b/src/core/Flowthru.Core/Data/Storage/Strategies/IStorageEntryFactory.cs @@ -29,8 +29,8 @@ namespace Flowthru.Data.Storage.Strategies; /// InitializeCatalogProperties(); /// } /// -/// public ICatalogEntry<IEnumerable<Company>> Companies => -/// GetOrCreateEntry(() => _storage.CreateEnumerable<Company>("Companies")); +/// public IItem<IEnumerable<Company>> Companies => +/// CreateEntry(() => _storage.CreateEnumerable<Company>("Companies")); /// } /// /// @@ -53,7 +53,7 @@ public interface IStorageEntryFactory /// (e.g., "Companies" → "Companies.csv" or "dbo.Companies"). /// /// - ICatalogEntry> CreateEnumerable(string label, StorageOptions? options = null) + IItem> CreateEnumerable(string label, StorageOptions? options = null) where T : notnull, IFlatSchema, ITextSerializable; /// @@ -72,6 +72,6 @@ ICatalogEntry> CreateEnumerable(string label, StorageOptions? /// Typically uses structured formats (JSON, MessagePack) for singletons. /// /// - ICatalogEntry CreateSingle(string label, StorageOptions? options = null) + IItem CreateSingle(string label, StorageOptions? options = null) where T : IStructuredSerializable; } diff --git a/src/core/Flowthru.Core/Data/Storage/Strategies/MemoryStorageEntryFactory.cs b/src/core/Flowthru.Core/Data/Storage/Strategies/MemoryStorageEntryFactory.cs index 4b5bb0f2..768e8953 100644 --- a/src/core/Flowthru.Core/Data/Storage/Strategies/MemoryStorageEntryFactory.cs +++ b/src/core/Flowthru.Core/Data/Storage/Strategies/MemoryStorageEntryFactory.cs @@ -33,19 +33,16 @@ namespace Flowthru.Data.Storage.Strategies; public sealed class MemoryStorageEntryFactory : IStorageEntryFactory { /// - public ICatalogEntry> CreateEnumerable( - string label, - StorageOptions? options = null - ) + public IItem> CreateEnumerable(string label, StorageOptions? options = null) where T : notnull, IFlatSchema, ITextSerializable { - return CatalogEntries.Enumerable.Memory(label); + return ItemFactory.Enumerable.Memory(label); } /// - public ICatalogEntry CreateSingle(string label, StorageOptions? options = null) + public IItem CreateSingle(string label, StorageOptions? options = null) where T : IStructuredSerializable { - return CatalogEntries.Single.Memory(label); + return ItemFactory.Single.Memory(label); } } diff --git a/src/core/Flowthru.Core/Pipelines/DependencyAnalyzer.cs b/src/core/Flowthru.Core/Flows/DependencyAnalyzer.cs similarity index 76% rename from src/core/Flowthru.Core/Pipelines/DependencyAnalyzer.cs rename to src/core/Flowthru.Core/Flows/DependencyAnalyzer.cs index 46cc0b19..5cc34427 100644 --- a/src/core/Flowthru.Core/Pipelines/DependencyAnalyzer.cs +++ b/src/core/Flowthru.Core/Flows/DependencyAnalyzer.cs @@ -1,6 +1,6 @@ using Flowthru.Data; -namespace Flowthru.Pipelines; +namespace Flowthru.Flows; /// /// Analyzes pipeline node dependencies and performs topological sort to determine execution order. @@ -18,8 +18,8 @@ namespace Flowthru.Pipelines; /// /// /// Layer Assignment: -/// - Layer 0: Nodes with no dependencies (read only external data) -/// - Layer N: Nodes whose dependencies are all in layers 0..N-1 +/// - Layer 0: Steps with no dependencies (read only external data) +/// - Layer N: Steps whose dependencies are all in layers 0..N-1 /// /// /// Pipeline Slicing: @@ -43,7 +43,7 @@ internal static class DependencyAnalyzer /// This method combines BuildDependencyGraph and AssignLayers for convenience. /// For sliced pipelines, call these methods separately to recalculate layers post-slice. /// - public static void AnalyzeAndAssignLayers(List nodes) + public static void AnalyzeAndAssignLayers(List nodes) { BuildDependencyGraph(nodes); AssignLayers(nodes); @@ -60,7 +60,7 @@ public static void AnalyzeAndAssignLayers(List nodes) /// This phase must occur before slicing, as the slicing logic traverses node dependencies /// to determine which nodes to include. Layer assignment should happen separately after slicing. /// - public static void BuildDependencyGraph(List nodes) + public static void BuildDependencyGraph(List nodes) { // Step 1: Build producer map (catalog entry → node that produces it) var producerMap = BuildProducerMap(nodes); @@ -77,9 +77,9 @@ public static void BuildDependencyGraph(List nodes) /// /// Thrown if multiple nodes write to the same catalog entry /// - private static Dictionary BuildProducerMap(List nodes) + private static Dictionary BuildProducerMap(List nodes) { - var producerMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + var producerMap = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var node in nodes) { @@ -107,8 +107,8 @@ private static Dictionary BuildProducerMap(ListAll nodes in the pipeline /// Map of catalog entry labels to their producer nodes private static void ResolveDependencies( - List nodes, - Dictionary producerMap + List nodes, + Dictionary producerMap ) { foreach (var node in nodes) @@ -132,22 +132,22 @@ Dictionary producerMap /// /// Assigns execution layers to nodes via topological sort. /// - /// Nodes to assign layers to (full pipeline or sliced subset) + /// Steps to assign layers to (full pipeline or sliced subset) /// Thrown if a circular dependency is detected /// /// This method should be called after slicing to ensure Layer 0 correctly identifies /// nodes with no dependencies in the execution context. /// - public static void AssignLayers(List nodes) + public static void AssignLayers(List nodes) { // Track which nodes have been assigned layers - var assigned = new HashSet(); + var assigned = new HashSet(); var currentLayer = 0; // Keep assigning layers until all nodes are processed while (assigned.Count < nodes.Count) { - var nodesInCurrentLayer = new List(); + var nodesInCurrentLayer = new List(); // Find nodes whose dependencies are all already assigned foreach (var node in nodes) @@ -170,10 +170,10 @@ public static void AssignLayers(List nodes) // If no nodes were assigned this iteration, we have a cycle if (nodesInCurrentLayer.Count == 0) { - var unassignedNodes = nodes.Where(n => !assigned.Contains(n)).Select(n => n.Label); + var unassignedSteps = nodes.Where(n => !assigned.Contains(n)).Select(n => n.Label); throw new InvalidOperationException( $"Circular dependency detected in pipeline. " - + $"Unassigned nodes: {string.Join(", ", unassignedNodes)}" + + $"Unassigned nodes: {string.Join(", ", unassignedSteps)}" ); } @@ -191,8 +191,8 @@ public static void AssignLayers(List nodes) /// Groups nodes by their assigned execution layer. /// /// All nodes in the pipeline (must have layers assigned) - /// Nodes grouped by layer, ordered by layer number - public static IEnumerable> GroupByLayer(List nodes) + /// Steps grouped by layer, ordered by layer number + public static IEnumerable> GroupByLayer(List nodes) { return nodes.GroupBy(n => n.Layer).OrderBy(g => g.Key).Select(g => g.ToList()); } @@ -200,7 +200,7 @@ public static IEnumerable> GroupByLayer(List no /// /// Slices a pipeline to include only nodes matching the specified strategy. /// - /// All nodes in the pipeline + /// All nodes in the pipeline /// The slicing strategy to apply /// Filtered list of nodes forming a valid sub-DAG /// @@ -221,31 +221,28 @@ public static IEnumerable> GroupByLayer(List no /// sub-DAG that can execute without missing dependencies. /// /// - public static List SliceNodes( - List allNodes, - PipelineSliceStrategy strategy - ) + public static List SliceSteps(List allSteps, FlowSliceStrategy strategy) { if (!strategy.IsSliced) { - return allNodes; + return allSteps; } // Dependencies are already resolved by Pipeline.Build() before slicing // No need to call BuildProducerMap/ResolveDependencies here - var nodesByLabel = allNodes.ToDictionary(n => n.Label, StringComparer.OrdinalIgnoreCase); - var selectedNodes = new HashSet(allNodes); + var nodesByLabel = allSteps.ToDictionary(n => n.Label, StringComparer.OrdinalIgnoreCase); + var selectedSteps = new HashSet(allSteps); // Step 1: Apply pipeline filter (if specified, for merged pipelines) - if (strategy.Pipelines is { Count: > 0 }) + if (strategy.Flows is { Count: > 0 }) { - var pipelineFilter = new HashSet(); + var pipelineFilter = new HashSet(); - foreach (var pipelineName in strategy.Pipelines) + foreach (var pipelineName in strategy.Flows) { - // Find nodes that belong to this pipeline (prefix match: "PipelineName.NodeName") - var pipelineNodes = allNodes.Where(n => + // Find nodes that belong to this pipeline (prefix match: "FlowName.StepName") + var pipelineSteps = allSteps.Where(n => { var dotIndex = n.Label.IndexOf('.'); if (dotIndex <= 0) @@ -253,27 +250,27 @@ PipelineSliceStrategy strategy return false; // Not a merged pipeline node } - var nodePipelineName = n.Label.Substring(0, dotIndex); - return nodePipelineName.Equals(pipelineName, StringComparison.OrdinalIgnoreCase); + var nodeFlowName = n.Label.Substring(0, dotIndex); + return nodeFlowName.Equals(pipelineName, StringComparison.OrdinalIgnoreCase); }); - pipelineFilter.UnionWith(pipelineNodes); + pipelineFilter.UnionWith(pipelineSteps); } if (pipelineFilter.Count == 0) { throw new InvalidOperationException( - $"Pipelines filter did not match any nodes. Specified: {string.Join(", ", strategy.Pipelines)}" + $"Pipelines filter did not match any nodes. Specified: {string.Join(", ", strategy.Flows)}" ); } - selectedNodes.IntersectWith(pipelineFilter); + selectedSteps.IntersectWith(pipelineFilter); } // Step 2: Apply OnlyNodes filter (explicit allowlist + dependencies) if (strategy.OnlyNodes is { Count: > 0 }) { - var explicitNodes = new HashSet(); + var explicitSteps = new HashSet(); foreach (var nodeName in strategy.OnlyNodes) { if (!nodesByLabel.TryGetValue(nodeName, out var node)) @@ -282,33 +279,33 @@ PipelineSliceStrategy strategy $"OnlyNodes references non-existent node: '{nodeName}'" ); } - explicitNodes.Add(node); + explicitSteps.Add(node); } // Include all upstream dependencies to maintain runnability - var withDependencies = ExpandUpstream(explicitNodes); - selectedNodes.IntersectWith(withDependencies); + var withDependencies = ExpandUpstream(explicitSteps); + selectedSteps.IntersectWith(withDependencies); } // Step 3: Apply FromData (find consumers, expand downstream) - var fromNodesExpanded = new HashSet(); + var fromStepsExpanded = new HashSet(); if (strategy.FromData is { Count: > 0 }) { // Find nodes that consume any of the specified catalog entries foreach (var dataLabel in strategy.FromData) { - var consumingNodes = allNodes.Where(n => + var consumingSteps = allSteps.Where(n => n.Inputs.Any(entry => entry.Label.Equals(dataLabel, StringComparison.OrdinalIgnoreCase)) ); - if (!consumingNodes.Any()) + if (!consumingSteps.Any()) { throw new InvalidOperationException( $"FromData references catalog entry '{dataLabel}' which is not consumed by any node" ); } - fromNodesExpanded.UnionWith(consumingNodes); + fromStepsExpanded.UnionWith(consumingSteps); } } @@ -323,35 +320,35 @@ PipelineSliceStrategy strategy $"FromNodes references non-existent node: '{nodeName}'" ); } - fromNodesExpanded.Add(node); + fromStepsExpanded.Add(node); } } - if (fromNodesExpanded.Count > 0) + if (fromStepsExpanded.Count > 0) { - var withDownstream = ExpandDownstream(fromNodesExpanded, allNodes); - selectedNodes.IntersectWith(withDownstream); + var withDownstream = ExpandDownstream(fromStepsExpanded, allSteps); + selectedSteps.IntersectWith(withDownstream); } // Step 5: Apply ToData (find producers, expand upstream) - var toNodesExpanded = new HashSet(); + var toStepsExpanded = new HashSet(); if (strategy.ToData is { Count: > 0 }) { // Find nodes that produce any of the specified catalog entries foreach (var dataLabel in strategy.ToData) { - var producingNode = allNodes.FirstOrDefault(n => + var producingStep = allSteps.FirstOrDefault(n => n.Outputs.Any(entry => entry.Label.Equals(dataLabel, StringComparison.OrdinalIgnoreCase)) ); - if (producingNode == null) + if (producingStep == null) { throw new InvalidOperationException( $"ToData references catalog entry '{dataLabel}' which is not produced by any node" ); } - toNodesExpanded.Add(producingNode); + toStepsExpanded.Add(producingStep); } } @@ -366,21 +363,21 @@ PipelineSliceStrategy strategy $"ToNodes references non-existent node: '{nodeName}'" ); } - toNodesExpanded.Add(node); + toStepsExpanded.Add(node); } } - if (toNodesExpanded.Count > 0) + if (toStepsExpanded.Count > 0) { - var withUpstream = ExpandUpstream(toNodesExpanded); - selectedNodes.IntersectWith(withUpstream); + var withUpstream = ExpandUpstream(toStepsExpanded); + selectedSteps.IntersectWith(withUpstream); } - var slicedList = selectedNodes.ToList(); + var slicedList = selectedSteps.ToList(); // Filter each node's dependencies to only include nodes in the sliced set // Dependencies pointing outside the slice become external inputs in the sliced context - var slicedSet = new HashSet(slicedList); + var slicedSet = new HashSet(slicedList); foreach (var node in slicedList) { node.Dependencies.RemoveAll(dep => !slicedSet.Contains(dep)); @@ -392,10 +389,10 @@ PipelineSliceStrategy strategy /// /// Expands a set of nodes to include all upstream dependencies (transitive closure). /// - private static HashSet ExpandUpstream(HashSet nodes) + private static HashSet ExpandUpstream(HashSet nodes) { - var result = new HashSet(); - var toVisit = new Queue(nodes); + var result = new HashSet(); + var toVisit = new Queue(nodes); while (toVisit.Count > 0) { @@ -415,15 +412,15 @@ private static HashSet ExpandUpstream(HashSet nodes) /// /// Expands a set of nodes to include all downstream dependents (transitive closure). /// - private static HashSet ExpandDownstream( - HashSet nodes, - List allNodes + private static HashSet ExpandDownstream( + HashSet nodes, + List allSteps ) { - var result = new HashSet(nodes); - var dependencyMap = BuildDependencyMap(allNodes); + var result = new HashSet(nodes); + var dependencyMap = BuildDependencyMap(allSteps); - var toVisit = new Queue(nodes); + var toVisit = new Queue(nodes); while (toVisit.Count > 0) { var current = toVisit.Dequeue(); @@ -445,19 +442,17 @@ List allNodes /// /// Builds a reverse dependency map (node → nodes that depend on it). /// - private static Dictionary> BuildDependencyMap( - List allNodes - ) + private static Dictionary> BuildDependencyMap(List allSteps) { - var map = new Dictionary>(); + var map = new Dictionary>(); - foreach (var node in allNodes) + foreach (var node in allSteps) { foreach (var dependency in node.Dependencies) { if (!map.ContainsKey(dependency)) { - map[dependency] = new List(); + map[dependency] = new List(); } map[dependency].Add(node); } diff --git a/src/core/Flowthru.Core/Pipelines/DryRunOption.cs b/src/core/Flowthru.Core/Flows/DryRunOption.cs similarity index 97% rename from src/core/Flowthru.Core/Pipelines/DryRunOption.cs rename to src/core/Flowthru.Core/Flows/DryRunOption.cs index 8851eb43..f9a83ad7 100644 --- a/src/core/Flowthru.Core/Pipelines/DryRunOption.cs +++ b/src/core/Flowthru.Core/Flows/DryRunOption.cs @@ -1,4 +1,4 @@ -namespace Flowthru.Pipelines; +namespace Flowthru.Flows; /// /// Represents a dry-run configuration. Can be assigned from a diff --git a/src/core/Flowthru.Core/Pipelines/ExecutionOptions.cs b/src/core/Flowthru.Core/Flows/ExecutionOptions.cs similarity index 91% rename from src/core/Flowthru.Core/Pipelines/ExecutionOptions.cs rename to src/core/Flowthru.Core/Flows/ExecutionOptions.cs index 8cc29aee..c5640453 100644 --- a/src/core/Flowthru.Core/Pipelines/ExecutionOptions.cs +++ b/src/core/Flowthru.Core/Flows/ExecutionOptions.cs @@ -1,6 +1,6 @@ using Flowthru.Results; -namespace Flowthru.Pipelines; +namespace Flowthru.Flows; /// /// Configuration options for pipeline execution. @@ -47,7 +47,7 @@ public class ExecutionOptions /// /// Defaults to ConsoleResultFormatter if not specified. /// - public IPipelineResultFormatter? ResultFormatter { get; set; } + public IFlowResultFormatter? ResultFormatter { get; set; } /// /// Optional slicing strategy to apply when executing pipelines. @@ -56,13 +56,13 @@ public class ExecutionOptions /// When provided, only nodes matching the slice strategy will be executed. /// Used when slicing flags are provided without a specific pipeline name. /// - public PipelineSliceStrategy? SliceStrategy { get; set; } + public FlowSliceStrategy? SliceStrategy { get; set; } /// /// Gets the configured formatter or creates a default one. /// /// The result formatter to use - internal IPipelineResultFormatter GetFormatter() + internal IFlowResultFormatter GetFormatter() { return ResultFormatter ?? new ConsoleResultFormatter(); } diff --git a/src/core/Flowthru.Core/Pipelines/Pipeline.cs b/src/core/Flowthru.Core/Flows/Pipeline.cs similarity index 62% rename from src/core/Flowthru.Core/Pipelines/Pipeline.cs rename to src/core/Flowthru.Core/Flows/Pipeline.cs index 34613ae5..b6009fa9 100644 --- a/src/core/Flowthru.Core/Pipelines/Pipeline.cs +++ b/src/core/Flowthru.Core/Flows/Pipeline.cs @@ -5,104 +5,104 @@ using Flowthru.Meta.Models; using Microsoft.Extensions.Logging; -namespace Flowthru.Pipelines; +namespace Flowthru.Flows; /// -/// Represents a complete data pipeline with nodes, dependencies, and execution order. +/// Represents a complete data Flow with steps, dependencies, and execution order. /// /// /// -/// A pipeline is a directed acyclic graph (DAG) of transformation nodes. -/// Each node reads data from catalog entries, performs transformations, +/// A Flow is a directed acyclic graph (DAG) of transformation steps. +/// Each step reads data from catalog entries, performs transformations, /// and writes results back to catalog entries. /// /// /// Execution Model: /// /// -/// Nodes are organized into layers via topological sort -/// Nodes in layer 0 have no dependencies (read external data only) -/// Nodes in layer N depend only on nodes in layers 0..N-1 -/// Sequential execution: Execute all nodes in layer order -/// Parallel execution (Phase 2): Execute nodes within same layer concurrently +/// Steps are organized into layers via topological sort +/// Steps in layer 0 have no dependencies (read external data only) +/// Steps in layer N depend only on steps in layers 0..N-1 +/// Sequential execution: Execute all steps in layer order +/// Parallel execution (Phase 2): Execute steps within same layer concurrently /// /// /// Single Producer Rule: Each catalog entry can be written by at most -/// one node. This ensures deterministic execution order and prevents race conditions. +/// one step. This ensures deterministic execution order and prevents race conditions. /// /// -public class Pipeline +public class Flow { /// - /// All nodes in this pipeline, in the order they were added. + /// All steps in this flow, in the order they were added. /// /// - /// Exposed as public to enable validation hooks (Phase 4) to inspect nodes. - /// The collection is read-only - nodes can only be added via PipelineBuilder. + /// Exposed as public to enable validation hooks (Phase 4) to inspect steps. + /// The collection is read-only - steps can only be added via FlowBuilder. /// - public IReadOnlyList Nodes => _nodes.AsReadOnly(); + public IReadOnlyList Steps => _steps.AsReadOnly(); /// - /// Internal accessor for the mutable node list. Used by pipeline internals. + /// Internal accessor for the mutable step list. Used by Flow internals. /// - internal List NodesList => _nodes; + internal List StepsList => _steps; - private readonly List _nodes = new(); + private readonly List _steps = new(); /// - /// Subset of nodes to execute after slicing is applied. - /// Null if no slicing was applied (execute all nodes). + /// Subset of steps to execute after slicing is applied. + /// Null if no slicing was applied (execute all steps). /// - private List? _slicedNodes; + private List? _slicedSteps; /// - /// Nodes grouped by execution layer. + /// Steps grouped by execution layer. /// Populated after Build() is called. /// - internal IReadOnlyList>? ExecutionLayers { get; private set; } + internal IReadOnlyList>? ExecutionLayers { get; private set; } /// /// The slice strategy applied during the most recent Build() call, if any. /// /// /// Cached to enable metadata export to include slice criteria. - /// Null if pipeline was built without slicing. + /// Null if Flow was built without slicing. /// - internal PipelineSliceStrategy? AppliedSlice { get; private set; } + internal FlowSliceStrategy? AppliedSlice { get; private set; } /// - /// Optional logger for pipeline execution. + /// Optional logger for Flow execution. /// public ILogger? Logger { get; set; } /// - /// Optional service provider for dependency injection into nodes. + /// Optional service provider for dependency injection into steps. /// /// - /// Set by the service layer before pipeline execution to enable nodes + /// Set by the service layer before Flow execution to enable steps /// to resolve services (e.g., database connections, external APIs). /// public IServiceProvider? ServiceProvider { get; set; } /// - /// Pipeline name for identification and logging. + /// Flow name for identification and logging. /// /// - /// Set by PipelineRegistry during pipeline registration. + /// Set by FlowRegistry during Flow registration. /// public string? Name { get; internal set; } /// - /// Optional description of what this pipeline does. + /// Optional description of what this Flow does. /// public string? Description { get; internal set; } /// - /// Validation options for this pipeline. + /// Validation options for this flow. /// /// /// Configures how external data sources (Layer 0 inputs) are validated - /// before pipeline execution begins. + /// before Flow execution begins. /// public Validation.ValidationOptions ValidationOptions { get; internal set; } = Validation.ValidationOptions.Default(); @@ -112,208 +112,208 @@ public class Pipeline /// /// /// - /// Extensions can register hooks to validate their own node types during pre-flight. + /// Extensions can register hooks to validate their own step types during pre-flight. /// Hooks are invoked after DAG analysis but before external input inspection. /// /// /// Hook execution order: /// /// - /// Pipeline.Build() - DAG construction and layer assignment + /// Flow.Build() - DAG construction and layer assignment /// ValidationHooks.ValidateAsync() - Extension-specific validation - /// Pipeline.ValidateExternalInputsAsync() - External input inspection + /// Flow.ValidateExternalInputsAsync() - External input inspection /// /// /// Example (Python extension): /// /// - /// pipeline.ValidationHooks.Add(new PythonNodeValidator(executor, runtime)); + /// flow.ValidationHooks.Add(new PythonStepValidator(executor, runtime)); /// /// - public List ValidationHooks { get; } = new(); + public List ValidationHooks { get; } = new(); /// - /// Indicates whether the pipeline has been built (dependencies analyzed and layers assigned). + /// Indicates whether the Flow has been built (dependencies analyzed and layers assigned). /// public bool IsBuilt => ExecutionLayers != null; /// - /// Gets the sliced subset of nodes (if slicing was applied), otherwise null. + /// Gets the sliced subset of steps (if slicing was applied), otherwise null. /// /// - /// Used by metadata export to ensure only nodes that will execute are included in the DAG. + /// Used by metadata export to ensure only steps that will execute are included in the DAG. /// - internal List? GetSlicedNodes() => _slicedNodes; + internal List? GetSlicedSteps() => _slicedSteps; /// - /// Adds a node to the pipeline. + /// Adds a step to the flow. /// - /// The pipeline node to add - /// Thrown if pipeline has already been built - internal void AddNode(PipelineNode node) + /// The Flow step to add + /// Thrown if Flow has already been built + internal void AddStep(FlowStep step) { if (IsBuilt) { throw new InvalidOperationException( - "Cannot add nodes to a pipeline that has already been built. " - + "Create a new pipeline or use PipelineBuilder." + "Cannot add steps to a flow that has already been built. " + + "Create a new flow or use FlowBuilder." ); } - _nodes.Add(node); + _steps.Add(step); } /// - /// Merges multiple pipelines into a single pipeline by combining all their nodes. + /// Merges multiple flows into a single Flow by combining all their steps. /// - /// Dictionary of pipeline names to pipeline instances - /// A new pipeline containing all nodes from all input pipelines + /// Dictionary of flow names to Flow instances + /// A new Flow containing all steps from all input flows /// /// - /// This method creates a new pipeline by combining all nodes from the input pipelines. - /// Node names are prefixed with their source pipeline name (e.g., "data_processing.PreprocessCompanies") + /// This method creates a new Flow by combining all steps from the input flows. + /// Step names are prefixed with their source Flow name (e.g., "data_processing.PreprocessCompanies") /// to ensure uniqueness and maintain traceability in logs. /// /// - /// The existing DependencyAnalyzer will automatically resolve cross-pipeline dependencies - /// based on catalog entries. The single producer rule is enforced - if multiple pipelines + /// The existing DependencyAnalyzer will automatically resolve cross-flow dependencies + /// based on catalog entries. The single producer rule is enforced - if multiple flows /// attempt to write to the same catalog entry, Build() will throw an InvalidOperationException. /// /// - public static Pipeline Merge(Dictionary pipelines) + public static Flow Merge(Dictionary flows) { - var mergedPipeline = new Pipeline + var mergedFlows = new Flow { - Name = "Pipelines", - Description = $"Combined execution of: {string.Join(", ", pipelines.Keys)}", + Name = "Flows", + Description = $"Combined execution of: {string.Join(", ", flows.Keys)}", }; - // Combine all nodes from all pipelines, prefixing node names with pipeline name - foreach (var (pipelineName, pipeline) in pipelines) + // Combine all steps from all flows, prefixing step names with Flow name + foreach (var (flowName, flow) in flows) { - foreach (var node in pipeline.Nodes) + foreach (var step in flow.Steps) { - // Create a new node with prefixed name - var prefixedNode = new PipelineNode( - label: $"{pipelineName}.{node.Label}", - description: node.Description, - node: node.TransformFunction, - inputs: node.Inputs, - outputs: node.Outputs + // Create a new step with prefixed name + var prefixedStep = new FlowStep( + label: $"{flowName}.{step.Label}", + description: step.Description, + step: step.TransformFunction, + inputs: step.Inputs, + outputs: step.Outputs ); - mergedPipeline.AddNode(prefixedNode); + mergedFlows.AddStep(prefixedStep); } } - return mergedPipeline; + return mergedFlows; } /// - /// Builds the pipeline by analyzing dependencies and assigning execution layers. - /// Must be called before executing the pipeline. + /// Builds the Flow by analyzing dependencies and assigning execution layers. + /// Must be called before executing the flow. /// - /// Optional slicing strategy to filter nodes before execution + /// Optional slicing strategy to filter steps before execution /// /// Thrown if: - /// - Multiple nodes write to the same catalog entry (single producer rule) + /// - Multiple steps write to the same catalog entry (single producer rule) /// - Circular dependency is detected - /// - Slice strategy references non-existent nodes or catalog entries + /// - Slice strategy references non-existent steps or catalog entries /// /// /// - /// Slicing: If a slicing strategy is provided, only nodes matching + /// Slicing: If a slicing strategy is provided, only steps matching /// the strategy will be included in the execution. The slice always forms a valid /// sub-DAG with all required dependencies. /// /// - public void Build(PipelineSliceStrategy? sliceStrategy = null) + public void Build(FlowSliceStrategy? sliceStrategy = null) { if (IsBuilt) { - Logger?.LogWarning("Pipeline.Build() called on already-built pipeline. Rebuilding..."); + Logger?.LogWarning("Flow.Build() called on already-built flow. Rebuilding..."); } - Logger?.LogInformation("Building pipeline with {NodeCount} nodes", _nodes.Count); + Logger?.LogInformation("Building flow with {StepCount} steps", _steps.Count); // Cache the slice strategy for metadata export AppliedSlice = sliceStrategy?.IsSliced == true ? sliceStrategy : null; - // Step 1: Build dependency graph on the FULL node set + // Step 1: Build dependency graph on the FULL step set // This must happen before slicing, as slicing logic traverses dependencies - DependencyAnalyzer.BuildDependencyGraph(_nodes); + DependencyAnalyzer.BuildDependencyGraph(_steps); // Step 2: Apply slicing if requested if (sliceStrategy?.IsSliced == true) { - Logger?.LogInformation("Applying pipeline slice strategy"); - _slicedNodes = DependencyAnalyzer.SliceNodes(_nodes, sliceStrategy); + Logger?.LogInformation("Applying flow slice strategy"); + _slicedSteps = DependencyAnalyzer.SliceSteps(_steps, sliceStrategy); Logger?.LogInformation( - "Slice reduced pipeline from {OriginalCount} to {SlicedCount} nodes", - _nodes.Count, - _slicedNodes.Count + "Slice reduced flow from {OriginalCount} to {SlicedCount} steps", + _steps.Count, + _slicedSteps.Count ); } else { - _slicedNodes = null; // No slicing - execute all nodes + _slicedSteps = null; // No slicing - execute all steps } - // Step 3: Assign execution layers to the final node set (sliced or full) + // Step 3: Assign execution layers to the final step set (sliced or full) // This ensures Layer 0 correctly identifies external inputs in the execution context - var nodesToExecute = _slicedNodes ?? _nodes; - DependencyAnalyzer.AssignLayers(nodesToExecute); + var stepsToExecute = _slicedSteps ?? _steps; + DependencyAnalyzer.AssignLayers(stepsToExecute); - // Step 4: Group nodes by layer for execution - ExecutionLayers = DependencyAnalyzer.GroupByLayer(nodesToExecute).ToList(); + // Step 4: Group steps by layer for execution + ExecutionLayers = DependencyAnalyzer.GroupByLayer(stepsToExecute).ToList(); Logger?.LogInformation( - "Pipeline built successfully. Execution will proceed in {LayerCount} layers", + "Flow built successfully. Execution will proceed in {LayerCount} layers", ExecutionLayers.Count ); // Log layer details for (int i = 0; i < ExecutionLayers.Count; i++) { - var layerNodes = ExecutionLayers[i]; + var layerSteps = ExecutionLayers[i]; Logger?.LogDebug( - "Layer {LayerIndex}: {NodeCount} nodes ({NodeNames})", + "Layer {LayerIndex}: {StepCount} steps ({StepNames})", i, - layerNodes.Count, - string.Join(", ", layerNodes.Select(n => n.Label)) + layerSteps.Count, + string.Join(", ", layerSteps.Select(n => n.Label)) ); } } /// - /// Exports DAG metadata for this pipeline. + /// Exports DAG metadata for this Flow. /// - /// Complete DAG metadata including nodes, catalog entries, and edges - /// Thrown if pipeline has not been built + /// Complete DAG metadata including steps, catalog entries, and edges + /// Thrown if Flow has not been built /// /// - /// This method extracts structural metadata from the built pipeline, creating + /// This method extracts structural metadata from the built Flow , creating /// a complete representation of the DAG (Directed Acyclic Graph) that can be /// serialized to JSON for visualization in Flowthru.Viz. /// /// - /// Prerequisites: Pipeline must be built before calling this method. + /// Prerequisites: Flow must be built before calling this method. /// Call Build() first if IsBuilt is false. /// /// /// Usage: /// /// - /// var pipeline = DataProcessingPipeline.Create(catalog); - /// pipeline.Build(); + /// var Flow = DataProcessingFlow.Create(catalog); + /// flow.Build(); /// - /// var dag = pipeline.ExportDag(); + /// var dag = flow.ExportDag(); /// var json = dag.ToJson(); /// File.WriteAllText("dag.json", json); /// /// /// This method is non-destructive and idempotent - it can be called multiple - /// times without affecting the pipeline state. + /// times without affecting the Flow state. /// /// public DagMetadata ExportDag() @@ -321,34 +321,31 @@ public DagMetadata ExportDag() if (!IsBuilt) { throw new InvalidOperationException( - "Cannot export DAG metadata from an unbuilt pipeline. Call Build() first." + "Cannot export DAG metadata from an unbuilt flow. Call Build() first." ); } - Logger?.LogDebug( - "Exporting DAG metadata for pipeline '{PipelineName}'", - Name ?? "UnnamedPipeline" - ); + Logger?.LogDebug("Exporting DAG metadata for flow '{FlowName}'", Name ?? "UnnamedFlow"); return DagBuilder.Build(this); } /// - /// Validates all external inputs before pipeline execution. + /// Validates all external inputs before Flow execution. /// /// Cancellation token for validation I/O operations /// ValidationResult containing any errors found - /// Thrown if pipeline has not been built + /// Thrown if Flow has not been built /// /// - /// This method inspects catalog entries that are consumed by the pipeline but not - /// produced by any node in the execution set. These are pre-existing external data - /// sources (files, databases, APIs) that must exist and be valid before the pipeline + /// This method inspects catalog entries that are consumed by the Flow but not + /// produced by any step in the execution set. These are pre-existing external data + /// sources (files, databases, APIs) that must exist and be valid before the flow /// can execute. /// /// - /// Slicing Support: In sliced pipelines, catalog entries that were - /// produced by nodes outside the slice are correctly identified as external inputs + /// Slicing Support: In sliced flows, catalog entries that were + /// produced by steps outside the slice are correctly identified as external inputs /// and validated. This prevents runtime failures from missing intermediate data. /// /// @@ -368,20 +365,20 @@ public DagMetadata ExportDag() /// Otherwise → Shallow (all storage adapters support inspection) /// /// - /// Important: Only external inputs are inspected. Intermediate pipeline + /// Important: Only external inputs are inspected. Intermediate flow /// outputs produced within the execution set are never inspected, as they don't exist yet. /// /// /// Usage: /// /// - /// pipeline.Build(); - /// var validationResult = await pipeline.ValidateExternalInputsAsync(); + /// flow.Build(); + /// var validationResult = await flow.ValidateExternalInputsAsync(); /// if (!validationResult.IsValid) { /// // Handle validation errors before execution /// validationResult.ThrowIfInvalid(); /// } - /// await pipeline.RunAsync(); + /// await flow.RunAsync(); /// /// public async Task ValidateExternalInputsAsync( @@ -391,20 +388,20 @@ public DagMetadata ExportDag() if (!IsBuilt) { throw new InvalidOperationException( - "Pipeline must be built before validation. Call Build() first." + "Flow must be built before validation. Call Build() first." ); } var result = Data.Validation.ValidationResult.Success(); - // No nodes? No validation needed + // No steps? No validation needed if (ExecutionLayers!.Count == 0) { - Logger?.LogInformation("No nodes in pipeline, nothing to validate"); + Logger?.LogInformation("No steps in flow, nothing to validate"); return result; } - // Phase 4: Invoke validation hooks (e.g., Python node validation) + // Phase 4: Invoke validation hooks (e.g., Python step validation) if (ValidationHooks.Count > 0) { Logger?.LogInformation("Running {HookCount} validation hook(s)", ValidationHooks.Count); @@ -447,17 +444,17 @@ public DagMetadata ExportDag() } } - // Build a set of catalog entries produced by nodes in the execution set - var nodesToExecute = ExecutionLayers.SelectMany(layer => layer).ToList(); + // Build a set of catalog entries produced by stepsin the execution set + var stepsToExecute = ExecutionLayers.SelectMany(layer => layer).ToList(); var producedEntries = new HashSet( - nodesToExecute.SelectMany(node => node.Outputs.Select(entry => entry.Label)), + stepsToExecute.SelectMany(step => step.Outputs.Select(entry => entry.Label)), StringComparer.OrdinalIgnoreCase ); - // Find all catalog entries consumed by nodes that are NOT produced by any node - // These are external inputs in the execution context (including sliced pipelines) - var externalInputs = nodesToExecute - .SelectMany(node => node.Inputs) + // Find all catalog entries consumed by steps that are NOT produced by any steps + // These are external inputs in the execution context (including sliced flows) + var externalInputs = stepsToExecute + .SelectMany(step => step.Inputs) .Where(entry => !producedEntries.Contains(entry.Label)) .DistinctBy(entry => entry.Label) .ToList(); @@ -553,53 +550,53 @@ public DagMetadata ExportDag() } /// - /// /// Builds and executes the pipeline, returning comprehensive execution results. + /// /// Builds and executes the flow, returning comprehensive execution results. /// /// Cancellation token to signal graceful shutdown - /// PipelineResult containing execution status, timing, and node results + /// FlowResult containing execution status, timing, and Flow results /// /// - /// This is the primary high-level API for executing pipelines. It automatically - /// calls Build() if the pipeline hasn't been built yet, then executes and tracks results. + /// This is the primary high-level API for executing flows. It automatically + /// calls Build() if the Flow hasn't been built yet, then executes and tracks results. /// /// - public async Task RunAsync(CancellationToken cancellationToken) + public async Task RunAsync(CancellationToken cancellationToken) { var stopwatch = Stopwatch.StartNew(); - var nodeResults = new Dictionary(); + var stepResults = new Dictionary(); try { - // Ensure pipeline is built + // Ensure Flow is built if (!IsBuilt) { - Logger?.LogInformation("Building pipeline before execution"); + Logger?.LogInformation("Building flow before execution"); Build(); } - Logger?.LogInformation("Starting pipeline execution via RunAsync()"); + Logger?.LogInformation("Starting flow execution via RunAsync()"); // Execute all layers foreach (var layer in ExecutionLayers!) { - Logger?.LogInformation("Executing layer with {NodeCount} nodes", layer.Count); + Logger?.LogInformation("Executing layer with {StepCount} steps", layer.Count); - foreach (var pipelineNode in layer) + foreach (var flowStep in layer) { - // Check for cancellation before starting each node + // Check for cancellation before starting each step cancellationToken.ThrowIfCancellationRequested(); - var nodeResult = await ExecuteNodeWithTrackingAsync(pipelineNode, cancellationToken); - nodeResults[pipelineNode.Label] = nodeResult; + var stepResult = await ExecuteStepWithTrackingAsync(flowStep, cancellationToken); + stepResults[flowStep.Label] = stepResult; - // If node failed, stop execution - if (!nodeResult.Success) + // If step failed, stop execution + if (!stepResult.Success) { stopwatch.Stop(); - return PipelineResult.CreateFailure( + return FlowResult.CreateFailure( stopwatch.Elapsed, - nodeResult.Exception!, - nodeResults, + stepResult.Exception!, + stepResults, Name ); } @@ -608,11 +605,11 @@ public async Task RunAsync(CancellationToken cancellationToken) stopwatch.Stop(); Logger?.LogInformation( - "Pipeline execution completed successfully in {ElapsedMs}ms", + "Flow execution completed successfully in {ElapsedMs}ms", stopwatch.ElapsedMilliseconds ); - return PipelineResult.CreateSuccess(stopwatch.Elapsed, nodeResults, Name); + return FlowResult.CreateSuccess(stopwatch.Elapsed, stepResults, Name); } catch (OperationCanceledException) { @@ -624,22 +621,22 @@ public async Task RunAsync(CancellationToken cancellationToken) catch (Exception ex) { stopwatch.Stop(); - Logger?.LogError(ex, "Pipeline execution failed: {ErrorMessage}", ex.Message); - return PipelineResult.CreateFailure(stopwatch.Elapsed, ex, nodeResults, Name); + Logger?.LogError(ex, "Flow execution failed: {ErrorMessage}", ex.Message); + return FlowResult.CreateFailure(stopwatch.Elapsed, ex, stepResults, Name); } } /// - /// Executes the pipeline sequentially, layer by layer. + /// Executes the Flow sequentially, layer by layer. /// /// Cancellation token to signal graceful shutdown - /// Task representing the pipeline execution - /// Thrown if pipeline has not been built + /// Task representing the Flow execution + /// Thrown if Flow has not been built /// /// - /// This method executes nodes in topological order: - /// 1. Execute all nodes in layer 0 sequentially - /// 2. Execute all nodes in layer 1 sequentially + /// This method executes Flow in topological order: + /// 1. Execute all Flow in layer 0 sequentially + /// 2. Execute all Flow in layer 1 sequentially /// 3. Continue until all layers are complete /// /// @@ -648,7 +645,7 @@ public async Task RunAsync(CancellationToken cancellationToken) /// /// /// In Phase 2, this will be replaced with a parallel executor that can run - /// nodes within the same layer concurrently. + /// steps within the same layer concurrently. /// /// public async Task ExecuteAsync(CancellationToken cancellationToken) @@ -656,50 +653,50 @@ public async Task ExecuteAsync(CancellationToken cancellationToken) if (!IsBuilt) { throw new InvalidOperationException( - "Pipeline must be built before execution. Call Build() first." + "Flow must be built before execution. Call Build() first." ); } - Logger?.LogInformation("Starting pipeline execution"); + Logger?.LogInformation("Starting flow execution"); try { foreach (var layer in ExecutionLayers!) { - Logger?.LogInformation("Executing layer with {NodeCount} nodes", layer.Count); + Logger?.LogInformation("Executing layer with {StepCount} steps", layer.Count); - foreach (var pipelineNode in layer) + foreach (var flowStep in layer) { - // Check for cancellation before starting each node + // Check for cancellation before starting each step cancellationToken.ThrowIfCancellationRequested(); - await ExecuteNodeAsync(pipelineNode, cancellationToken); + await ExecuteStepAsync(flowStep, cancellationToken); } } - Logger?.LogInformation("Pipeline execution completed successfully"); + Logger?.LogInformation("Flow execution completed successfully"); } catch (Exception ex) { - Logger?.LogError(ex, "Pipeline execution failed: {ErrorMessage}", ex.Message); + Logger?.LogError(ex, "Flow execution failed: {ErrorMessage}", ex.Message); throw; } } /// - /// Determines whether a node's transformation function accepts a CancellationToken parameter. + /// Determines whether a step's transformation function accepts a CancellationToken parameter. /// /// The transformation function delegate /// True if the function accepts a CancellationToken as its last parameter /// - /// Supports optional cancellation awareness in node functions. Nodes can opt-in to cancellation + /// Supports optional cancellation awareness in step functions. Steps can opt-in to cancellation /// by accepting a CancellationToken as the last parameter: /// /// Func<TIn, CancellationToken, Task<TOut>> - single input with cancellation /// Func<(TIn1, TIn2), CancellationToken, Task<TOut>> - multi-input with cancellation /// /// - private static bool NodeAcceptsCancellationToken(Delegate transformFunc) + private static bool StepAcceptsCancellationToken(Delegate transformFunc) { var invokeMethod = transformFunc.GetType().GetMethod("Invoke"); var parameters = invokeMethod!.GetParameters(); @@ -709,13 +706,13 @@ private static bool NodeAcceptsCancellationToken(Delegate transformFunc) } /// - /// Executes a single node with execution tracking and returns detailed results. + /// Executes a single step with execution tracking and returns detailed results. /// - /// The node to execute + /// The step to execute /// Cancellation token for I/O operations - /// NodeResult with execution details - private async Task ExecuteNodeWithTrackingAsync( - PipelineNode pipelineNode, + /// StepResult with execution details + private async Task ExecuteStepWithTrackingAsync( + FlowStep flowStep, CancellationToken cancellationToken ) { @@ -724,35 +721,35 @@ CancellationToken cancellationToken try { // Get input counts for diagnostics (before loading data) - var inputCountAffs = pipelineNode.Inputs.Select(entry => entry.GetCountAsync()); + var inputCountAffs = flowStep.Inputs.Select(entry => entry.GetCountAsync()); var inputCountTasks = inputCountAffs.Select(aff => aff.Run(cancellationToken).AsTask()); var inputCountResults = await Task.WhenAll(inputCountTasks); var inputCounts = inputCountResults; var totalInputCount = inputCounts.Sum(); Logger?.LogInformation( - "Executing node: {NodeName} (inputs: {InputCount} observations from {EntryCount} entries)", - pipelineNode.Label, + "Executing step: {StepName} (inputs: {InputCount} observations from {EntryCount} entries)", + flowStep.Label, totalInputCount, - pipelineNode.Inputs.Count + flowStep.Inputs.Count ); // Load inputs from catalog entries // LoadUntyped() returns T directly (singleton or collection), no wrapping needed - var inputAffs = pipelineNode.Inputs.Select(entry => entry.LoadUntyped()); + var inputAffs = flowStep.Inputs.Select(entry => entry.LoadUntyped()); var inputLoadTasks = inputAffs.Select(aff => aff.Run(cancellationToken).AsTask()); var inputResults = await Task.WhenAll(inputLoadTasks); var inputs = inputResults; // Prepare input parameter for function invocation - // For single-input nodes: pass data directly (T) - // For multi-input nodes: construct tuple (T1, T2, ...) or pass as object[] for fan-in + // For single-input steps: pass data directly (T) + // For multi-input steps: construct tuple (T1, T2, ...) or pass as object[] for fan-in object inputParameter; - if (pipelineNode.Inputs.Count == 1) + if (flowStep.Inputs.Count == 1) { // Single input: pass data directly, unless this is a fan-in wrapper (Func) - // in which case the node still expects an object[] even with a single entry. - var singleFuncType = pipelineNode.TransformFunction.GetType(); + // in which case the step still expects an object[] even with a single entry. + var singleFuncType = flowStep.TransformFunction.GetType(); var singleParams = singleFuncType.GetMethod("Invoke")!.GetParameters(); if (singleParams.Length == 1 && singleParams[0].ParameterType == typeof(object[])) inputParameter = (object)inputs; // fan-in wrapper with single shard @@ -761,16 +758,16 @@ CancellationToken cancellationToken } else { - // Multi-input: construct tuple from loaded values, or pass as object[] for fan-in nodes. + // Multi-input: construct tuple from loaded values, or pass as object[] for fan-in steps. // Use the function's actual parameter type to ensure correct tuple signature. - var funcType = pipelineNode.TransformFunction.GetType(); + var funcType = flowStep.TransformFunction.GetType(); var invokeMethod = funcType.GetMethod("Invoke"); var parameters = invokeMethod!.GetParameters(); if (parameters.Length != 1) { throw new InvalidOperationException( - $"Transform function for node {pipelineNode.Label} should have exactly 1 parameter (tuple), but has {parameters.Length}" + $"Transform function for step {flowStep.Label} should have exactly 1 parameter (tuple), but has {parameters.Length}" ); } @@ -778,7 +775,7 @@ CancellationToken cancellationToken if (paramType == typeof(object[])) { - // Fan-in node: the PipelineBuilder wraps Func, TOut> into + // Fan-in step: the FlowBuilder wraps Func, TOut> into // Func. Pass the loaded array as a single boxed object so // DynamicInvoke receives new object[]{ inputs } — no array-spreading occurs. inputParameter = (object)inputs; @@ -799,7 +796,7 @@ CancellationToken cancellationToken catch (Exception ex) { throw new InvalidOperationException( - $"Failed to create {inputs.Length}-tuple for node {pipelineNode.Label}. " + $"Failed to create {inputs.Length}-tuple for step {flowStep.Label}. " + $"Expected tuple type: {tupleType.FullName}, Input types: [{string.Join(", ", inputs.Select(v => v?.GetType().Name ?? "null"))}]", ex ); @@ -808,10 +805,10 @@ CancellationToken cancellationToken } // Invoke transformation function directly via DynamicInvoke - // Pass cancellation token if the node signature accepts it - var transformFunc = pipelineNode.TransformFunction; + // Pass cancellation token if the step signature accepts it + var transformFunc = flowStep.TransformFunction; object? result; - if (NodeAcceptsCancellationToken(transformFunc)) + if (StepAcceptsCancellationToken(transformFunc)) { result = transformFunc.DynamicInvoke(inputParameter, cancellationToken); } @@ -821,13 +818,13 @@ CancellationToken cancellationToken } File.AppendAllText( "/tmp/flowthru_diag.log", - $"[{DateTime.Now:HH:mm:ss.fff}] Pipeline: Transform invoked, result type={result?.GetType().Name ?? "null"}\n" + $"[{DateTime.Now:HH:mm:ss.fff}] Flow: Transform invoked, result type={result?.GetType().Name ?? "null"}\n" ); if (result == null) { throw new InvalidOperationException( - $"Transform function for node {pipelineNode.Label} returned null" + $"Transform function for step {flowStep.Label} returned null" ); } @@ -847,12 +844,12 @@ CancellationToken cancellationToken // Save outputs to catalog entries // SaveUntyped() accepts T directly (singleton or collection), no unwrapping needed - if (output != null && pipelineNode.Outputs.Count > 0) + if (output != null && flowStep.Outputs.Count > 0) { - if (pipelineNode.Outputs.Count == 1) + if (flowStep.Outputs.Count == 1) { // Single output: save directly - var catalogEntry = pipelineNode.Outputs[0]; + var catalogEntry = flowStep.Outputs[0]; await catalogEntry.SaveUntyped(output).Run(cancellationToken); } else @@ -862,23 +859,23 @@ CancellationToken cancellationToken if (!tupleType.IsGenericType || !tupleType.FullName!.StartsWith("System.ValueTuple")) { throw new InvalidOperationException( - $"Multi-output node '{pipelineNode.Label}' must return tuple, got: {tupleType.Name}" + $"Multi-output step '{flowStep.Label}' must return tuple, got: {tupleType.Name}" ); } // Get tuple fields (Item1, Item2, ...) var tupleFields = tupleType.GetFields(); - if (tupleFields.Length != pipelineNode.Outputs.Count) + if (tupleFields.Length != flowStep.Outputs.Count) { throw new InvalidOperationException( - $"Multi-output node '{pipelineNode.Label}': Tuple arity ({tupleFields.Length}) doesn't match output count ({pipelineNode.Outputs.Count})" + $"Multi-output step '{flowStep.Label}': Tuple arity ({tupleFields.Length}) doesn't match output count ({flowStep.Outputs.Count})" ); } // Save each output directly from tuple field - for (int i = 0; i < pipelineNode.Outputs.Count; i++) + for (int i = 0; i < flowStep.Outputs.Count; i++) { - var catalogEntry = pipelineNode.Outputs[i]; + var catalogEntry = flowStep.Outputs[i]; var field = tupleFields[i]; var outputData = field.GetValue(output); @@ -890,22 +887,22 @@ CancellationToken cancellationToken stopwatch.Stop(); // Get output counts for diagnostics (after saving data) - var outputCountAffs = pipelineNode.Outputs.Select(entry => entry.GetCountAsync()); + var outputCountAffs = flowStep.Outputs.Select(entry => entry.GetCountAsync()); var outputCountTasks = outputCountAffs.Select(aff => aff.Run(cancellationToken).AsTask()); var outputCountResults = await Task.WhenAll(outputCountTasks); var outputCounts = outputCountResults; var totalOutputCount = outputCounts.Sum(); Logger?.LogInformation( - "Node {NodeName} completed: {InputCount} observations in → {OutputCount} observations out ({ElapsedMs}ms)", - pipelineNode.Label, + "Step {StepName} completed: {InputCount} observations in → {OutputCount} observations out ({ElapsedMs}ms)", + flowStep.Label, totalInputCount, totalOutputCount, stopwatch.ElapsedMilliseconds ); - return NodeResult.CreateSuccess( - pipelineNode.Label, + return StepResult.CreateSuccess( + flowStep.Label, stopwatch.Elapsed, totalInputCount, totalOutputCount @@ -924,43 +921,35 @@ CancellationToken cancellationToken catch (Exception ex) { stopwatch.Stop(); - Logger?.LogError( - ex, - "Node {NodeName} failed: {ErrorMessage}", - pipelineNode.Label, - ex.Message - ); - return NodeResult.CreateFailure(pipelineNode.Label, stopwatch.Elapsed, ex); + Logger?.LogError(ex, "Step {StepName} failed: {ErrorMessage}", flowStep.Label, ex.Message); + return StepResult.CreateFailure(flowStep.Label, stopwatch.Elapsed, ex); } } /// - /// Executes a single node by loading its inputs, invoking the transformation, + /// Executes a single step by loading its inputs, invoking the transformation, /// and saving its outputs. /// - /// The node to execute + /// The step to execute /// Cancellation token for I/O operations - private async Task ExecuteNodeAsync( - PipelineNode pipelineNode, - CancellationToken cancellationToken - ) + private async Task ExecuteStepAsync(FlowStep flowStep, CancellationToken cancellationToken) { - Logger?.LogInformation("Executing node: {NodeName}", pipelineNode.Label); + Logger?.LogInformation("Executing step: {StepName}", flowStep.Label); try { // Load inputs from catalog entries - var inputAffs = pipelineNode.Inputs.Select(entry => entry.LoadUntyped()); + var inputAffs = flowStep.Inputs.Select(entry => entry.LoadUntyped()); var inputLoadTasks = inputAffs.Select(aff => aff.Run(cancellationToken).AsTask()); var inputResults = await Task.WhenAll(inputLoadTasks); var inputs = inputResults; // Prepare input parameter object inputParameter; - if (pipelineNode.Inputs.Count == 1) + if (flowStep.Inputs.Count == 1) { // Single input, unless this is a fan-in wrapper (Func) - var singleFuncType = pipelineNode.TransformFunction.GetType(); + var singleFuncType = flowStep.TransformFunction.GetType(); var singleParams = singleFuncType.GetMethod("Invoke")!.GetParameters(); if (singleParams.Length == 1 && singleParams[0].ParameterType == typeof(object[])) inputParameter = (object)inputs; @@ -970,14 +959,14 @@ CancellationToken cancellationToken else { // Use the function's actual parameter type to ensure correct tuple signature - var funcType = pipelineNode.TransformFunction.GetType(); + var funcType = flowStep.TransformFunction.GetType(); var invokeMethod = funcType.GetMethod("Invoke"); var parameters = invokeMethod!.GetParameters(); var paramType = parameters[0].ParameterType; if (paramType == typeof(object[])) { - // Fan-in node: pass the loaded array as a single boxed object. + // Fan-in step: pass the loaded array as a single boxed object. inputParameter = (object)inputs; } else @@ -986,16 +975,16 @@ CancellationToken cancellationToken inputParameter = Activator.CreateInstance(tupleType, inputs) ?? throw new InvalidOperationException( - $"Failed to create tuple for node {pipelineNode.Label}" + $"Failed to create tuple for step {flowStep.Label}" ); } } // Invoke transformation function - // Pass cancellation token if the node signature accepts it - var transformFunc = pipelineNode.TransformFunction; + // Pass cancellation token if the step signature accepts it + var transformFunc = flowStep.TransformFunction; Task? resultTask; - if (NodeAcceptsCancellationToken(transformFunc)) + if (StepAcceptsCancellationToken(transformFunc)) { resultTask = (Task?)transformFunc.DynamicInvoke(inputParameter, cancellationToken); } @@ -1007,7 +996,7 @@ CancellationToken cancellationToken if (resultTask == null) { throw new InvalidOperationException( - $"Transform function for {pipelineNode.Label} returned null" + $"Transform function for {flowStep.Label} returned null" ); } @@ -1015,31 +1004,26 @@ CancellationToken cancellationToken var output = GetTaskResult(resultTask); // Save outputs - if (output != null && pipelineNode.Outputs.Count > 0) + if (output != null && flowStep.Outputs.Count > 0) { - if (pipelineNode.Outputs.Count == 1) + if (flowStep.Outputs.Count == 1) { - await pipelineNode.Outputs[0].SaveUntyped(output).Run(cancellationToken); + await flowStep.Outputs[0].SaveUntyped(output).Run(cancellationToken); } else { var tupleFields = output.GetType().GetFields(); - for (int i = 0; i < pipelineNode.Outputs.Count; i++) + for (int i = 0; i < flowStep.Outputs.Count; i++) { var outputData = tupleFields[i].GetValue(output); - await pipelineNode.Outputs[i].SaveUntyped(outputData!).Run(cancellationToken); + await flowStep.Outputs[i].SaveUntyped(outputData!).Run(cancellationToken); } } } } catch (Exception ex) { - Logger?.LogError( - ex, - "Node {NodeName} failed: {ErrorMessage}", - pipelineNode.Label, - ex.Message - ); + Logger?.LogError(ex, "Step {StepName} failed: {ErrorMessage}", flowStep.Label, ex.Message); throw; } } diff --git a/src/core/Flowthru.Core/Pipelines/PipelineBuilder.cs b/src/core/Flowthru.Core/Flows/PipelineBuilder.cs similarity index 60% rename from src/core/Flowthru.Core/Pipelines/PipelineBuilder.cs rename to src/core/Flowthru.Core/Flows/PipelineBuilder.cs index 86e9fa5d..362a5a07 100644 --- a/src/core/Flowthru.Core/Pipelines/PipelineBuilder.cs +++ b/src/core/Flowthru.Core/Flows/PipelineBuilder.cs @@ -1,15 +1,15 @@ using Flowthru.Data; -using Flowthru.Nodes; +using Flowthru.Steps; -namespace Flowthru.Pipelines; +namespace Flowthru.Flows; /// -/// Fluent builder for constructing type-safe data pipelines with function-based nodes. +/// Fluent builder for constructing type-safe flows with function-based steps. /// /// /// /// Function-Based Design (v0.5.0): -/// Nodes are pure transformation functions with compile-time type safety. +/// Steps are pure transformation functions with compile-time type safety. /// Both synchronous and asynchronous functions are supported: /// - Sync: Func<TInput, TOutput> /// - Async: Func<TInput, Task<TOutput>> @@ -18,187 +18,190 @@ namespace Flowthru.Pipelines; /// /// /// Use synchronous functions for pure data transformations. Use asynchronous functions -/// only when your node performs I/O operations (external APIs, databases, etc.). +/// only when your step performs I/O operations (external APIs, databases, etc.). /// /// /// The compiler infers all types from function signatures and validates catalog entry -/// types at pipeline construction time, catching type mismatches before execution. +/// types at Flow construction time, catching type mismatches before execution. /// /// /// Usage Patterns: /// /// -/// var pipeline = PipelineBuilder.CreatePipeline(builder => +/// var Flow = FlowBuilder.CreateFlow(builder => /// { -/// // Simple synchronous node -/// builder.AddNode( +/// // Simple synchronous step +/// builder.AddStep( /// name: "Preprocess", -/// transform: PreprocessNode.Create(), +/// transform: PreprocessStep.Create(), /// input: catalog.RawData, /// output: catalog.ProcessedData /// ); /// -/// // Multi-input node: tuple → single output -/// builder.AddNode( +/// // Multi-input step: tuple → single output +/// builder.AddStep( /// name: "TrainModel", -/// transform: TrainModelNode.Create(), +/// transform: TrainModelStep.Create(), /// input: (catalog.XTrain, catalog.YTrain), /// output: catalog.Model /// ); /// -/// // Multi-output node: single input → tuple -/// builder.AddNode( +/// // Multi-output step: single input → tuple +/// builder.AddStep( /// name: "SplitData", -/// transform: SplitDataNode.Create(), +/// transform: SplitDataStep.Create(), /// input: catalog.Data, /// output: (catalog.XTrain, catalog.XTest, catalog.YTrain, catalog.YTest) /// ); /// -/// // Asynchronous node (only when needed for I/O) -/// builder.AddNode( +/// // Asynchronous step (only when needed for I/O) +/// builder.AddStep( /// name: "FetchExternalData", -/// transform: ExternalDataNode.Create(), +/// transform: ExternalDataStep.Create(), /// input: catalog.Config, /// output: catalog.ExternalData /// ); /// }); /// -/// pipeline.Build(); -/// await pipeline.ExecuteAsync(); +/// flow.Build(); +/// await flow.ExecuteAsync(); /// /// -public partial class PipelineBuilder +public partial class FlowBuilder { /// - /// Maximum number of inputs supported by generated AddNode overloads. + /// Maximum number of inputs supported by generated AddStep overloads. /// internal const int MaxInputs = 8; /// - /// Maximum number of outputs supported by generated AddNode overloads. + /// Maximum number of outputs supported by generated AddStep overloads. /// internal const int MaxOutputs = 8; - private readonly Pipeline _pipeline = new(); + private readonly Flow _flow = new(); /// - /// Creates and configures a new pipeline using the builder pattern. + /// Creates and configures a new Flow using the builder pattern. /// - /// Action to configure the pipeline by adding nodes - /// Configured but not yet built pipeline - public static Pipeline CreatePipeline(Action configure) + /// Action to configure the Flow by adding steps + /// Configured but not yet built flow + public static Flow CreateFlow(Action configure) { - var builder = new PipelineBuilder(); + var builder = new FlowBuilder(); configure(builder); - return builder._pipeline; + return builder._flow; } /// - /// Adds a node with single input and single output (asynchronous transformation). + /// Adds a step with single input and single output (asynchronous transformation). /// All types are inferred from the transformation function signature. /// /// Input type (inferred from transform) /// Output type (inferred from transform) - /// Unique identifier for this node + /// Unique identifier for this step /// Asynchronous transformation function from input to output /// Catalog entry providing input data /// Catalog entry to store output data + /// Optional description for this step /// This builder for method chaining - public PipelineBuilder AddNode( + public FlowBuilder AddStep( string label, Func> transform, - ICatalogEntry input, - ICatalogEntry output, + IItem input, + IItem output, string description = "" ) { - var pipelineNode = new PipelineNode( + var flowStep = new FlowStep( label: label, description: description, - node: transform, - inputs: new List { input }, - outputs: new List { output } + step: transform, + inputs: new List { input }, + outputs: new List { output } ); - _pipeline.AddNode(pipelineNode); + _flow.AddStep(flowStep); return this; } /// - /// Adds a node with single input and single output (asynchronous transformation with cancellation support). + /// Adds a step with single input and single output (asynchronous transformation with cancellation support). /// All types are inferred from the transformation function signature. /// /// Input type (inferred from transform) /// Output type (inferred from transform) - /// Unique identifier for this node + /// Unique identifier for this step /// Asynchronous transformation function from input to output with cancellation token /// Catalog entry providing input data /// Catalog entry to store output data + /// Optional description for this step /// This builder for method chaining - public PipelineBuilder AddNode( + public FlowBuilder AddStep( string label, Func> transform, - ICatalogEntry input, - ICatalogEntry output, + IItem input, + IItem output, string description = "" ) { - var pipelineNode = new PipelineNode( + var flowStep = new FlowStep( label: label, description: description, - node: transform, - inputs: new List { input }, - outputs: new List { output } + step: transform, + inputs: new List { input }, + outputs: new List { output } ); - _pipeline.AddNode(pipelineNode); + _flow.AddStep(flowStep); return this; } /// - /// Adds a node with single input and single output (synchronous transformation). + /// Adds a step with single input and single output (synchronous transformation). /// All types are inferred from the transformation function signature. /// /// Input type (inferred from transform) /// Output type (inferred from transform) - /// Unique identifier for this node + /// Unique identifier for this step /// Synchronous transformation function from input to output /// Catalog entry providing input data /// Catalog entry to store output data + /// Optional description for this step /// This builder for method chaining - public PipelineBuilder AddNode( + public FlowBuilder AddStep( string label, Func transform, - ICatalogEntry input, - ICatalogEntry output, + IItem input, + IItem output, string description = "" ) { - var pipelineNode = new PipelineNode( + var flowStep = new FlowStep( label: label, description: description, - node: transform, - inputs: new List { input }, - outputs: new List { output } + step: transform, + inputs: new List { input }, + outputs: new List { output } ); - _pipeline.AddNode(pipelineNode); + _flow.AddStep(flowStep); return this; } // Additional overloads (2-8 inputs, 1-8 outputs) are auto-generated via source generator. - // See: Flowthru.SourceGenerators/PipelineBuilderGenerator.cs + // See: Flowthru.SourceGenerators/FlowBuilderGenerator.cs /// - /// Adds a homogeneous fan-in node: N catalog entries of the same element type collapse - /// into a single node whose transform receives all N loaded collections as a typed list. + /// Adds a homogeneous fan-in step: N catalog entries of the same element type collapse + /// into a single step whose transform receives all N loaded collections as a typed list. /// /// Element type of each input catalog entry /// Output type produced by the transform - /// Unique identifier for this node + /// Unique identifier for this step /// Variable-length list of same-typed input entries /// Catalog entry to store the merged result - /// Transform function receiving all N loaded values as a typed read-only list + /// Transform function receiving all N loaded values as a typed read-only list /// Optional human-readable description /// This builder for method chaining /// @@ -207,11 +210,11 @@ public PipelineBuilder AddNode( /// IReadOnlyList<TIn> where each element corresponds to one input entry /// in declaration order. An empty inputs list is allowed but produces an empty list argument. /// - public PipelineBuilder AddNode( + public FlowBuilder AddStep( string label, - IReadOnlyList> inputs, - ICatalogEntry output, - Func, TOut> node, + IReadOnlyList> inputs, + IItem output, + Func, TOut> step, string description = "" ) { @@ -219,26 +222,26 @@ public PipelineBuilder AddNode( throw new ArgumentNullException(nameof(inputs)); if (output == null) throw new ArgumentNullException(nameof(output)); - if (node == null) - throw new ArgumentNullException(nameof(node)); + if (step == null) + throw new ArgumentNullException(nameof(step)); - var capturedNode = node; + var capturedStep = step; // Wrap into Func. The executor uses the parameter type as a signal: // when it is object[], it passes the loaded inputs array directly rather than building // a ValueTuple. inputParameter is declared as object in the executor so DynamicInvoke // receives new object[]{ inputParameter } — no array-spreading occurs. - Func wrappedNode = rawInputs => - capturedNode(rawInputs.Cast().ToList().AsReadOnly()); + Func wrappedStep = rawInputs => + capturedStep(rawInputs.Cast().ToList().AsReadOnly()); - var pipelineNode = new PipelineNode( + var flowStep = new FlowStep( label: label, description: description, - node: wrappedNode, - inputs: inputs.Cast().ToList(), - outputs: new List { output } + step: wrappedStep, + inputs: inputs.Cast().ToList(), + outputs: new List { output } ); - _pipeline.AddNode(pipelineNode); + _flow.AddStep(flowStep); return this; } } diff --git a/src/core/Flowthru.Core/Pipelines/PipelineNode.cs b/src/core/Flowthru.Core/Flows/PipelineNode.cs similarity index 53% rename from src/core/Flowthru.Core/Pipelines/PipelineNode.cs rename to src/core/Flowthru.Core/Flows/PipelineNode.cs index 0ea15556..25c55eaa 100644 --- a/src/core/Flowthru.Core/Pipelines/PipelineNode.cs +++ b/src/core/Flowthru.Core/Flows/PipelineNode.cs @@ -1,52 +1,46 @@ using Flowthru.Data; -using Flowthru.Nodes; +using Flowthru.Steps; -namespace Flowthru.Pipelines; +namespace Flowthru.Flows; /// -/// Represents a node within a pipeline, wrapping the transformation function with metadata +/// Represents a step within a flow, wrapping the transformation function with metadata /// about its inputs, outputs, and dependencies. /// /// /// -/// PipelineNode serves as the internal representation of a node during pipeline +/// FlowStep serves as the internal representation of a step during flow /// construction and execution. It tracks: /// - The transformation function (Func<TInput, Task<TOutput>>) /// - Input catalog entries (what data it reads) /// - Output catalog entries (what data it writes) -/// - Dependencies (other nodes that must run first) +/// - Dependencies (other steps that must run first) /// /// /// Single Producer Rule: Each catalog entry can be written by at most -/// one node in a pipeline. This constraint ensures deterministic dependency resolution +/// one step in a flow. This constraint ensures deterministic dependency resolution /// and enables simple DAG construction via topological sort. /// /// -/// Function-Based Architecture (v0.5.0): Nodes are now pure transformation -/// functions instead of class instances. This enables compile-time type safety through -/// generic type inference at the pipeline construction site. -/// -/// -/// Visibility (Phase 4): -/// Made public to enable validation hooks to inspect node properties. -/// This is necessary for extensions (e.g., Python) to validate their own node types. +/// Made public to enable validation hooks to inspect step properties. +/// This is necessary for extensions (e.g., Python) to validate their own step types. /// /// -public class PipelineNode +public class FlowStep { /// - /// Unique identifier for this node within the pipeline. - /// Typically the node type name or user-provided name. + /// Unique identifier for this step within the flow. + /// Typically the step type name or user-provided name. /// public string Label { get; } /// - /// String description of the node's purpose. + /// String description of the step's purpose. /// public string Description { get; } /// - /// The transformation function that performs the node's work. + /// The transformation function that performs the step's work. /// Type-erased to Delegate since we need to store different function signatures together. /// /// @@ -62,15 +56,15 @@ public class PipelineNode /// - Async multi-output: Func<TInput, Task<(TOut1, TOut2, ...)>> /// /// - /// Optional Cancellation Support: Nodes can opt-in to cancellation awareness + /// Optional Cancellation Support: Steps can opt-in to cancellation awareness /// by accepting a CancellationToken as the last parameter: /// - Func<TInput, CancellationToken, Task<TOutput>> /// - Func<(TIn1, TIn2), CancellationToken, Task<TOutput>> /// /// - /// When a node accepts a CancellationToken, the pipeline will pass the runtime token during - /// execution, allowing the node to cancel long-running operations cooperatively. Nodes that - /// do not accept a CancellationToken will only be cancelled between node executions. + /// When a Step accepts a CancellationToken, the Flow will pass the runtime token during + /// execution, allowing the step to cancel long-running operations cooperatively. Steps that + /// do not accept a CancellationToken will only be cancelled between step executions. /// /// /// The execution engine detects whether the result is a Task and awaits it if needed. @@ -79,52 +73,53 @@ public class PipelineNode public Delegate TransformFunction { get; } /// - /// Catalog entries that this node reads as input. - /// These may be produced by other nodes (dependencies) or be external prerequisites. + /// Catalog entries that this step reads as input. + /// These may be produced by other steps (dependencies) or be external prerequisites. /// - public IReadOnlyList Inputs { get; } + public IReadOnlyList Inputs { get; } /// - /// Catalog entries that this node writes as output. - /// Per the single producer rule, each entry here must be unique across all nodes. + /// Catalog entries that this step writes as output. + /// Per the single producer rule, each entry here must be unique across all steps. /// - public IReadOnlyList Outputs { get; } + public IReadOnlyList Outputs { get; } /// - /// Other pipeline nodes that must execute before this node. - /// Populated during dependency analysis by checking which nodes produce our inputs. + /// Other Flow steps that must execute before this step. + /// Populated during dependency analysis by checking which steps produce our inputs. /// /// /// This forms the edges of the execution DAG: - /// - If node A produces output X, and node B consumes input X, then B depends on A. + /// - If step A produces output X, and step B consumes input X, then B depends on A. /// - Topological sort uses these dependencies to determine execution order. /// - public List Dependencies { get; } = new(); + public List Dependencies { get; } = new(); /// /// Execution layer determined by topological sort. - /// Nodes in layer 0 have no dependencies. Nodes in layer N depend on nodes in layers 0..N-1. + /// Steps in layer 0 have no dependencies. Steps in layer N depend on steps in layers 0..N-1. /// public int Layer { get; set; } = -1; // -1 indicates not yet assigned /// - /// Creates a new pipeline node with a transformation function. + /// Creates a new Flow step with a transformation function. /// - /// Unique identifier for this node - /// The transformation function (Func<TInput, Task<TOutput>>) - /// Catalog entries this node reads - /// Catalog entries this node writes - public PipelineNode( + /// Unique identifier for this step + /// Optional description of this step + /// The transformation function (Func<TInput, Task<TOutput>>) + /// Catalog entries this step reads + /// Catalog entries this step writes + public FlowStep( string label, string? description, - Delegate node, - IReadOnlyList inputs, - IReadOnlyList outputs + Delegate step, + IReadOnlyList inputs, + IReadOnlyList outputs ) { Label = label; Description = description ?? string.Empty; - TransformFunction = node; + TransformFunction = step; Inputs = inputs; Outputs = outputs; } @@ -133,5 +128,5 @@ IReadOnlyList outputs /// Returns a string representation for debugging. /// public override string ToString() => - $"PipelineNode({Label}, Layer={Layer}, Inputs={Inputs.Count}, Outputs={Outputs.Count}, Dependencies={Dependencies.Count})"; + $"FlowStep({Label}, Layer={Layer}, Inputs={Inputs.Count}, Outputs={Outputs.Count}, Dependencies={Dependencies.Count})"; } diff --git a/src/core/Flowthru.Core/Pipelines/PipelineResult.cs b/src/core/Flowthru.Core/Flows/PipelineResult.cs similarity index 51% rename from src/core/Flowthru.Core/Pipelines/PipelineResult.cs rename to src/core/Flowthru.Core/Flows/PipelineResult.cs index ca5c0e22..eb516268 100644 --- a/src/core/Flowthru.Core/Pipelines/PipelineResult.cs +++ b/src/core/Flowthru.Core/Flows/PipelineResult.cs @@ -1,42 +1,42 @@ -namespace Flowthru.Pipelines; +namespace Flowthru.Flows; /// -/// Represents the result of a pipeline execution. +/// Represents the result of a Flow execution. /// /// /// /// This class provides comprehensive execution information including success status, -/// timing, individual node results, and error details. +/// timing, individual step results, and error details. /// /// /// Usage Pattern: /// /// -/// var result = await pipeline.RunAsync(); +/// var result = await flow.RunAsync(); /// /// if (result.Success) /// { -/// Console.WriteLine($"Pipeline completed in {result.ExecutionTime.TotalSeconds:F2}s"); -/// foreach (var nodeResult in result.NodeResults.Values) +/// Console.WriteLine($"Flow completed in {result.ExecutionTime.TotalSeconds:F2}s"); +/// foreach (var stepResult in result.StepResults.Values) /// { -/// Console.WriteLine($" {nodeResult.NodeName}: {nodeResult.ExecutionTime.TotalSeconds:F2}s"); +/// Console.WriteLine($" {stepResult.StepName}: {stepResult.ExecutionTime.TotalSeconds:F2}s"); /// } /// } /// else /// { -/// Console.WriteLine($"Pipeline failed: {result.Exception?.Message}"); +/// Console.WriteLine($"Flow failed: {result.Exception?.Message}"); /// } /// /// -public class PipelineResult +public class FlowResult { /// - /// The name of the pipeline that was executed. + /// The name of the Flow that was executed. /// - public string? PipelineName { get; init; } + public string? FlowName { get; init; } /// - /// Indicates whether the pipeline executed successfully. + /// Indicates whether the Flow executed successfully. /// public bool Success { get; init; } @@ -46,65 +46,65 @@ public class PipelineResult public bool IsDryRun { get; init; } /// - /// Total execution time for the entire pipeline. + /// Total execution time for the entire flow. /// public TimeSpan ExecutionTime { get; init; } /// - /// Results for individual nodes, keyed by node name. + /// Results for individual steps, keyed by step name. /// /// - /// Dictionary keys are the node names as specified in the pipeline definition. - /// Values contain execution details for each node. + /// Dictionary keys are the step names as specified in the Flow definition. + /// Values contain execution details for each step. /// - public Dictionary NodeResults { get; init; } = new(); + public Dictionary StepResults { get; init; } = new(); /// - /// Exception that caused pipeline failure, if any. + /// Exception that caused Flow failure, if any. /// /// /// Null if Success is true. Contains the first exception that caused - /// pipeline execution to halt if Success is false. + /// Flow execution to halt if Success is false. /// public Exception? Exception { get; init; } /// - /// Creates a successful pipeline result. + /// Creates a successful Flow result. /// - public static PipelineResult CreateSuccess( + public static FlowResult CreateSuccess( TimeSpan executionTime, - Dictionary nodeResults, - string? pipelineName = null + Dictionary stepResults, + string? flowName = null ) { - return new PipelineResult + return new FlowResult { Success = true, IsDryRun = false, ExecutionTime = executionTime, - NodeResults = nodeResults, - PipelineName = pipelineName, + StepResults = stepResults, + FlowName = flowName, }; } /// - /// Creates a failed pipeline result. + /// Creates a failed Flow result. /// - public static PipelineResult CreateFailure( + public static FlowResult CreateFailure( TimeSpan executionTime, Exception exception, - Dictionary? nodeResults = null, - string? pipelineName = null + Dictionary? stepResults = null, + string? flowName = null ) { - return new PipelineResult + return new FlowResult { Success = false, IsDryRun = false, ExecutionTime = executionTime, Exception = exception, - NodeResults = nodeResults ?? new(), - PipelineName = pipelineName, + StepResults = stepResults ?? new(), + FlowName = flowName, }; } @@ -112,90 +112,90 @@ public static PipelineResult CreateFailure( /// Creates a successful dry run result. /// /// Time spent on pre-flight checks - /// Total number of nodes in the pipeline + /// Total number of steps in the flow /// Number of execution layers /// Number of external inputs validated - /// Name of the pipeline + /// Name of the flow /// A successful dry run result - public static PipelineResult CreateDryRunSuccess( + public static FlowResult CreateDryRunSuccess( TimeSpan preFlightDuration, - int nodeCount, + int stepCount, int layerCount, int validatedInputCount, - string? pipelineName = null + string? flowName = null ) { - return new PipelineResult + return new FlowResult { Success = true, IsDryRun = true, ExecutionTime = preFlightDuration, - NodeResults = new Dictionary(), - PipelineName = pipelineName, + StepResults = new Dictionary(), + FlowName = flowName, }; } } /// -/// Represents the execution result of a single pipeline node. +/// Represents the execution result of a single Flow step. /// -public class NodeResult +public class StepResult { /// - /// The name of the node that was executed. + /// The name of the step that was executed. /// - public required string NodeName { get; init; } + public required string StepName { get; init; } /// - /// Indicates whether the node executed successfully. + /// Indicates whether the step executed successfully. /// public bool Success { get; init; } /// - /// Execution time for this specific node. + /// Execution time for this specific step. /// public TimeSpan ExecutionTime { get; init; } /// - /// Exception that occurred during node execution, if any. + /// Exception that occurred during step execution, if any. /// /// /// Null if Success is true. Contains the exception that caused - /// the node to fail if Success is false. + /// the step to fail if Success is false. /// public Exception? Exception { get; init; } /// - /// Number of input items processed by this node. + /// Number of input items processed by this step. /// /// - /// For multi-input nodes, this represents the total count across + /// For multi-input steps, this represents the total count across /// all input catalog entries. /// public int InputCount { get; init; } /// - /// Number of output items produced by this node. + /// Number of output items produced by this step. /// /// - /// For multi-output nodes, this represents the total count across + /// For multi-output steps, this represents the total count across /// all output catalog entries. /// public int OutputCount { get; init; } /// - /// Creates a successful node result. + /// Creates a successful step result. /// - public static NodeResult CreateSuccess( - string nodeName, + public static StepResult CreateSuccess( + string stepName, TimeSpan executionTime, int inputCount, int outputCount ) { - return new NodeResult + return new StepResult { - NodeName = nodeName, + StepName = stepName, Success = true, ExecutionTime = executionTime, InputCount = inputCount, @@ -204,18 +204,18 @@ int outputCount } /// - /// Creates a failed node result. + /// Creates a failed step result. /// - public static NodeResult CreateFailure( - string nodeName, + public static StepResult CreateFailure( + string stepName, TimeSpan executionTime, Exception exception, int inputCount = 0 ) { - return new NodeResult + return new StepResult { - NodeName = nodeName, + StepName = stepName, Success = false, ExecutionTime = executionTime, Exception = exception, diff --git a/src/core/Flowthru.Core/Pipelines/PipelineSliceStrategy.cs b/src/core/Flowthru.Core/Flows/PipelineSliceStrategy.cs similarity index 78% rename from src/core/Flowthru.Core/Pipelines/PipelineSliceStrategy.cs rename to src/core/Flowthru.Core/Flows/PipelineSliceStrategy.cs index 9392ad6b..8a743417 100644 --- a/src/core/Flowthru.Core/Pipelines/PipelineSliceStrategy.cs +++ b/src/core/Flowthru.Core/Flows/PipelineSliceStrategy.cs @@ -1,11 +1,11 @@ -namespace Flowthru.Pipelines; +namespace Flowthru.Flows; /// -/// Defines a strategy for slicing a pipeline to execute a subset of nodes. +/// Defines a strategy for slicing a Flow to execute a subset of nodes. /// /// /// -/// Pipeline slicing allows executing only specific portions of a DAG while maintaining +/// Flow slicing allows executing only specific portions of a DAG while maintaining /// execution validity. All slicing operations preserve the runnability guarantee: /// the resulting sub-DAG must be executable without missing dependencies. /// @@ -13,7 +13,7 @@ namespace Flowthru.Pipelines; /// Slicing Strategies: /// /// -/// Pipelines: Filter to nodes from specific named pipelines (in merged DAGs) +/// flows: Filter to nodes from specific named flows (in merged DAGs) /// FromNodes: Include specified nodes and all downstream dependents /// ToNodes: Include specified nodes and all upstream dependencies (run "up to" these nodes) /// FromData: Include nodes consuming specified catalog entries and all downstream dependents @@ -31,17 +31,17 @@ namespace Flowthru.Pipelines; /// are not supported. /// /// -public sealed class PipelineSliceStrategy +public sealed class FlowSliceStrategy { /// - /// Filter to nodes from these named pipelines (applies to merged pipelines). + /// Filter to nodes from these named flows (applies to merged flows). /// /// - /// In merged pipelines, nodes are prefixed with their pipeline name (e.g., "DataScience.TrainModel"). - /// This filter includes only nodes from the specified pipelines. - /// Pipeline names are case-insensitive. + /// In merged flows, nodes are prefixed with their Flow name (e.g., "DataScience.TrainModel"). + /// This filter includes only nodes from the specified flows. + /// Flow names are case-insensitive. /// - public IReadOnlySet? Pipelines { get; init; } + public IReadOnlySet? Flows { get; init; } /// /// Start from these nodes, including all downstream dependents. @@ -58,7 +58,7 @@ public sealed class PipelineSliceStrategy /// /// Expands to include all transitive dependencies needed to run these nodes. /// Equivalent to "run everything up to and including these nodes". - /// Useful for testing specific outputs without running the entire pipeline. + /// Useful for testing specific outputs without running the entire flow. /// public IReadOnlySet? ToNodes { get; init; } @@ -68,6 +68,8 @@ public sealed class PipelineSliceStrategy /// /// Finds all nodes that read the specified catalog entries, then expands downstream. /// Useful for impact analysis - "what breaks if I change this data?" + /// + /// TODO: Remove this, as it is now sufficient to reference both steps and data entries as nodes. /// public IReadOnlySet? FromData { get; init; } @@ -77,6 +79,8 @@ public sealed class PipelineSliceStrategy /// /// Finds the nodes that write the specified catalog entries, then expands upstream. /// Useful for targeted execution - "run everything needed to produce this data". + /// + /// TODO: Remove this, as it is now sufficient to reference both steps and data entries as nodes. /// public IReadOnlySet? ToData { get; init; } @@ -93,7 +97,7 @@ public sealed class PipelineSliceStrategy /// Whether any slicing is configured. /// public bool IsSliced => - Pipelines != null + Flows != null || FromNodes != null || ToNodes != null || FromData != null @@ -101,7 +105,7 @@ public sealed class PipelineSliceStrategy || OnlyNodes != null; /// - /// No filtering - execute entire pipeline. + /// No filtering - execute entire flow. /// - public static PipelineSliceStrategy All() => new(); + public static FlowSliceStrategy All() => new(); } diff --git a/src/core/Flowthru.Core/Pipelines/Validation/IPipelineValidationHook.cs b/src/core/Flowthru.Core/Flows/Validation/IPipelineValidationHook.cs similarity index 65% rename from src/core/Flowthru.Core/Pipelines/Validation/IPipelineValidationHook.cs rename to src/core/Flowthru.Core/Flows/Validation/IPipelineValidationHook.cs index 9b276134..deb93363 100644 --- a/src/core/Flowthru.Core/Pipelines/Validation/IPipelineValidationHook.cs +++ b/src/core/Flowthru.Core/Flows/Validation/IPipelineValidationHook.cs @@ -1,22 +1,22 @@ using Flowthru.Data.Validation; -namespace Flowthru.Pipelines.Validation; +namespace Flowthru.Flows.Validation; /// -/// Validation hook that runs during pipeline pre-flight checks. +/// Validation hook that runs during Flow pre-flight checks. /// /// /// /// Validation hooks provide an extensibility point for extensions to contribute /// their own validation logic during the pre-flight phase. Hooks are invoked after /// DAG analysis but before external input inspection, allowing extensions to -/// validate their own node types. +/// validate their own step types. /// /// /// Example use cases: /// -/// Python extension validates @node decorators match C# types -/// Custom extensions validate node-specific configuration +/// Python extension validates @step decorators match C# types +/// Custom extensions validate step-specific configuration /// Third-party plugins validate external dependencies /// /// @@ -24,9 +24,9 @@ namespace Flowthru.Pipelines.Validation; /// Hook execution order: /// /// -/// Pipeline.Build() - DAG construction and layer assignment +/// Flow.Build() - DAG construction and layer assignment /// ValidationHooks.ValidateAsync() - Extension-specific validation -/// Pipeline.ValidateExternalInputsAsync() - External input inspection +/// Flow.ValidateExternalInputsAsync() - External input inspection /// /// /// Error handling: @@ -34,12 +34,12 @@ namespace Flowthru.Pipelines.Validation; /// Multiple hooks may run, and all errors are aggregated into a single result. /// /// -public interface IPipelineValidationHook +public interface IFlowValidationHook { /// - /// Validates pipeline nodes during pre-flight checks. + /// Validates Flow steps during pre-flight checks. /// - /// The pipeline being validated + /// The Flow being validated /// Cancellation token for async operations /// Validation result containing any errors found /// @@ -48,8 +48,8 @@ public interface IPipelineValidationHook /// /// Never throw exceptions (return errors in ValidationResult) /// Be idempotent (safe to call multiple times) - /// Be reasonably fast (executed during pre-flight, blocks pipeline start) - /// Only validate nodes they understand (ignore other node types) + /// Be reasonably fast (executed during pre-flight, blocks Flow start) + /// Only validate steps they understand (ignore other step types) /// /// /// @@ -57,17 +57,17 @@ public interface IPipelineValidationHook /// /// /// public async Task<ValidationResult> ValidateAsync( - /// Pipeline pipeline, + /// Flow flow, /// CancellationToken cancellationToken) /// { /// var result = ValidationResult.Success(); /// - /// foreach (var node in pipeline.Nodes) + /// foreach (var step in flow.Steps) /// { - /// if (IsPythonNode(node)) + /// if (IsPythonStep(step)) /// { - /// var nodeResult = await ValidatePythonNode(node, cancellationToken); - /// result.Merge(nodeResult); + /// var stepResult = await ValidatePythonStep(step, cancellationToken); + /// result.Merge(stepResult); /// } /// } /// @@ -75,5 +75,5 @@ public interface IPipelineValidationHook /// } /// /// - Task ValidateAsync(Pipeline pipeline, CancellationToken cancellationToken); + Task ValidateAsync(Flow flow, CancellationToken cancellationToken); } diff --git a/src/core/Flowthru.Core/Pipelines/Validation/ValidationOptions.cs b/src/core/Flowthru.Core/Flows/Validation/ValidationOptions.cs similarity index 93% rename from src/core/Flowthru.Core/Pipelines/Validation/ValidationOptions.cs rename to src/core/Flowthru.Core/Flows/Validation/ValidationOptions.cs index 3d343551..0f483e03 100644 --- a/src/core/Flowthru.Core/Pipelines/Validation/ValidationOptions.cs +++ b/src/core/Flowthru.Core/Flows/Validation/ValidationOptions.cs @@ -1,7 +1,7 @@ using Flowthru.Data; using Flowthru.Data.Validation; -namespace Flowthru.Pipelines.Validation; +namespace Flowthru.Flows.Validation; /// /// Configuration for pipeline validation behavior. @@ -10,7 +10,7 @@ namespace Flowthru.Pipelines.Validation; /// /// ValidationOptions provides pipeline-level overrides for validation configuration. /// The primary mechanism for validation configuration is catalog-level via the -/// property and the fluent +/// property and the fluent /// .WithInspectionLevel() API. /// /// @@ -26,7 +26,7 @@ namespace Flowthru.Pipelines.Validation; /// /// /// public ICatalogDataset<Company> Companies => -/// GetOrCreateDataset(() => new CsvCatalogDataset<Company>("companies", "data/companies.csv") +/// CreateDataset(() => new CsvCatalogDataset<Company>("companies", "data/companies.csv") /// .WithInspectionLevel(InspectionLevel.Deep)); /// /// @@ -64,7 +64,7 @@ public class ValidationOptions /// This configuration only applies to Layer 0 inputs (external data). /// Intermediate outputs are never inspected regardless of this setting. /// - public ValidationOptions Inspect(ICatalogEntry catalogEntry, InspectionLevel level) + public ValidationOptions Inspect(IItem catalogEntry, InspectionLevel level) { if (catalogEntry == null) { @@ -112,7 +112,7 @@ public ValidationOptions Inspect(ICatalogEntry catalogEntry, InspectionLevel lev /// (e.g., skipping validation in performance tests, or enabling deep validation during debugging). /// /// - internal InspectionLevel GetEffectiveInspectionLevel(ICatalogEntry catalogEntry) + internal InspectionLevel GetEffectiveInspectionLevel(IItem catalogEntry) { if (catalogEntry == null) { diff --git a/src/core/Flowthru.Core/Pipelines/ValidationDepth.cs b/src/core/Flowthru.Core/Flows/ValidationDepth.cs similarity index 94% rename from src/core/Flowthru.Core/Pipelines/ValidationDepth.cs rename to src/core/Flowthru.Core/Flows/ValidationDepth.cs index 8bea7440..24d1af46 100644 --- a/src/core/Flowthru.Core/Pipelines/ValidationDepth.cs +++ b/src/core/Flowthru.Core/Flows/ValidationDepth.cs @@ -1,4 +1,4 @@ -namespace Flowthru.Pipelines; +namespace Flowthru.Flows; /// /// Controls how deeply a dry run validates the pipeline before stopping. diff --git a/src/core/Flowthru.Core/Meta/Builders/DagBuilder.cs b/src/core/Flowthru.Core/Meta/Builders/DagBuilder.cs index bded1a66..1e0c7f91 100644 --- a/src/core/Flowthru.Core/Meta/Builders/DagBuilder.cs +++ b/src/core/Flowthru.Core/Meta/Builders/DagBuilder.cs @@ -1,6 +1,6 @@ using Flowthru.Data; +using Flowthru.Flows; using Flowthru.Meta.Models; -using Flowthru.Pipelines; namespace Flowthru.Meta.Builders; @@ -26,7 +26,7 @@ internal static class DagBuilder /// The pipeline to extract metadata from (must be built) /// Complete DAG metadata including nodes, catalog entries, and edges /// Thrown if pipeline is not built - public static DagMetadata Build(Pipeline pipeline) + public static DagMetadata Build(Flow pipeline) { if (!pipeline.IsBuilt) { @@ -36,21 +36,21 @@ public static DagMetadata Build(Pipeline pipeline) } // Always build from the full DAG to provide complete context - var allNodes = pipeline.NodesList; - var slicedNodes = pipeline.GetSlicedNodes(); + var allSteps = pipeline.StepsList; + var slicedSteps = pipeline.GetSlicedSteps(); var dag = new DagMetadata { - PipelineName = pipeline.Name ?? "UnnamedPipeline", + FlowName = pipeline.Name ?? "UnnamedPipeline", GeneratedAt = DateTime.UtcNow, AppliedSlice = pipeline.AppliedSlice != null ? DagSliceMetadata.FromStrategy(pipeline.AppliedSlice) : null, - SlicedNodeIds = slicedNodes != null ? slicedNodes.Select(n => n.Label).ToHashSet() : null, - SlicedCatalogEntryKeys = - slicedNodes != null - ? slicedNodes + SlicedStepIds = slicedSteps != null ? slicedSteps.Select(n => n.Label).ToHashSet() : null, + SlicedCatalogItemIds = + slicedSteps != null + ? slicedSteps .SelectMany(n => n.Outputs) - .SelectMany(ExpandCatalogEntry) + .SelectMany(ExpandCatalogItem) .Where(e => !e.Label.StartsWith("_nodata", StringComparison.OrdinalIgnoreCase)) .Select(e => GetQualifiedLabel(e)) .ToHashSet() @@ -58,214 +58,211 @@ public static DagMetadata Build(Pipeline pipeline) }; // Step 1: Extract all catalog entries from full DAG - var allCatalogEntries = ExtractCatalogEntries(allNodes); + var allCatalogItems = ExtractCatalogItems(allSteps); // Step 2: Build node metadata with layer information - dag.Nodes.AddRange(BuildNodeMetadata(allNodes)); + dag.Steps.AddRange(BuildStepMetadata(allSteps)); // Step 3: Build catalog entry metadata with producer-consumer relationships - dag.CatalogEntries.AddRange(BuildCatalogEntryMetadata(allCatalogEntries, dag.Nodes)); + dag.CatalogItems.AddRange(BuildCatalogItemMetadata(allCatalogItems, dag.Steps)); // Step 4: Generate edges representing data flow - dag.Edges.AddRange(BuildEdges(dag.Nodes, allCatalogEntries)); + dag.Edges.AddRange(BuildEdges(dag.Steps, allCatalogItems)); return dag; } /// - /// Extracts all unique catalog entries from the pipeline nodes. + /// Extracts all unique catalog items from the Flow steps. /// /// - /// Handles both simple catalog entries and CatalogMap entries by expanding - /// maps into their constituent catalog entries. + /// Handles both simple catalog items and CatalogMap items by expanding + /// maps into their constituent parts. /// - private static Dictionary ExtractCatalogEntries(List nodes) + private static Dictionary ExtractCatalogItems(List steps) { - var catalogEntries = new Dictionary(); + var catalogItems = new Dictionary(); - foreach (var node in nodes) + foreach (var step in steps) { // Process inputs - foreach (var input in node.Inputs) + foreach (var input in step.Inputs) { - AddCatalogEntry(catalogEntries, input); + AddCatalogItem(catalogItems, input); } // Process outputs - foreach (var output in node.Outputs) + foreach (var output in step.Outputs) { - AddCatalogEntry(catalogEntries, output); + AddCatalogItem(catalogItems, output); } } - return catalogEntries; + return catalogItems; } /// - /// Adds a catalog entry to the dictionary, expanding CatalogMaps if necessary. + /// Adds a catalog item to the dictionary, expanding CatalogMaps if necessary. /// - private static void AddCatalogEntry( - Dictionary catalogEntries, - ICatalogEntry entry - ) + private static void AddCatalogItem(Dictionary catalogItems, IItem item) { - // Skip _nodata entries (placeholder entries that don't represent actual data) - if (entry.Label.StartsWith("_nodata", StringComparison.OrdinalIgnoreCase)) + // Skip _nodata items (placeholder items that don't represent actual data) + if (item.Label.StartsWith("_nodata", StringComparison.OrdinalIgnoreCase)) { return; } - // Check if this is a CatalogMap (which needs to be expanded into individual entries) - var entryType = entry.GetType(); - if (entryType.IsGenericType && entryType.GetGenericTypeDefinition().Name == "CatalogMap`1") + // Check if this is a CatalogMap (which needs to be expanded into individual items) + var itemType = item.GetType(); + if (itemType.IsGenericType && itemType.GetGenericTypeDefinition().Name == "CatalogMap`1") { - // Use reflection to get the mapped entries from CatalogMap - var getMappedEntriesMethod = entryType.GetMethod( - "GetMappedEntries", + // Use reflection to get the mapped items from CatalogMap + var getMappedItemsMethod = itemType.GetMethod( + "GetMappedItems", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic ); - if (getMappedEntriesMethod?.Invoke(entry, null) is IEnumerable mappedEntries) + if (getMappedItemsMethod?.Invoke(item, null) is IEnumerable mappedItems) { - foreach (var mappedEntry in mappedEntries) + foreach (var mappedItem in mappedItems) { - // Skip _nodata entries in mapped entries too - if (!mappedEntry.Label.StartsWith("_nodata", StringComparison.OrdinalIgnoreCase)) + // Skip _nodata in mapped items too + if (!mappedItem.Label.StartsWith("_nodata", StringComparison.OrdinalIgnoreCase)) { - catalogEntries.TryAdd(GetQualifiedLabel(mappedEntry), mappedEntry); + catalogItems.TryAdd(GetQualifiedLabel(mappedItem), mappedItem); } } } } else { - // Simple catalog entry - catalogEntries.TryAdd(GetQualifiedLabel(entry), entry); + // Simple catalog item + catalogItems.TryAdd(GetQualifiedLabel(item), item); } } /// - /// Builds metadata for all nodes in the pipeline. + /// Builds metadata for all steps in the flow. /// - private static List BuildNodeMetadata(List nodes) + private static List BuildStepMetadata(List steps) { - var nodeMetadataList = new List(); + var stepMetadataList = new List(); - foreach (var pipelineNode in nodes) + foreach (var flowStep in steps) { - // Use node name directly (no longer extracting from instance type) - var nodeTypeName = pipelineNode.Label; + // Use step name directly (no longer extracting from instance type) + var stepLabel = flowStep.Label; // Get input catalog keys (expanding CatalogMaps, filtering _nodata) - var inputKeys = pipelineNode - .Inputs.SelectMany(ExpandCatalogEntry) + var inputKeys = flowStep + .Inputs.SelectMany(ExpandCatalogItem) .Where(e => !e.Label.StartsWith("_nodata", StringComparison.OrdinalIgnoreCase)) .Select(e => GetQualifiedLabel(e)) .ToList(); // Get output catalog keys (expanding CatalogMaps, filtering _nodata) - var outputKeys = pipelineNode - .Outputs.SelectMany(ExpandCatalogEntry) + var outputKeys = flowStep + .Outputs.SelectMany(ExpandCatalogItem) .Where(e => !e.Label.StartsWith("_nodata", StringComparison.OrdinalIgnoreCase)) .Select(e => GetQualifiedLabel(e)) .ToList(); - // Extract original pipeline name from node name if merged - // Merged nodes have format: "PipelineName.NodeName" - var originalPipelineName = ExtractOriginalPipelineName(pipelineNode.Label); + // Extract original flow name from Flow label if merged + // Merged nodes have format: "FlowName.StepName" + var originalFlowName = ExtractOriginalFlowName(flowStep.Label); - nodeMetadataList.Add( - new NodeMetadata + stepMetadataList.Add( + new StepMetadata { - Id = pipelineNode.Label, - Label = pipelineNode.Label, - NodeType = nodeTypeName, - Layer = pipelineNode.Layer, - PipelineName = originalPipelineName ?? "UnnamedPipeline", + Id = flowStep.Label, + Label = flowStep.Label, + StepType = stepLabel, + Layer = flowStep.Layer, + FlowName = originalFlowName ?? "UnnamedFlow", Inputs = inputKeys, Outputs = outputKeys, } ); } - return nodeMetadataList; + return stepMetadataList; } /// - /// Extracts the original pipeline name from a node name in a merged pipeline. + /// Extracts the original Flow name from a step name in a merged pipeline. /// - /// The node name (may be prefixed with pipeline name) - /// The original pipeline name if detected, otherwise null + /// The step name (may be prefixed with Flow name) + /// The original Flow name if detected, otherwise null /// - /// In merged pipelines, node names are prefixed with their original pipeline name + /// In merged flows, step names are prefixed with their original subflow name /// (e.g., "DataProcessing.PreprocessCompanies"). This method extracts that prefix. - /// For non-merged pipelines, returns null. + /// For non-merged flows, returns null. /// - private static string? ExtractOriginalPipelineName(string nodeName) + private static string? ExtractOriginalFlowName(string stepName) { - // Check if node name contains a dot (indicating it's from a merged pipeline) - var dotIndex = nodeName.IndexOf('.'); + // Check if step name contains a dot (indicating it's from a merged flow) + var dotIndex = stepName.IndexOf('.'); if (dotIndex > 0) { - // Extract the prefix before the first dot as the original pipeline name - return nodeName.Substring(0, dotIndex); + // Extract the prefix before the first dot as the original Flow name + return stepName.Substring(0, dotIndex); } - // No prefix found - use the current pipeline name - return "UnnamedPipeline"; + // No prefix found - use the current Flow name + return "UnnamedFlow"; } /// - /// Expands a catalog entry, returning multiple entries if it's a CatalogMap. + /// Expands a catalog item, returning multiple items if it's a CatalogMap. /// - private static IEnumerable ExpandCatalogEntry(ICatalogEntry entry) + private static IEnumerable ExpandCatalogItem(IItem item) { - var entryType = entry.GetType(); - if (entryType.IsGenericType && entryType.GetGenericTypeDefinition().Name == "CatalogMap`1") + var itemType = item.GetType(); + if (itemType.IsGenericType && itemType.GetGenericTypeDefinition().Name == "CatalogMap`1") { - var getMappedEntriesMethod = entryType.GetMethod( - "GetMappedEntries", + var getMappedItemsMethod = itemType.GetMethod( + "GetMappedItems", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic ); - if (getMappedEntriesMethod?.Invoke(entry, null) is IEnumerable mappedEntries) + if (getMappedItemsMethod?.Invoke(item, null) is IEnumerable mappedItems) { - return mappedEntries; + return mappedItems; } } - return new[] { entry }; + return new[] { item }; } /// - /// Builds metadata for all catalog entries with producer-consumer relationships. + /// Builds metadata for all catalog items with producer-consumer relationships. /// - private static List BuildCatalogEntryMetadata( - Dictionary catalogEntries, - List nodes + private static List BuildCatalogItemMetadata( + Dictionary catalogItems, + List steps ) { - var entries = new List(); + var entries = new List(); - foreach (var (key, entry) in catalogEntries) + foreach (var (key, entry) in catalogItems) { // Find producer (node that outputs this catalog entry) - var producer = nodes.FirstOrDefault(n => n.Outputs.Contains(key)); + var producer = steps.FirstOrDefault(n => n.Outputs.Contains(key)); // Find consumers (nodes that input this catalog entry) - var consumers = nodes.Where(n => n.Inputs.Contains(key)).Select(n => n.Id).ToList(); + var consumers = steps.Where(n => n.Inputs.Contains(key)).Select(n => n.Id).ToList(); // Extract simple type name from DataType var dataTypeName = GetSimpleTypeName(entry.DataType); // Build fields dictionary with additional metadata - var fields = BuildCatalogEntryFields(entry); + var fields = BuildCatalogItemFields(entry); // Generate schema (will be implemented in SchemaInference) var schema = SchemaInference.InferSchema(entry.DataType); entries.Add( - new CatalogEntryMetadata + new ItemMetadata { Key = key, Label = key, @@ -288,10 +285,10 @@ List nodes /// Extracts metadata like filepath, catalog type, read-only status, etc. /// using reflection to check for well-known properties. /// - private static Dictionary BuildCatalogEntryFields(ICatalogEntry entry) + private static Dictionary BuildCatalogItemFields(IItem item) { var fields = new Dictionary(); - var entryType = entry.GetType(); + var entryType = item.GetType(); // Add catalog type name fields["catalogType"] = GetSimpleTypeName(entryType); @@ -300,7 +297,7 @@ private static Dictionary BuildCatalogEntryFields(ICatalogEntry var filePathProperty = entryType.GetProperty("FilePath"); if (filePathProperty != null) { - var filePath = filePathProperty.GetValue(entry); + var filePath = filePathProperty.GetValue(item); if (filePath != null) { fields["filepath"] = filePath; @@ -308,7 +305,7 @@ private static Dictionary BuildCatalogEntryFields(ICatalogEntry } // Check if read-only using StorageTraits - var adapter = entry.GetType().GetProperty("Adapter")?.GetValue(entry); + var adapter = item.GetType().GetProperty("Adapter")?.GetValue(item); if (adapter != null) { var traitsProperty = adapter.GetType().GetProperty("Traits"); @@ -323,35 +320,35 @@ private static Dictionary BuildCatalogEntryFields(ICatalogEntry } // Get inspection level if configured - if (entry.PreferredInspectionLevel.HasValue) + if (item.PreferredInspectionLevel.HasValue) { - fields["inspectionLevel"] = entry.PreferredInspectionLevel.Value.ToString(); + fields["inspectionLevel"] = item.PreferredInspectionLevel.Value.ToString(); } return fields; } /// - /// Builds edges representing data flow in the DAG. + /// Builds edges representing data Flow in the DAG. /// /// /// Creates two types of edges: - /// 1. Catalog Entry → Node (node reads from catalog) - /// 2. Node → Catalog Entry (node writes to catalog) + /// 1. Catalog Item → Step (node reads from catalog) + /// 2. Step → Catalog Item (node writes to catalog) /// private static List BuildEdges( - List nodes, - Dictionary catalogEntries + List steps, + Dictionary catalogItems ) { var edges = new List(); - foreach (var node in nodes) + foreach (var step in steps) { - // Create edges for inputs (Catalog → Node) - foreach (var inputKey in node.Inputs) + // Create edges for inputs (Catalog Item → Step) + foreach (var inputKey in step.Inputs) { - if (catalogEntries.TryGetValue(inputKey, out var catalogEntry)) + if (catalogItems.TryGetValue(inputKey, out var catalogEntry)) { var dataTypeName = GetSimpleTypeName(catalogEntry.DataType); @@ -359,24 +356,24 @@ Dictionary catalogEntries new EdgeMetadata { Source = inputKey, - Target = node.Id, + Target = step.Id, DataType = dataTypeName, } ); } } - // Create edges for outputs (Node → Catalog) - foreach (var outputKey in node.Outputs) + // Create edges for outputs (Step → Catalog) + foreach (var outputKey in step.Outputs) { - if (catalogEntries.TryGetValue(outputKey, out var catalogEntry)) + if (catalogItems.TryGetValue(outputKey, out var catalogEntry)) { var dataTypeName = GetSimpleTypeName(catalogEntry.DataType); edges.Add( new EdgeMetadata { - Source = node.Id, + Source = step.Id, Target = outputKey, DataType = dataTypeName, } @@ -412,11 +409,11 @@ private static string GetSimpleTypeName(Type type) } /// - /// Returns the fully-qualified metadata label for a catalog entry. - /// When the entry was created via DataCatalogBase.GetOrCreateEntry the catalog's - /// CatalogLabel is prepended: "CatalogName.EntryLabel". - /// Falls back to entry.Label for entries created outside a catalog. + /// Returns the fully-qualified metadata label for a catalog item. + /// When the item was created via DataCatalogBase.CreateItem the catalog's + /// CatalogLabel is prepended: "CatalogName.ItemLabel". + /// Falls back to item.Label for items created outside a catalog. /// - private static string GetQualifiedLabel(ICatalogEntry entry) => - entry.OwningCatalogLabel is { } catalog ? $"{catalog}.{entry.Label}" : entry.Label; + private static string GetQualifiedLabel(IItem item) => + item.OwningCatalogLabel is { } catalog ? $"{catalog}.{item.Label}" : item.Label; } diff --git a/src/core/Flowthru.Core/Meta/FilenameTemplateParser.cs b/src/core/Flowthru.Core/Meta/FilenameTemplateParser.cs index 18428bcf..c4d77d0b 100644 --- a/src/core/Flowthru.Core/Meta/FilenameTemplateParser.cs +++ b/src/core/Flowthru.Core/Meta/FilenameTemplateParser.cs @@ -12,7 +12,7 @@ namespace Flowthru.Meta; /// Supports tokens for pipeline metadata and slice criteria: /// /// -/// {PipelineName} - Sanitized pipeline name +/// {FlowName} - Sanitized pipeline name /// {Timestamp} - Formatted timestamp (empty if disabled) /// {SliceType} - Slice descriptor: "FromNodes", "Tags", "Mixed", or empty /// {Pipelines} - Comma-separated list of pipelines @@ -32,7 +32,7 @@ namespace Flowthru.Meta; /// Example: /// /// -/// Template: "dag-{PipelineName}-{Timestamp}-{SliceType}" +/// Template: "dag-{FlowName}-{Timestamp}-{SliceType}" /// Unsliced: "dag-DataProcessing-20260304-153045" /// Sliced: "dag-DataProcessing-20260304-153045-FromNodes" /// @@ -57,7 +57,7 @@ public static string Render(DagMetadata dag, string template, string? timestamp) var token = match.Groups[1].Value; return token switch { - "PipelineName" => SanitizeFilename(dag.PipelineName), + "FlowName" => SanitizeFilename(dag.FlowName), "Timestamp" => timestamp ?? string.Empty, "SliceType" => dag.AppliedSlice?.GetSliceTypeDescriptor() ?? string.Empty, "Pipelines" => FormatList(dag.AppliedSlice?.Pipelines), diff --git a/src/core/Flowthru.Core/Meta/JsonMetadataProviderBuilder.cs b/src/core/Flowthru.Core/Meta/JsonMetadataProviderBuilder.cs index 86723c3b..aba82114 100644 --- a/src/core/Flowthru.Core/Meta/JsonMetadataProviderBuilder.cs +++ b/src/core/Flowthru.Core/Meta/JsonMetadataProviderBuilder.cs @@ -9,7 +9,7 @@ namespace Flowthru.Meta; public class JsonMetadataProviderBuilder { private string _outputDirectory = "metadata"; - private string _filenameTemplate = "dag-{PipelineName}-{Timestamp}-{SliceType}"; + private string _filenameTemplate = "dag-{FlowName}-{Timestamp}-{SliceType}"; private TimestampConfiguration _timestampConfig = new(); private bool _useCompactFormat = false; private ILogger? _logger; @@ -28,7 +28,7 @@ public JsonMetadataProviderBuilder WithOutputDirectory(string directory) /// /// Sets the filename template for metadata files. /// - /// Template with placeholders: {PipelineName}, {Timestamp}, {SliceType} + /// Template with placeholders: {FlowName}, {Timestamp}, {SliceType} /// This builder for fluent chaining public JsonMetadataProviderBuilder WithFilenameTemplate(string template) { diff --git a/src/core/Flowthru.Core/Meta/MermaidMetadataExtensions.cs b/src/core/Flowthru.Core/Meta/MermaidMetadataExtensions.cs index 14481674..19fe0a6b 100644 --- a/src/core/Flowthru.Core/Meta/MermaidMetadataExtensions.cs +++ b/src/core/Flowthru.Core/Meta/MermaidMetadataExtensions.cs @@ -13,11 +13,11 @@ namespace Flowthru.Meta; /// /// /// The generated diagram uses Mermaid flowchart syntax with: -/// - Nodes as rectangles with rounded corners -/// - Catalog entries as cylindrical database shapes -/// - Pipeline subgraphs grouping nodes by their origin pipeline +/// - Steps as rectangles with rounded corners +/// - Catalog items as cylindrical database shapes +/// - Flow subgraphs grouping nodes by their origin flow /// - External data (no producer) shown with special styling -/// - Produced data (has producer) inside their producer's pipeline subgraph +/// - Produced data (has producer) inside their producer's Flow subgraph /// /// public static class MermaidMetadataExtensions @@ -27,8 +27,8 @@ public static class MermaidMetadataExtensions /// /// The DAG metadata to visualize /// Flow direction code (TB, LR, BT, RL). Defaults to TB (Top to Bottom). - /// Hex color for active (sliced) nodes. Defaults to #2E7D32. - /// Hex color for active (sliced) catalog entries. Defaults to #2E7D32. + /// Hex color for active (sliced) steps. Defaults to #2E7D32. + /// Hex color for active (sliced) catalog items. Defaults to #2E7D32. /// Complete Markdown document with Mermaid code fence /// /// @@ -55,8 +55,8 @@ public static class MermaidMetadataExtensions public static string ToMermaidDiagram( this DagMetadata dag, string direction = "TB", - string activeNodeColor = "#2E7D32", - string activeDataColor = "#2E7D32" + string activeStepColor = "#2E7D32", + string activeItemColor = "#2E7D32" ) { var sb = new StringBuilder(); @@ -66,98 +66,96 @@ public static string ToMermaidDiagram( sb.AppendLine($"flowchart {direction}"); sb.AppendLine(); - // Classify catalog entries into external and produced - var externalEntries = dag.CatalogEntries.Where(e => string.IsNullOrEmpty(e.Producer)).ToList(); + // Classify catalog items into external and produced + var externalItems = dag.CatalogItems.Where(e => string.IsNullOrEmpty(e.Producer)).ToList(); - var producedEntries = dag.CatalogEntries.Where(e => !string.IsNullOrEmpty(e.Producer)).ToList(); + var producedItems = dag.CatalogItems.Where(e => !string.IsNullOrEmpty(e.Producer)).ToList(); // Define all external data inputs first (cylindrical database shape) - if (externalEntries.Any()) + if (externalItems.Any()) { sb.AppendLine(" %% External Data Inputs"); - foreach (var entry in externalEntries) + foreach (var item in externalItems) { - sb.AppendLine($" {SanitizeId(entry.Key)}[(\"{EscapeLabel(entry.Label)}\")]"); + sb.AppendLine($" {SanitizeId(item.Key)}[(\"{EscapeLabel(item.Label)}\")]"); } sb.AppendLine(); } - // Group nodes by pipeline - var pipelineGroups = dag.Nodes.GroupBy(n => n.PipelineName).OrderBy(g => g.Key); + // Group steps by flow + var flowGroups = dag.Steps.GroupBy(n => n.FlowName).OrderBy(g => g.Key); - foreach (var pipelineGroup in pipelineGroups) + foreach (var flowGroup in flowGroups) { - var pipelineName = pipelineGroup.Key; - var pipelineNodes = pipelineGroup.OrderBy(n => n.Layer).ThenBy(n => n.Id).ToList(); + var flowName = flowGroup.Key; + var flowSteps = flowGroup.OrderBy(n => n.Layer).ThenBy(n => n.Id).ToList(); - sb.AppendLine($" subgraph {SanitizeId(pipelineName)}[\"{EscapeLabel(pipelineName)}\"]"); + sb.AppendLine($" subgraph {SanitizeId(flowName)}[\"{EscapeLabel(flowName)}\"]"); - // Find produced catalog entries that belong to this pipeline - var pipelineCatalogEntries = producedEntries - .Where(e => pipelineNodes.Any(n => n.Id == e.Producer)) - .ToList(); + // Find produced catalog items that belong to this flow + var flowItems = producedItems.Where(e => flowSteps.Any(n => n.Id == e.Producer)).ToList(); - // Define nodes (rectangles) with styling for sliced nodes - foreach (var node in pipelineNodes) + // Define steps (rectangles) with styling for steps in the slice + foreach (var step in flowSteps) { - var nodeId = SanitizeId(node.Id); - var nodeLabel = EscapeLabel(node.Label); + var stepId = SanitizeId(step.Id); + var stepLabel = EscapeLabel(step.Label); - // Apply color fill to nodes in the execution slice - if (dag.SlicedNodeIds != null && dag.SlicedNodeIds.Contains(node.Id)) + // Apply color fill to steps in the execution slice + if (dag.SlicedStepIds != null && dag.SlicedStepIds.Contains(step.Id)) { - sb.AppendLine($" {nodeId}[\"{nodeLabel}\"]"); - sb.AppendLine($" style {nodeId} fill:{activeNodeColor}"); + sb.AppendLine($" {stepId}[\"{stepLabel}\"]"); + sb.AppendLine($" style {stepId} fill:{activeStepColor}"); } else { - sb.AppendLine($" {nodeId}[\"{nodeLabel}\"]"); + sb.AppendLine($" {stepId}[\"{stepLabel}\"]"); } } - // Define catalog entries produced by this pipeline (cylindrical database shape) - foreach (var entry in pipelineCatalogEntries) + // Define catalog items produced by this Flow (cylindrical database shape) + foreach (var item in flowItems) { - var entryId = SanitizeId(entry.Key); - var entryLabel = EscapeLabel(entry.Label); + var itemId = SanitizeId(item.Key); + var itemLabel = EscapeLabel(item.Label); - sb.AppendLine($" {entryId}[(\"{entryLabel}\")]"); + sb.AppendLine($" {itemId}[(\"{itemLabel}\")]"); - // Apply color fill to catalog entries in the execution slice - if (dag.SlicedCatalogEntryKeys != null && dag.SlicedCatalogEntryKeys.Contains(entry.Key)) + // Apply color fill to catalog items in the execution slice + if (dag.SlicedCatalogItemIds != null && dag.SlicedCatalogItemIds.Contains(item.Key)) { - sb.AppendLine($" style {entryId} fill:{activeDataColor}"); + sb.AppendLine($" style {itemId} fill:{activeItemColor}"); } } sb.AppendLine(); - // Generate edges for this pipeline - foreach (var node in pipelineNodes) + // Generate edges for this flow + foreach (var step in flowSteps) { - // Input edges - only include if the input is produced by this pipeline (not external!) - foreach (var input in node.Inputs) + // Input edges - only include if the input is produced by this Flow (not external!) + foreach (var input in step.Inputs) { - var inputEntry = dag.CatalogEntries.FirstOrDefault(e => e.Key == input); - if (inputEntry != null) + var inputItem = dag.CatalogItems.FirstOrDefault(e => e.Key == input); + if (inputItem != null) { - var isProducedByThisPipeline = pipelineCatalogEntries.Any(e => e.Key == input); + var isProducedByThisFlow = flowItems.Any(e => e.Key == input); - // Only include edges from data produced within this pipeline - if (isProducedByThisPipeline) + // Only include edges from data produced within this flow + if (isProducedByThisFlow) { - sb.AppendLine($" {SanitizeId(input)} --> {SanitizeId(node.Id)}"); + sb.AppendLine($" {SanitizeId(input)} --> {SanitizeId(step.Id)}"); } } } - // Output edges - node to its produced catalog entries - foreach (var output in node.Outputs) + // Output edges - step to its produced catalog items + foreach (var output in step.Outputs) { - var catalogEntry = pipelineCatalogEntries.FirstOrDefault(e => e.Key == output); - if (catalogEntry != null) + var catalogItem = flowItems.FirstOrDefault(e => e.Key == output); + if (catalogItem != null) { - sb.AppendLine($" {SanitizeId(node.Id)} --> {SanitizeId(output)}"); + sb.AppendLine($" {SanitizeId(step.Id)} --> {SanitizeId(output)}"); } } } @@ -166,47 +164,47 @@ public static string ToMermaidDiagram( sb.AppendLine(); } - // Generate external data to node edges (outside subgraphs) - sb.AppendLine(" %% External Data to Pipeline Edges"); - foreach (var entry in externalEntries) + // Generate external data to step edges (outside subgraphs) + sb.AppendLine(" %% External Data to Flow Edges"); + foreach (var item in externalItems) { - foreach (var consumer in entry.Consumers) + foreach (var consumer in item.Consumers) { - var consumerNode = dag.Nodes.FirstOrDefault(n => n.Id == consumer); - if (consumerNode != null) + var consumerStep = dag.Steps.FirstOrDefault(n => n.Id == consumer); + if (consumerStep != null) { - sb.AppendLine($" {SanitizeId(entry.Key)} --> {SanitizeId(consumer)}"); + sb.AppendLine($" {SanitizeId(item.Key)} --> {SanitizeId(consumer)}"); } } } sb.AppendLine(); - // Generate cross-pipeline edges (catalog entries that connect different pipelines) - var crossPipelineEdges = new List<(string source, string target)>(); + // Generate cross-flow edges (catalog items that connect different flows) + var crossFlowEdges = new List<(string source, string target)>(); - foreach (var entry in producedEntries) + foreach (var item in producedItems) { - var producerNode = dag.Nodes.FirstOrDefault(n => n.Id == entry.Producer); - if (producerNode == null) + var producerStep = dag.Steps.FirstOrDefault(n => n.Id == item.Producer); + if (producerStep == null) { continue; } - foreach (var consumer in entry.Consumers) + foreach (var consumer in item.Consumers) { - var consumerNode = dag.Nodes.FirstOrDefault(n => n.Id == consumer); - if (consumerNode != null && consumerNode.PipelineName != producerNode.PipelineName) + var consumerStep = dag.Steps.FirstOrDefault(n => n.Id == consumer); + if (consumerStep != null && consumerStep.FlowName != producerStep.FlowName) { - // This catalog entry connects two different pipelines - crossPipelineEdges.Add((entry.Key, consumer)); + // This catalog item connects two different flows + crossFlowEdges.Add((item.Key, consumer)); } } } - if (crossPipelineEdges.Any()) + if (crossFlowEdges.Any()) { - sb.AppendLine(" %% Cross-Pipeline Data Flow"); - foreach (var (source, target) in crossPipelineEdges.Distinct()) + sb.AppendLine(" %% Cross-Flow Data Flow"); + foreach (var (source, target) in crossFlowEdges.Distinct()) { sb.AppendLine($" {SanitizeId(source)} -.-> {SanitizeId(target)}"); } diff --git a/src/core/Flowthru.Core/Meta/MermaidMetadataProviderBuilder.cs b/src/core/Flowthru.Core/Meta/MermaidMetadataProviderBuilder.cs index 26549f9b..85f47c93 100644 --- a/src/core/Flowthru.Core/Meta/MermaidMetadataProviderBuilder.cs +++ b/src/core/Flowthru.Core/Meta/MermaidMetadataProviderBuilder.cs @@ -9,12 +9,12 @@ namespace Flowthru.Meta; public class MermaidMetadataProviderBuilder { private string _outputDirectory = "metadata"; - private string _filenameTemplate = "dag-{PipelineName}-{Timestamp}-{SliceType}"; + private string _filenameTemplate = "dag-{FlowName}-{Timestamp}-{SliceType}"; private TimestampConfiguration _timestampConfig = new(); private MermaidMetadataProvider.MermaidFlowchartDirection _direction = MermaidMetadataProvider .MermaidFlowchartDirection .TopToBottom; - private string _activeNodeColor = "#2E7D32"; + private string _activeStepColor = "#2E7D32"; private string _activeDataColor = "#2E7D32"; private ILogger? _logger; @@ -32,7 +32,7 @@ public MermaidMetadataProviderBuilder WithOutputDirectory(string directory) /// /// Sets the filename template for metadata files. /// - /// Template with placeholders: {PipelineName}, {Timestamp}, {SliceType} + /// Template with placeholders: {FlowName}, {Timestamp}, {SliceType} /// This builder for fluent chaining public MermaidMetadataProviderBuilder WithFilenameTemplate(string template) { @@ -69,9 +69,9 @@ MermaidMetadataProvider.MermaidFlowchartDirection direction /// /// Hex color code (e.g., "#2E7D32") /// This builder for fluent chaining - public MermaidMetadataProviderBuilder WithActiveNodeColor(string color) + public MermaidMetadataProviderBuilder WithActiveStepColor(string color) { - _activeNodeColor = color ?? throw new ArgumentNullException(nameof(color)); + _activeStepColor = color ?? throw new ArgumentNullException(nameof(color)); return this; } @@ -108,7 +108,7 @@ public MermaidMetadataProvider Build() _filenameTemplate, _timestampConfig, _direction, - _activeNodeColor, + _activeStepColor, _activeDataColor, _logger ); diff --git a/src/core/Flowthru.Core/Meta/MetadataJsonExtensions.cs b/src/core/Flowthru.Core/Meta/MetadataJsonExtensions.cs index 5b98a28e..814f4dbb 100644 --- a/src/core/Flowthru.Core/Meta/MetadataJsonExtensions.cs +++ b/src/core/Flowthru.Core/Meta/MetadataJsonExtensions.cs @@ -33,7 +33,7 @@ public static class MetadataJsonExtensions /// Output format uses: /// /// - /// camelCase property names (pipelineName, not PipelineName) + /// camelCase property names (pipelineName, not FlowName) /// Indented formatting for readability /// Null properties omitted /// Enums serialized as strings diff --git a/src/core/Flowthru.Core/Meta/Models/CatalogEntryMetadata.cs b/src/core/Flowthru.Core/Meta/Models/CatalogEntryMetadata.cs index 2a461ab6..2aeb3c8e 100644 --- a/src/core/Flowthru.Core/Meta/Models/CatalogEntryMetadata.cs +++ b/src/core/Flowthru.Core/Meta/Models/CatalogEntryMetadata.cs @@ -3,17 +3,17 @@ namespace Flowthru.Meta.Models; /// -/// Metadata describing a single catalog entry (dataset) in the pipeline. +/// Metadata describing a single catalog item (dataset, json object, API response, whatever) in the flow. /// /// -/// Catalog entries represent data sources and sinks. They can be external files, -/// intermediate pipeline outputs, or final results. Each entry is uniquely identified +/// Catalog items represent data sources and sinks. They can be external files, +/// intermediate Flow outputs, or final results. Each item is uniquely identified /// by its key. /// -public class CatalogEntryMetadata +public class ItemMetadata { /// - /// Unique key identifying this catalog entry. + /// Unique key identifying this catalog item. /// /// /// Corresponds to the catalog property name or explicitly set key. @@ -23,7 +23,7 @@ public class CatalogEntryMetadata public required string Key { get; init; } /// - /// Human-readable display label for this catalog entry. + /// Human-readable display label for this catalog item. /// /// /// May be formatted for better display in Flowthru.Viz. @@ -33,7 +33,7 @@ public class CatalogEntryMetadata public required string Label { get; init; } /// - /// The C# type name of data stored in this catalog entry. + /// The C# type name of data stored in this catalog item. /// /// /// Simple type name without namespace. @@ -53,7 +53,7 @@ public class CatalogEntryMetadata public SchemaMetadata? Schema { get; init; } /// - /// Additional metadata fields specific to the catalog entry type. + /// Additional metadata fields specific to the catalog item type. /// /// /// Examples of fields: @@ -68,20 +68,20 @@ public class CatalogEntryMetadata public Dictionary Fields { get; init; } = new(); /// - /// Node ID that produces (writes to) this catalog entry. + /// Step ID that produces (writes to) this catalog item. /// /// - /// Null for external inputs (Layer 0 inputs that exist before pipeline execution). + /// Null for external inputs (Layer 0 inputs that exist before Flow execution). /// Example: "PreprocessCompanies" /// [JsonPropertyName("producer")] public string? Producer { get; init; } /// - /// List of node IDs that consume (read from) this catalog entry. + /// List of step IDs that consume (read from) this catalog item. /// /// - /// Empty for pipeline outputs that aren't consumed by other nodes. + /// Empty for item outputs that aren't consumed by other steps. /// Example: ["CreateModelInputTable", "ValidateData"] /// [JsonPropertyName("consumers")] diff --git a/src/core/Flowthru.Core/Meta/Models/DagMetadata.cs b/src/core/Flowthru.Core/Meta/Models/DagMetadata.cs index f04d779f..816e5d8b 100644 --- a/src/core/Flowthru.Core/Meta/Models/DagMetadata.cs +++ b/src/core/Flowthru.Core/Meta/Models/DagMetadata.cs @@ -3,20 +3,20 @@ namespace Flowthru.Meta.Models; /// -/// Root metadata model representing a complete pipeline DAG (Directed Acyclic Graph). +/// Root metadata model representing a complete FlowthruService DAG (Directed Acyclic Graph). /// /// -/// This model captures the structure of a built pipeline, including all nodes, -/// catalog entries, and their relationships. It serves as the backbone for +/// This model captures the structure of a built flow, including all steps, +/// catalog items, and their relationships. It serves as the backbone for /// Flowthru.Viz visualization. /// public class DagMetadata { /// - /// Name of the pipeline this DAG represents. + /// Name of the Flow this DAG represents. /// - [JsonPropertyName("pipelineName")] - public required string PipelineName { get; init; } + [JsonPropertyName("flowName")] + public required string FlowName { get; init; } /// /// Timestamp when this metadata was generated. @@ -25,23 +25,23 @@ public class DagMetadata public DateTime GeneratedAt { get; init; } = DateTime.UtcNow; /// - /// All nodes in the pipeline with their metadata. + /// All steps in the Flow with their metadata. /// - [JsonPropertyName("nodes")] - public List Nodes { get; init; } = new(); + [JsonPropertyName("steps")] + public List Steps { get; init; } = new(); /// - /// All catalog entries (datasets) involved in the pipeline. + /// All catalog items involved in the flow. /// - [JsonPropertyName("catalogEntries")] - public List CatalogEntries { get; init; } = new(); + [JsonPropertyName("catalogItems")] + public List CatalogItems { get; init; } = new(); /// - /// All edges representing data flow in the DAG. + /// All edges representing data Flow in the DAG. /// /// - /// Edges connect catalog entries to nodes and nodes to catalog entries, - /// forming the complete data flow graph. + /// Edges connect catalog items to steps and steps to catalog items, + /// forming the complete graph. /// [JsonPropertyName("edges")] public List Edges { get; init; } = new(); @@ -50,8 +50,8 @@ public class DagMetadata /// Slice criteria applied to generate this DAG, if any. /// /// - /// Present when the DAG represents a sliced subset of the full pipeline. - /// Null when the DAG represents the complete, unsliced pipeline. + /// Present when the DAG represents a sliced subset of the full Flow. + /// Null when the DAG represents the complete, unsliced flow. /// Used for reproducibility, debugging, and filename generation. /// [JsonPropertyName("appliedSlice")] @@ -59,28 +59,28 @@ public class DagMetadata public DagSliceMetadata? AppliedSlice { get; init; } /// - /// Node IDs that are in the active execution slice, if a slice was applied. + /// Step IDs that are in the active execution slice, if a slice was applied. /// /// - /// When a slice is applied, this contains the IDs of nodes that will actually execute. - /// The Nodes collection contains the full DAG, while this set identifies the subset. - /// Null when no slice was applied (all nodes execute). + /// When a slice is applied, this contains the IDs of steps that will actually execute. + /// The Steps collection contains the full DAG, while this set identifies the subset. + /// Null when no slice was applied (all steps execute). /// Enables visualization tools to highlight execution paths while showing full context. /// - [JsonPropertyName("slicedNodeIds")] + [JsonPropertyName("slicedStepIds")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public HashSet? SlicedNodeIds { get; init; } + public HashSet? SlicedStepIds { get; init; } /// - /// Catalog entry keys that are produced by nodes in the active execution slice. + /// Catalog item IDsthat are produced by steps in the active execution slice. /// /// - /// When a slice is applied, this contains the keys of catalog entries (data) that - /// will be written during execution. Derived from the outputs of sliced nodes. + /// When a slice is applied, this contains the keys of catalog items (data) that + /// will be written during execution. Derived from the outputs of sliced steps. /// Null when no slice was applied (all data may be updated). - /// Enables visualization tools to highlight both nodes and the data they produce. + /// Enables visualization tools to highlight both steps and the data they produce. /// - [JsonPropertyName("slicedCatalogEntryKeys")] + [JsonPropertyName("slicedCatalogItemIds")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public HashSet? SlicedCatalogEntryKeys { get; init; } + public HashSet? SlicedCatalogItemIds { get; init; } } diff --git a/src/core/Flowthru.Core/Meta/Models/DagSliceMetadata.cs b/src/core/Flowthru.Core/Meta/Models/DagSliceMetadata.cs index 488d865b..4f6ca096 100644 --- a/src/core/Flowthru.Core/Meta/Models/DagSliceMetadata.cs +++ b/src/core/Flowthru.Core/Meta/Models/DagSliceMetadata.cs @@ -1,5 +1,5 @@ using System.Text.Json.Serialization; -using Flowthru.Pipelines; +using Flowthru.Flows; namespace Flowthru.Meta.Models; @@ -61,9 +61,9 @@ public class DagSliceMetadata public string[]? OnlyNodes { get; init; } /// - /// Creates a from a . + /// Creates a from a . /// - internal static DagSliceMetadata? FromStrategy(PipelineSliceStrategy? strategy) + internal static DagSliceMetadata? FromStrategy(FlowSliceStrategy? strategy) { if (strategy == null) { @@ -72,7 +72,7 @@ public class DagSliceMetadata return new DagSliceMetadata { - Pipelines = strategy.Pipelines?.ToArray(), + Pipelines = strategy.Flows?.ToArray(), FromNodes = strategy.FromNodes?.ToArray(), ToNodes = strategy.ToNodes?.ToArray(), FromData = strategy.FromData?.ToArray(), diff --git a/src/core/Flowthru.Core/Meta/Models/EdgeMetadata.cs b/src/core/Flowthru.Core/Meta/Models/EdgeMetadata.cs index 6bffb075..d4a0a780 100644 --- a/src/core/Flowthru.Core/Meta/Models/EdgeMetadata.cs +++ b/src/core/Flowthru.Core/Meta/Models/EdgeMetadata.cs @@ -7,16 +7,16 @@ namespace Flowthru.Meta.Models; /// /// /// -/// Edges represent data flow between catalog entries and nodes. The DAG contains +/// Edges represent data Flow between catalog entries and nodes. The DAG contains /// two types of edges: /// /// -/// Catalog → Node: A node reads from a catalog entry -/// Node → Catalog: A node writes to a catalog entry +/// Catalog → Step: A node reads from a catalog entry +/// Step → Catalog: A node writes to a catalog entry /// /// /// Together, these edges form the complete data flow: -/// CatalogEntry → Node → CatalogEntry → Node → ... +/// Item → Step → Item → Step → ... /// /// public class EdgeMetadata @@ -25,8 +25,8 @@ public class EdgeMetadata /// Source identifier (either a catalog entry key or node ID). /// /// - /// For Catalog → Node edges, this is a catalog entry key. - /// For Node → Catalog edges, this is a node ID. + /// For Catalog → Step edges, this is a catalog entry key. + /// For Step → Catalog edges, this is a node ID. /// [JsonPropertyName("source")] public required string Source { get; init; } @@ -35,8 +35,8 @@ public class EdgeMetadata /// Target identifier (either a node ID or catalog entry key). /// /// - /// For Catalog → Node edges, this is a node ID. - /// For Node → Catalog edges, this is a catalog entry key. + /// For Catalog → Step edges, this is a node ID. + /// For Step → Catalog edges, this is a catalog entry key. /// [JsonPropertyName("target")] public required string Target { get; init; } diff --git a/src/core/Flowthru.Core/Meta/Models/NodeMetadata.cs b/src/core/Flowthru.Core/Meta/Models/NodeMetadata.cs index 495d938e..4c857456 100644 --- a/src/core/Flowthru.Core/Meta/Models/NodeMetadata.cs +++ b/src/core/Flowthru.Core/Meta/Models/NodeMetadata.cs @@ -3,27 +3,27 @@ namespace Flowthru.Meta.Models; /// -/// Metadata describing a single node in the pipeline DAG. +/// Metadata describing a single step in the Flow DAG. /// /// -/// Nodes are the processing units in a pipeline. Each node reads from one or more +/// Steps are the processing units in a flow. Each step reads from one or more /// catalog entries (inputs), performs a transformation, and writes to one or more /// catalog entries (outputs). /// -public class NodeMetadata +public class StepMetadata { /// - /// Unique identifier for this node within the pipeline. + /// Unique identifier for this step within the flow. /// /// - /// Typically the node name as defined when adding it to the pipeline. + /// Typically the step name as defined when adding it to the flow. /// Example: "PreprocessCompanies", "TrainModel" /// [JsonPropertyName("id")] public required string Id { get; init; } /// - /// Human-readable display label for this node. + /// Human-readable display label for this step. /// /// /// May be formatted for better display in Flowthru.Viz. @@ -33,50 +33,50 @@ public class NodeMetadata public required string Label { get; init; } /// - /// The C# class type name implementing this node. + /// The C# class type name implementing this step. /// /// /// Simple type name without namespace or generic parameters. - /// Example: "PreprocessCompaniesNode", "TrainModelNode" + /// Example: "PreprocessCompaniesStep", "TrainModelStep" /// - [JsonPropertyName("nodeType")] - public required string NodeType { get; init; } + [JsonPropertyName("stepType")] + public required string StepType { get; init; } /// /// Execution layer assigned by the dependency analyzer. /// /// - /// Layer 0 nodes have no dependencies (read external data only). - /// Layer N nodes depend only on nodes in layers 0..N-1. + /// Layer 0 steps have no dependencies (read external data only). + /// Layer N steps depend only on steps in layers 0..N-1. /// [JsonPropertyName("layer")] public int Layer { get; init; } /// - /// Name of the parent pipeline this node belongs to. + /// Name of the parent Flow this step belongs to. /// /// - /// Important for merged pipelines where nodes from multiple pipelines + /// Important for merged flows where steps from multiple flows /// are combined into a single DAG. /// - [JsonPropertyName("pipelineName")] - public required string PipelineName { get; init; } + [JsonPropertyName("flowName")] + public required string FlowName { get; init; } /// - /// List of catalog entry keys this node reads from. + /// List of catalog entry keys this step reads from. /// /// - /// For multi-input nodes using CatalogMap, this contains all mapped entries. + /// For multi-input steps using CatalogMap, this contains all mapped entries. /// Example: ["Companies", "Shuttles", "Reviews"] /// [JsonPropertyName("inputs")] public List Inputs { get; init; } = new(); /// - /// List of catalog entry keys this node writes to. + /// List of catalog entry keys this step writes to. /// /// - /// For multi-output nodes using CatalogMap, this contains all mapped entries. + /// For multi-output steps using CatalogMap, this contains all mapped entries. /// Example: ["XTrain", "XTest", "YTrain", "YTest"] /// [JsonPropertyName("outputs")] diff --git a/src/core/Flowthru.Core/Meta/Providers/JsonMetadataProvider.cs b/src/core/Flowthru.Core/Meta/Providers/JsonMetadataProvider.cs index 1a2b7854..cb35d6cb 100644 --- a/src/core/Flowthru.Core/Meta/Providers/JsonMetadataProvider.cs +++ b/src/core/Flowthru.Core/Meta/Providers/JsonMetadataProvider.cs @@ -80,9 +80,9 @@ public void Consume(DagMetadata dag) File.Move(tempPath, filePath); _logger?.LogInformation( - "Successfully exported JSON metadata ({Nodes} nodes, {Entries} catalog entries, {Edges} edges)", - dag.Nodes.Count, - dag.CatalogEntries.Count, + "Successfully exported JSON metadata ({Steps} nodes, {Entries} catalog entries, {Edges} edges)", + dag.Steps.Count, + dag.CatalogItems.Count, dag.Edges.Count ); } diff --git a/src/core/Flowthru.Core/Meta/Providers/MermaidMetadataProvider.cs b/src/core/Flowthru.Core/Meta/Providers/MermaidMetadataProvider.cs index 2e4994f7..ab29434c 100644 --- a/src/core/Flowthru.Core/Meta/Providers/MermaidMetadataProvider.cs +++ b/src/core/Flowthru.Core/Meta/Providers/MermaidMetadataProvider.cs @@ -14,7 +14,7 @@ namespace Flowthru.Meta.Providers; public class MermaidMetadataProvider : IMetadataProvider { private readonly MermaidFlowchartDirection _direction; - private readonly string _activeNodeColor; + private readonly string _activeStepColor; private readonly string _activeDataColor; private readonly string _outputDirectory; private readonly string _filenameTemplate; @@ -46,7 +46,7 @@ public enum MermaidFlowchartDirection /// Template for generating output filenames /// Configuration for timestamp handling in filenames /// Flow direction for the diagram - /// Hex color for active (sliced) nodes + /// Hex color for active (sliced) nodes /// Hex color for active (sliced) catalog entries /// Optional logger for diagnostic messages public MermaidMetadataProvider( @@ -54,7 +54,7 @@ public MermaidMetadataProvider( string filenameTemplate, TimestampConfiguration timestampConfig, MermaidFlowchartDirection direction = MermaidFlowchartDirection.TopToBottom, - string activeNodeColor = "#2E7D32", + string activeStepColor = "#2E7D32", string activeDataColor = "#2E7D32", ILogger? logger = null ) @@ -64,7 +64,7 @@ public MermaidMetadataProvider( filenameTemplate ?? throw new ArgumentNullException(nameof(filenameTemplate)); _timestampConfig = timestampConfig ?? throw new ArgumentNullException(nameof(timestampConfig)); _direction = direction; - _activeNodeColor = activeNodeColor; + _activeStepColor = activeStepColor; _activeDataColor = activeDataColor; _logger = logger; } @@ -90,7 +90,7 @@ public void Consume(DagMetadata dag) // Generate Mermaid diagram with configured direction and colors var mermaid = dag.ToMermaidDiagram( GetDirectionCode(_direction), - _activeNodeColor, + _activeStepColor, _activeDataColor ); @@ -137,7 +137,7 @@ public void Consume(DagMetadata dag) } /// - /// Converts flow direction enum to Mermaid direction code. + /// Converts Flow direction enum to Mermaid direction code. /// private static string GetDirectionCode(MermaidFlowchartDirection direction) { diff --git a/src/core/Flowthru.Core/Nodes/Factory/NodeFactory.cs b/src/core/Flowthru.Core/Nodes/Factory/NodeFactory.cs deleted file mode 100644 index 8f447934..00000000 --- a/src/core/Flowthru.Core/Nodes/Factory/NodeFactory.cs +++ /dev/null @@ -1,42 +0,0 @@ -namespace Flowthru.Nodes.Factory; - -/// -/// Factory for creating node instances using TypeActivator. -/// -/// -/// -/// Design Pattern: Factory Pattern - provides a centralized location for -/// node instantiation logic. -/// -/// -/// This is a thin wrapper around TypeActivator, providing a domain-specific API for -/// creating nodes. Could be extended in the future with: -/// - Node validation logic -/// - Pre/post-creation hooks -/// - Node decoration/wrapping -/// -/// -public static class NodeFactory -{ - /// - /// Creates a new instance of the specified node type. - /// - /// The node type to instantiate - /// A new node instance - /// - /// - /// Requirements: - /// - TNode must inherit from NodeBase<TInput, TOutput> - /// - TNode must have a parameterless constructor - /// - /// - /// These requirements are enforced at compile-time via generic constraints in - /// PipelineBuilder.AddNode methods. - /// - /// - public static TNode Create() - where TNode : new() - { - return TypeActivator.Create(); - } -} diff --git a/src/core/Flowthru.Core/Registry/IPipelineRegistrar.cs b/src/core/Flowthru.Core/Registry/IPipelineRegistrar.cs index 4aca3702..432e16e8 100644 --- a/src/core/Flowthru.Core/Registry/IPipelineRegistrar.cs +++ b/src/core/Flowthru.Core/Registry/IPipelineRegistrar.cs @@ -1,28 +1,28 @@ using Flowthru.Data; -using Flowthru.Pipelines; +using Flowthru.Flows; namespace Flowthru.Registry; /// -/// Fluent interface for registering pipelines in a type-safe manner. +/// Fluent interface for registering flows in a type-safe manner. /// -/// The catalog type that pipelines will use +/// The catalog type that flows will use /// /// -/// This interface provides compile-time type safety by tying pipeline factories +/// This interface provides compile-time type safety by tying Flow factories /// to a specific catalog type. The registrar validates that all registered -/// pipelines accept the correct catalog type. +/// flows accept the correct catalog type. /// /// /// Usage: /// -/// protected override void RegisterPipelines(IPipelineRegistrar<MyCatalog> registrar) +/// protected override void RegisterFlows(IFlowRegistrar<MyCatalog> registrar) /// { -/// // Pipeline without parameters -/// registrar.Register("processing", ProcessingPipeline.Create); +/// // Flow without parameters +/// registrar.Register("processing", ProcessingFlow.Create); /// -/// // Pipeline with parameters -/// registrar.Register("training", TrainPipeline.Create, new TrainOptions()); +/// // Flow with parameters +/// registrar.Register("training", TrainFlow.Create, new TrainOptions()); /// /// // Add metadata /// registrar.WithDescription("processing", "Cleans and transforms raw data"); @@ -30,55 +30,55 @@ namespace Flowthru.Registry; /// /// /// -public interface IPipelineRegistrar - where TCatalog : DataCatalogBase +public interface IFlowRegistrar + where TCatalog : CatalogAbstract { /// - /// Registers a pipeline with a parameterless factory function. + /// Registers a Flow with a parameterless factory function. /// - /// Unique pipeline name - /// Factory function that creates the pipeline from catalog + /// Unique Flow name + /// Factory function that creates the Flow from catalog /// This registrar for method chaining /// - /// Use this overload when the pipeline doesn't require parameters. + /// Use this overload when the Flow doesn't require parameters. /// - IPipelineRegistrar Register(string name, Func pipelineFactory); + IFlowRegistrar Register(string name, Func flowFactory); /// - /// Registers a pipeline with a parameterized factory function. + /// Registers a Flow with a parameterized factory function. /// - /// The type of parameters the pipeline requires - /// Unique pipeline name - /// Factory function that creates the pipeline from catalog and parameters - /// Parameter instance to pass to the pipeline + /// The type of parameters the Flow requires + /// Unique Flow name + /// Factory function that creates the Flow from catalog and parameters + /// Parameter instance to pass to the flow /// This registrar for method chaining /// /// - /// Use this overload when the pipeline requires configuration parameters. + /// Use this overload when the Flow requires configuration parameters. /// Parameters are strongly typed and checked at compile time. /// /// - /// The factory signature must match: Func<TCatalog, TParams, Pipeline> + /// The factory signature must match: Func<TCatalog, TParams, Flow> /// /// - IPipelineRegistrar Register( + IFlowRegistrar Register( string name, - Func pipelineFactory, + Func flowFactory, TParams parameters ); /// - /// Adds a description to the most recently registered pipeline. + /// Adds a description to the most recently registered flow. /// - /// Human-readable description of what the pipeline does + /// Human-readable description of what the Flow does /// This registrar for method chaining /// /// Use this overload when fluently chaining after Register(). /// - IPipelineRegistrar WithDescription(string description); + IFlowRegistrar WithDescription(string description); /// - /// Configures validation options for the most recently registered pipeline. + /// Configures validation options for the most recently registered flow. /// /// Action to configure validation behavior /// This registrar for method chaining @@ -91,14 +91,17 @@ TParams parameters /// Example: /// /// - /// registrar.Register("data_processing", ProcessingPipeline.Create) + /// registrar.Register("data_processing", ProcessingFlow.Create) + /// .WithValidation(validation => { + /// validation.Inspect(catalog.Companies, InspectionLevel.Deep); + /// + /// + /// registrar.Register("data_processing", ProcessingFlow.Create) /// .WithValidation(validation => { /// validation.Inspect(catalog.Companies, InspectionLevel.Deep); /// validation.Inspect(catalog.Shuttles, InspectionLevel.Deep); /// }); /// /// - IPipelineRegistrar WithValidation( - Action configure - ); + IFlowRegistrar WithValidation(Action configure); } diff --git a/src/core/Flowthru.Core/Registry/PipelineMetadata.cs b/src/core/Flowthru.Core/Registry/PipelineMetadata.cs index 84513b30..74a6da78 100644 --- a/src/core/Flowthru.Core/Registry/PipelineMetadata.cs +++ b/src/core/Flowthru.Core/Registry/PipelineMetadata.cs @@ -1,28 +1,28 @@ -using Flowthru.Pipelines.Validation; +using Flowthru.Flows.Validation; namespace Flowthru.Registry; /// -/// Metadata describing a registered pipeline. +/// Metadata describing a registered flow. /// /// -/// Used internally by the pipeline registry to store pipeline information -/// beyond just the pipeline instance itself. +/// Used internally by the flow registry to store Flow information +/// beyond just the Flow instance itself. /// -internal class PipelineMetadata +internal class FlowRegistration { /// - /// Pipeline name (unique identifier). + /// Flow name (unique identifier). /// public required string Name { get; init; } /// - /// Optional human-readable description of what the pipeline does. + /// Optional human-readable description of what the Flow does. /// public string? Description { get; set; } /// - /// Validation configuration for this pipeline. + /// Validation configuration for this flow. /// public ValidationOptions ValidationOptions { get; set; } = ValidationOptions.Default(); } diff --git a/src/core/Flowthru.Core/Registry/PipelineRegistrar.cs b/src/core/Flowthru.Core/Registry/PipelineRegistrar.cs index bc201a29..52a79af3 100644 --- a/src/core/Flowthru.Core/Registry/PipelineRegistrar.cs +++ b/src/core/Flowthru.Core/Registry/PipelineRegistrar.cs @@ -1,151 +1,143 @@ using Flowthru.Data; -using Flowthru.Pipelines; +using Flowthru.Flows; +using Flowthru.Flows.Validation; namespace Flowthru.Registry; /// -/// Implementation of IPipelineRegistrar that builds a dictionary of pipelines. +/// Implementation of IFlowRegistrar that builds a dictionary of flows. /// -/// The catalog type that pipelines will use -internal class PipelineRegistrar : IPipelineRegistrar - where TCatalog : DataCatalogBase +/// The catalog type that flows will use +internal class FlowRegistrar : IFlowRegistrar + where TCatalog : CatalogAbstract { private readonly TCatalog _catalog; - private readonly Dictionary> _factories = new(); - private readonly Dictionary _metadata = new(); - private string? _lastRegisteredPipeline; + private readonly Dictionary> _factories = new(); + private readonly Dictionary _metadata = new(); + private string? _lastRegisteredFlow; /// - /// Initializes a new instance of PipelineRegistrar. + /// Initializes a new instance of FlowRegistrar. /// - /// The catalog instance to pass to pipeline factories - internal PipelineRegistrar(TCatalog catalog) + /// The catalog instance to pass to Flow factories + internal FlowRegistrar(TCatalog catalog) { _catalog = catalog; } /// - public IPipelineRegistrar Register( - string name, - Func pipelineFactory - ) + public IFlowRegistrar Register(string name, Func flowFactory) { if (string.IsNullOrWhiteSpace(name)) { - throw new ArgumentException("Pipeline name cannot be null or empty", nameof(name)); + throw new ArgumentException("Flow name cannot be null or empty", nameof(name)); } if (_factories.ContainsKey(name)) { - throw new InvalidOperationException($"Pipeline '{name}' is already registered"); + throw new InvalidOperationException($"Flow '{name}' is already registered"); } // Wrap factory to capture catalog - _factories[name] = () => pipelineFactory(_catalog); + _factories[name] = () => flowFactory(_catalog); // Initialize metadata - _metadata[name] = new PipelineMetadata { Name = name }; - _lastRegisteredPipeline = name; + _metadata[name] = new FlowRegistration { Name = name }; + _lastRegisteredFlow = name; return this; } /// - public IPipelineRegistrar Register( + public IFlowRegistrar Register( string name, - Func pipelineFactory, + Func flowFactory, TParams parameters ) { if (string.IsNullOrWhiteSpace(name)) { - throw new ArgumentException("Pipeline name cannot be null or empty", nameof(name)); + throw new ArgumentException("Flow name cannot be null or empty", nameof(name)); } if (_factories.ContainsKey(name)) { - throw new InvalidOperationException($"Pipeline '{name}' is already registered"); + throw new InvalidOperationException($"Flow '{name}' is already registered"); } // Wrap factory to capture catalog and parameters - _factories[name] = () => pipelineFactory(_catalog, parameters); + _factories[name] = () => flowFactory(_catalog, parameters); // Initialize metadata - _metadata[name] = new PipelineMetadata { Name = name }; - _lastRegisteredPipeline = name; + _metadata[name] = new FlowRegistration { Name = name }; + _lastRegisteredFlow = name; return this; } /// - public IPipelineRegistrar WithDescription(string description) + public IFlowRegistrar WithDescription(string description) { - if (_lastRegisteredPipeline == null) + if (_lastRegisteredFlow == null) { throw new InvalidOperationException( - "No pipeline has been registered yet. Call Register() first." + "No flow has been registered yet. Call Register() first." ); } - if (!_metadata.ContainsKey(_lastRegisteredPipeline)) + if (!_metadata.ContainsKey(_lastRegisteredFlow)) { - throw new InvalidOperationException( - $"Pipeline '{_lastRegisteredPipeline}' has not been registered" - ); + throw new InvalidOperationException($"Flow '{_lastRegisteredFlow}' has not been registered"); } - _metadata[_lastRegisteredPipeline].Description = description; + _metadata[_lastRegisteredFlow].Description = description; return this; } /// - public IPipelineRegistrar WithValidation( - Action configure - ) + public IFlowRegistrar WithValidation(Action configure) { - if (_lastRegisteredPipeline == null) + if (_lastRegisteredFlow == null) { throw new InvalidOperationException( - "No pipeline has been registered yet. Call Register() first." + "No Flow has been registered yet. Call Register() first." ); } - if (!_metadata.ContainsKey(_lastRegisteredPipeline)) + if (!_metadata.ContainsKey(_lastRegisteredFlow)) { - throw new InvalidOperationException( - $"Pipeline '{_lastRegisteredPipeline}' has not been registered" - ); + throw new InvalidOperationException($"Flow '{_lastRegisteredFlow}' has not been registered"); } - configure(_metadata[_lastRegisteredPipeline].ValidationOptions); + configure(_metadata[_lastRegisteredFlow].ValidationOptions); return this; } /// - /// Builds and returns all registered pipelines with their metadata applied. + /// Builds and returns all registered flows with their metadata applied. /// - /// Dictionary of pipeline names to pipeline instances - internal Dictionary Build() + /// Dictionary of flow names to Flow instances + internal Dictionary Build() { - var pipelines = new Dictionary(); + var flows = new Dictionary(); foreach (var (name, factory) in _factories) { - // Invoke factory to create pipeline - var pipeline = factory(); + // Invoke factory to create flow + var flow = factory(); // Apply metadata - pipeline.Name = name; + flow.Name = name; if (_metadata.TryGetValue(name, out var metadata)) { - pipeline.Description = metadata.Description; - pipeline.ValidationOptions = metadata.ValidationOptions; + flow.Description = metadata.Description; + flow.ValidationOptions = metadata.ValidationOptions; } - pipelines[name] = pipeline; + flows[name] = flow; } - return pipelines; + return flows; } } diff --git a/src/core/Flowthru.Core/Results/ConsoleResultFormatter.cs b/src/core/Flowthru.Core/Results/ConsoleResultFormatter.cs index 9aa8fc96..f5a7059e 100644 --- a/src/core/Flowthru.Core/Results/ConsoleResultFormatter.cs +++ b/src/core/Flowthru.Core/Results/ConsoleResultFormatter.cs @@ -1,4 +1,4 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using Microsoft.Extensions.Logging; namespace Flowthru.Results; @@ -10,10 +10,10 @@ namespace Flowthru.Results; /// This is the default formatter used by the CLI. /// Produces colorful, detailed output suitable for interactive terminal sessions. /// -public class ConsoleResultFormatter : IPipelineResultFormatter +public class ConsoleResultFormatter : IFlowResultFormatter { /// - public void Format(PipelineResult result, ILogger logger) + public void Format(FlowResult result, ILogger logger) { if (result.Success) { @@ -25,26 +25,26 @@ public void Format(PipelineResult result, ILogger logger) } } - private void FormatSuccess(PipelineResult result, ILogger logger) + private void FormatSuccess(FlowResult result, ILogger logger) { logger.LogInformation("================================================================"); - logger.LogInformation("Pipeline: {PipelineName}", result.PipelineName ?? "Unknown"); + logger.LogInformation("Pipeline: {FlowName}", result.FlowName ?? "Unknown"); logger.LogInformation("Status: ✓ SUCCESS"); logger.LogInformation("Duration: {Duration:F2}s", result.ExecutionTime.TotalSeconds); logger.LogInformation("================================================================"); logger.LogInformation(""); - if (result.NodeResults.Count > 0) + if (result.StepResults.Count > 0) { - logger.LogInformation("Nodes Executed ({Count}):", result.NodeResults.Count); + logger.LogInformation("Steps Executed ({Count}):", result.StepResults.Count); - foreach (var (nodeName, nodeResult) in result.NodeResults) + foreach (var (nodeName, nodeResult) in result.StepResults) { if (nodeResult.Success) { logger.LogInformation( - " ✓ {NodeName,-40} {Duration,6:F2}s ({InputCount,6} → {OutputCount,6} records)", - nodeResult.NodeName, + " ✓ {StepName,-40} {Duration,6:F2}s ({InputCount,6} → {OutputCount,6} records)", + nodeResult.StepName, nodeResult.ExecutionTime.TotalSeconds, nodeResult.InputCount, nodeResult.OutputCount @@ -54,8 +54,8 @@ private void FormatSuccess(PipelineResult result, ILogger logger) { // This shouldn't happen in a successful pipeline, but handle it anyway logger.LogWarning( - " ✗ {NodeName,-40} {Duration,6:F2}s FAILED", - nodeResult.NodeName, + " ✗ {StepName,-40} {Duration,6:F2}s FAILED", + nodeResult.StepName, nodeResult.ExecutionTime.TotalSeconds ); } @@ -67,27 +67,27 @@ private void FormatSuccess(PipelineResult result, ILogger logger) logger.LogInformation("================================================================"); } - private void FormatFailure(PipelineResult result, ILogger logger) + private void FormatFailure(FlowResult result, ILogger logger) { logger.LogError("================================================================"); - logger.LogError("Pipeline: {PipelineName}", result.PipelineName ?? "Unknown"); + logger.LogError("Pipeline: {FlowName}", result.FlowName ?? "Unknown"); logger.LogError("Status: ✗ FAILED"); logger.LogError("Duration: {Duration:F2}s", result.ExecutionTime.TotalSeconds); logger.LogError("================================================================"); logger.LogError(""); // Show which nodes succeeded before failure - var succeededNodes = result.NodeResults.Values.Where(n => n.Success).ToList(); - var failedNode = result.NodeResults.Values.FirstOrDefault(n => !n.Success); + var succeededSteps = result.StepResults.Values.Where(n => n.Success).ToList(); + var failedStep = result.StepResults.Values.FirstOrDefault(n => !n.Success); - if (succeededNodes.Any()) + if (succeededSteps.Any()) { - logger.LogInformation("Nodes Completed Before Failure ({Count}):", succeededNodes.Count); - foreach (var nodeResult in succeededNodes) + logger.LogInformation("Steps Completed Before Failure ({Count}):", succeededSteps.Count); + foreach (var nodeResult in succeededSteps) { logger.LogInformation( - " ✓ {NodeName,-40} {Duration,6:F2}s", - nodeResult.NodeName, + " ✓ {StepName,-40} {Duration,6:F2}s", + nodeResult.StepName, nodeResult.ExecutionTime.TotalSeconds ); } @@ -95,19 +95,19 @@ private void FormatFailure(PipelineResult result, ILogger logger) } // Show failed node - if (failedNode != null) + if (failedStep != null) { - logger.LogError("Failed Node:"); - logger.LogError(" ✗ {NodeName}", failedNode.NodeName); - logger.LogError(" Duration: {Duration:F2}s", failedNode.ExecutionTime.TotalSeconds); + logger.LogError("Failed Step:"); + logger.LogError(" ✗ {StepName}", failedStep.StepName); + logger.LogError(" Duration: {Duration:F2}s", failedStep.ExecutionTime.TotalSeconds); - if (failedNode.Exception != null) + if (failedStep.Exception != null) { - logger.LogError(" Error: {ErrorMessage}", failedNode.Exception.Message); + logger.LogError(" Error: {ErrorMessage}", failedStep.Exception.Message); logger.LogError(" Stack Trace:"); // Format stack trace with indentation - var stackLines = failedNode.Exception.StackTrace?.Split('\n') ?? Array.Empty(); + var stackLines = failedStep.Exception.StackTrace?.Split('\n') ?? Array.Empty(); foreach (var line in stackLines.Take(10)) // Limit to first 10 lines { logger.LogError(" {StackLine}", line.TrimEnd()); diff --git a/src/core/Flowthru.Core/Results/IPipelineResultFormatter.cs b/src/core/Flowthru.Core/Results/IPipelineResultFormatter.cs index 387273a5..f829eba4 100644 --- a/src/core/Flowthru.Core/Results/IPipelineResultFormatter.cs +++ b/src/core/Flowthru.Core/Results/IPipelineResultFormatter.cs @@ -1,14 +1,14 @@ -using Flowthru.Pipelines; +using Flowthru.Flows; using Microsoft.Extensions.Logging; namespace Flowthru.Results; /// -/// Interface for formatting pipeline execution results. +/// Interface for formatting Flow execution results. /// /// /// -/// Result formatters transform a PipelineResult into human-readable or +/// Result formatters transform a FlowResult into human-readable or /// machine-readable output via logging. /// /// @@ -19,12 +19,12 @@ namespace Flowthru.Results; /// Future formatters: JSON, Markdown, compact CI/CD format. /// /// -public interface IPipelineResultFormatter +public interface IFlowResultFormatter { /// - /// Formats and outputs the pipeline result. + /// Formats and outputs the Flow result. /// - /// The pipeline execution result + /// The Flow execution result /// The logger to write output to - void Format(PipelineResult result, ILogger logger); + void Format(FlowResult result, ILogger logger); } diff --git a/src/core/Flowthru.Core/Serialization/EnumMetadataRegistry.cs b/src/core/Flowthru.Core/Serialization/EnumMetadataRegistry.cs index 44395fb5..8f6395a9 100644 --- a/src/core/Flowthru.Core/Serialization/EnumMetadataRegistry.cs +++ b/src/core/Flowthru.Core/Serialization/EnumMetadataRegistry.cs @@ -37,7 +37,7 @@ internal static class EnumMetadataRegistry /// This method is thread-safe. If multiple threads request metadata for the same enum type /// simultaneously, the metadata will be computed only once. /// - public static EnumMetadataCache GetOrCreate() + public static EnumMetadataCache Create() where TEnum : struct, Enum { Type enumType = typeof(TEnum); diff --git a/src/core/Flowthru.Core/Serialization/EnumSerializationHelper.cs b/src/core/Flowthru.Core/Serialization/EnumSerializationHelper.cs index ecad64d8..f1505d91 100644 --- a/src/core/Flowthru.Core/Serialization/EnumSerializationHelper.cs +++ b/src/core/Flowthru.Core/Serialization/EnumSerializationHelper.cs @@ -30,9 +30,9 @@ public static object ParseEnumFromString(Type enumType, string value) throw new ArgumentException($"Type '{enumType.Name}' is not an enum type.", nameof(enumType)); } - // Use reflection to call EnumMetadataRegistry.GetOrCreate() + // Use reflection to call EnumMetadataRegistry.Create() var getOrCreateMethod = typeof(EnumMetadataRegistry) - .GetMethod(nameof(EnumMetadataRegistry.GetOrCreate))! + .GetMethod(nameof(EnumMetadataRegistry.Create))! .MakeGenericMethod(enumType); var metadata = getOrCreateMethod.Invoke(null, null)!; @@ -68,9 +68,9 @@ public static string FormatEnumToString(Type enumType, object value) throw new ArgumentException($"Type '{enumType.Name}' is not an enum type.", nameof(enumType)); } - // Use reflection to call EnumMetadataRegistry.GetOrCreate() + // Use reflection to call EnumMetadataRegistry.Create() var getOrCreateMethod = typeof(EnumMetadataRegistry) - .GetMethod(nameof(EnumMetadataRegistry.GetOrCreate))! + .GetMethod(nameof(EnumMetadataRegistry.Create))! .MakeGenericMethod(enumType); var metadata = getOrCreateMethod.Invoke(null, null)!; diff --git a/src/core/Flowthru.Core/Services/FlowthruService.cs b/src/core/Flowthru.Core/Services/FlowthruService.cs index 0f9c3b77..a6e4fe88 100644 --- a/src/core/Flowthru.Core/Services/FlowthruService.cs +++ b/src/core/Flowthru.Core/Services/FlowthruService.cs @@ -1,9 +1,10 @@ using System.Diagnostics; using Flowthru.Data; using Flowthru.Data.Validation; +using Flowthru.Flows; +using Flowthru.Flows.Validation; using Flowthru.Meta; using Flowthru.Meta.Models; -using Flowthru.Pipelines; using Flowthru.Services.Models; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -15,13 +16,13 @@ namespace Flowthru.Services; /// /// /// This service wraps pipeline execution logic in a CLI-agnostic interface. -/// It delegates to existing execution methods while +/// It delegates to existing execution methods while /// providing a cleaner API for programmatic use. /// internal sealed class FlowthruService : IFlowthruService { - private readonly IReadOnlyList _catalogs; - private readonly Dictionary _pipelines; + private readonly IReadOnlyList _catalogs; + private readonly Dictionary _pipelines; private readonly IServiceProvider _services; private readonly ILogger _logger; private readonly FlowthruMetadataBuilder? _metadataBuilder; @@ -30,8 +31,8 @@ internal sealed class FlowthruService : IFlowthruService /// Initializes a new instance of FlowthruService. /// public FlowthruService( - IReadOnlyList catalogs, - Dictionary pipelines, + IReadOnlyList catalogs, + Dictionary pipelines, IServiceProvider services, ILogger logger, FlowthruMetadataBuilder? metadataBuilder = null @@ -50,9 +51,7 @@ public FlowthruService( } // Resolve validation hooks from DI (Phase 4: extensions can register hooks) - var validationHooks = _services - .GetServices() - .ToList(); + var validationHooks = _services.GetServices().ToList(); // Inject services into each pipeline and build foreach (var pipeline in _pipelines.Values) @@ -60,7 +59,7 @@ public FlowthruService( pipeline.Logger = _logger; pipeline.ServiceProvider = _services; - // Register validation hooks (e.g., PythonNodeValidator from Python extension) + // Register validation hooks (e.g., PythonStepValidator from Python extension) foreach (var hook in validationHooks) { pipeline.ValidationHooks.Add(hook); @@ -71,13 +70,13 @@ public FlowthruService( } /// - public IReadOnlyCollection PipelineNames => _pipelines.Keys; + public IReadOnlyCollection FlowNames => _pipelines.Keys; /// - public IReadOnlyList Catalogs => _catalogs; + public IReadOnlyList Catalogs => _catalogs; /// - public async Task ExecutePipelineAsync( + public async Task ExecuteFlowAsync( ExecutionOptions? options = null, bool exportMetadata = true, string? metadataOutputDirectory = null, @@ -87,10 +86,10 @@ public async Task ExecutePipelineAsync( var totalStopwatch = Stopwatch.StartNew(); _logger.LogInformation("Merging all pipelines into unified DAG."); - _logger.LogInformation("Available pipelines: {Pipelines}", string.Join(", ", PipelineNames)); + _logger.LogInformation("Available pipelines: {Pipelines}", string.Join(", ", FlowNames)); // Merge all pipelines into a single DAG - var mergedPipeline = Pipeline.Merge(_pipelines); + var mergedPipeline = Flow.Merge(_pipelines); // Inject services and logger mergedPipeline.Logger = _logger; @@ -112,8 +111,8 @@ public async Task ExecutePipelineAsync( _logger.LogInformation("→ Building pipeline and analyzing dependencies..."); mergedPipeline.Build(options.SliceStrategy); _logger.LogInformation( - " ✓ {NodeCount} nodes organized into {LayerCount} execution layers", - mergedPipeline.Nodes.Count, + " ✓ {StepCount} nodes organized into {LayerCount} execution layers", + mergedPipeline.Steps.Count, mergedPipeline.ExecutionLayers!.Count ); @@ -136,8 +135,8 @@ public async Task ExecutePipelineAsync( validationResult.ThrowIfInvalid(); } - var layer0Nodes = mergedPipeline.ExecutionLayers![0]; - validatedInputCount = layer0Nodes.SelectMany(node => node.Inputs).Distinct().Count(); + var layer0Steps = mergedPipeline.ExecutionLayers![0]; + validatedInputCount = layer0Steps.SelectMany(node => node.Inputs).Distinct().Count(); _logger.LogInformation(" ✓ {Count} external data sources validated", validatedInputCount); } @@ -173,8 +172,8 @@ public async Task ExecutePipelineAsync( _logger.LogInformation("════════════════════════════════════════"); _logger.LogInformation(""); _logger.LogInformation( - "Nodes: {Count} nodes across {Layers} layers", - mergedPipeline.Nodes.Count, + "Steps: {Count} nodes across {Layers} layers", + mergedPipeline.Steps.Count, mergedPipeline.ExecutionLayers!.Count ); _logger.LogInformation("External Inputs: {Count} validated", validatedInputCount); @@ -184,9 +183,9 @@ public async Task ExecutePipelineAsync( _logger.LogInformation(""); totalStopwatch.Stop(); - return PipelineResult.CreateDryRunSuccess( + return FlowResult.CreateDryRunSuccess( totalStopwatch.Elapsed, - mergedPipeline.Nodes.Count, + mergedPipeline.Steps.Count, mergedPipeline.ExecutionLayers!.Count, validatedInputCount, "Pipeline" @@ -213,12 +212,12 @@ public async Task ExecutePipelineAsync( } /// - public PipelineMetadata GetPipelineMetadata(string pipelineName) + public FlowMetadata GetFlowMetadata(string pipelineName) { if (!_pipelines.TryGetValue(pipelineName, out var pipeline)) { throw new KeyNotFoundException( - $"Pipeline '{pipelineName}' not found. " + $"Available: {string.Join(", ", PipelineNames)}" + $"Pipeline '{pipelineName}' not found. " + $"Available: {string.Join(", ", FlowNames)}" ); } @@ -229,11 +228,11 @@ public PipelineMetadata GetPipelineMetadata(string pipelineName) .Distinct() .ToList() ?? new List(); - return new PipelineMetadata + return new FlowMetadata { Name = pipeline.Name ?? pipelineName, Description = pipeline.Description, - NodeCount = pipeline.Nodes.Count, + StepCount = pipeline.Steps.Count, LayerCount = pipeline.ExecutionLayers?.Count ?? 0, ExternalInputs = externalInputs, IsBuilt = pipeline.IsBuilt, @@ -243,22 +242,21 @@ public PipelineMetadata GetPipelineMetadata(string pipelineName) /// public DagMetadata GetDagMetadata( string? pipelineName = null, - PipelineSliceStrategy? sliceStrategy = null + FlowSliceStrategy? sliceStrategy = null ) { - Dictionary toMerge; + Dictionary toMerge; if (pipelineName is not null) { if (!_pipelines.TryGetValue(pipelineName, out var namedPipeline)) { throw new KeyNotFoundException( - $"Pipeline '{pipelineName}' not found. " - + $"Available: {string.Join(", ", PipelineNames)}" + $"Pipeline '{pipelineName}' not found. " + $"Available: {string.Join(", ", FlowNames)}" ); } - toMerge = new Dictionary { [pipelineName] = namedPipeline }; + toMerge = new Dictionary { [pipelineName] = namedPipeline }; } else { @@ -267,7 +265,7 @@ public DagMetadata GetDagMetadata( // Always merge to produce a fresh pipeline instance — avoids mutating // the registered pipelines' Build/Slice state as a side effect. - var pipeline = Pipeline.Merge(toMerge); + var pipeline = Flow.Merge(toMerge); pipeline.Logger = _logger; pipeline.ServiceProvider = _services; pipeline.Build(sliceStrategy); @@ -276,7 +274,7 @@ public DagMetadata GetDagMetadata( } /// - public async Task ValidatePipelineAsync( + public async Task ValidateFlowAsync( string pipelineName, CancellationToken cancellationToken = default ) @@ -284,7 +282,7 @@ public async Task ValidatePipelineAsync( if (!_pipelines.TryGetValue(pipelineName, out var pipeline)) { throw new KeyNotFoundException( - $"Pipeline '{pipelineName}' not found. " + $"Available: {string.Join(", ", PipelineNames)}" + $"Pipeline '{pipelineName}' not found. " + $"Available: {string.Join(", ", FlowNames)}" ); } @@ -292,7 +290,7 @@ public async Task ValidatePipelineAsync( } private async Task ExportPipelineMetadataAsync( - Pipeline pipeline, + Flow pipeline, string pipelineName, string? outputDirectory ) diff --git a/src/core/Flowthru.Core/Services/FlowthruServiceBuilder.cs b/src/core/Flowthru.Core/Services/FlowthruServiceBuilder.cs index f2beada8..dba212d0 100644 --- a/src/core/Flowthru.Core/Services/FlowthruServiceBuilder.cs +++ b/src/core/Flowthru.Core/Services/FlowthruServiceBuilder.cs @@ -1,6 +1,7 @@ using Flowthru.Configuration; using Flowthru.Data; using Flowthru.Data.Storage.Strategies; +using Flowthru.Flows; using Flowthru.Meta; using Flowthru.Meta.Providers; using Microsoft.Extensions.Configuration; @@ -15,15 +16,15 @@ namespace Flowthru.Services; /// /// /// This builder configures the service layer without CLI coupling. -/// Use it to register catalogs, pipelines, and optional features. +/// Use it to register catalogs, flows, and optional features. /// /// /// Basic Usage: /// /// services.AddFlowthru(flowthru => /// { -/// flowthru.UseCatalog(_ => new MyCatalog(dataPath)); -/// flowthru.RegisterPipeline("my_pipeline", MyPipeline.Create); +/// flowthru.RegisterCatalog(_ => new MyCatalog(dataPath)); +/// flowthru.RegisterFlow("my_flow", MyFlow.Create); /// }); /// /// @@ -33,11 +34,11 @@ public sealed class FlowthruServiceBuilder private readonly IServiceCollection _services; private readonly List _registeredCatalogTypes = new(); private readonly List< - Func> + Func> > _dynamicCatalogFactories = new(); - private readonly List _registrations = new(); - private Func>? _pipelineFactory; - private PipelineRegistrationEntry? _lastRegistration; + private readonly List _registrations = new(); + private Func>? _flowFactory; + private FlowRegistrationEntry? _lastRegistration; private IConfiguration? _configuration; internal FlowthruServiceBuilder(IServiceCollection services) @@ -46,19 +47,16 @@ internal FlowthruServiceBuilder(IServiceCollection services) } /// - /// Internal entry type that carries a pipeline factory and its associated metadata. - /// Replaces the PipelineRegistrar indirection for cleaner multi-catalog support. + /// Internal entry type that carries a Flow factory and its associated metadata. + /// Replaces the FlowRegistrar indirection for cleaner multi-catalog support. /// - internal sealed class PipelineRegistrationEntry + internal sealed class FlowRegistrationEntry { public string Label { get; } - public Func Factory { get; } + public Func Factory { get; } public string Description { get; set; } = ""; - internal PipelineRegistrationEntry( - string label, - Func factory - ) + internal FlowRegistrationEntry(string label, Func factory) { Label = label; Factory = factory; @@ -74,8 +72,8 @@ internal PipelineRegistrationEntry( /// The catalog will be resolved from the DI container, allowing constructor /// parameter injection (e.g., IConfiguration, IOptions). /// - public FlowthruServiceBuilder UseCatalog() - where TCatalog : DataCatalogBase + public FlowthruServiceBuilder RegisterCatalog() + where TCatalog : CatalogAbstract { _services.AddSingleton(); _registeredCatalogTypes.Add(typeof(TCatalog)); @@ -90,7 +88,7 @@ public FlowthruServiceBuilder UseCatalog() /// /// Use this when the catalog doesn't require dependency injection. /// - public FlowthruServiceBuilder UseCatalog(DataCatalogBase catalog) + public FlowthruServiceBuilder RegisterCatalog(CatalogAbstract catalog) { if (catalog == null) { @@ -112,10 +110,10 @@ public FlowthruServiceBuilder UseCatalog(DataCatalogBase catalog) /// Use this when the catalog needs to resolve services during construction, /// or when construction requires parameters unavailable at the call site. /// - public FlowthruServiceBuilder UseCatalog( + public FlowthruServiceBuilder RegisterCatalog( Func catalogFactory ) - where TCatalog : DataCatalogBase + where TCatalog : CatalogAbstract { if (catalogFactory == null) { @@ -136,10 +134,10 @@ Func catalogFactory /// This builder for method chaining /// /// All registered catalogs will receive DI service injection and appear in - /// . Use with to - /// wire per-catalog pipelines in a loop. + /// . Use with to + /// wire per-catalog flows in a loop. /// - public FlowthruServiceBuilder UseCatalogs(IEnumerable catalogs) + public FlowthruServiceBuilder RegisterCatalogs(IEnumerable catalogs) { if (catalogs == null) throw new ArgumentNullException(nameof(catalogs)); @@ -155,8 +153,8 @@ public FlowthruServiceBuilder UseCatalogs(IEnumerable catalogs) /// /// Factory that returns the catalog collection /// This builder for method chaining - public FlowthruServiceBuilder UseCatalogs( - Func> catalogsFactory + public FlowthruServiceBuilder RegisterCatalogs( + Func> catalogsFactory ) { if (catalogsFactory == null) @@ -167,64 +165,64 @@ Func> catalogsFactory } /// - /// Escape-hatch for registering pipelines via a full-access service provider factory. + /// Escape-hatch for registering flows via a full-access service provider factory. /// - /// Factory function that receives the service provider and returns the pipeline dictionary + /// Factory function that receives the service provider and returns the Flow dictionary /// This builder for method chaining /// - /// Prefer for standard pipeline registration. - /// Use this only when you need full service provider access during pipeline construction. + /// Prefer for standard Flow registration. + /// Use this only when you need full service provider access during Flow construction. /// - public FlowthruServiceBuilder UsePipelines( - Func> pipelineFactory + public FlowthruServiceBuilder RegisterFlows( + Func> flowFactory ) { - if (pipelineFactory == null) + if (flowFactory == null) { - throw new ArgumentNullException(nameof(pipelineFactory)); + throw new ArgumentNullException(nameof(flowFactory)); } - _pipelineFactory = pipelineFactory; + _flowFactory = flowFactory; return this; } /// - /// Registers a pipeline by inspecting the delegate's parameter types at runtime. - /// Parameters that extend are resolved from DI as catalogs. + /// Registers a Flow by inspecting the delegate's parameter types at runtime. + /// Parameters that extend are resolved from DI as catalogs. /// All other parameters are resolved from DI as services. /// - /// Unique pipeline name - /// A method group or delegate whose parameters are catalogs, services, or config objects + /// Unique Flow name + /// A method group or delegate whose parameters are catalogs, services, or config objects /// /// Optional configuration section path. When provided, the last non-catalog, non-service parameter /// is bound from configuration instead of DI. /// /// This builder for method chaining - public FlowthruServiceBuilder RegisterPipeline( + public FlowthruServiceBuilder RegisterFlow( string label, - Delegate pipeline, + Delegate flow, string? configurationSection = null ) { if (string.IsNullOrWhiteSpace(label)) - throw new ArgumentException("Pipeline label cannot be null or empty.", nameof(label)); - if (pipeline == null) - throw new ArgumentNullException(nameof(pipeline)); + throw new ArgumentException("Flow label cannot be null or empty.", nameof(label)); + if (flow == null) + throw new ArgumentNullException(nameof(flow)); - var method = pipeline.Method; + var method = flow.Method; var parameters = method.GetParameters(); // Validate return type - if (method.ReturnType != typeof(Pipelines.Pipeline)) + if (method.ReturnType != typeof(Flow)) { throw new ArgumentException( - $"Pipeline delegate must return Pipeline, but '{method.Name}' returns {method.ReturnType.Name}.", - nameof(pipeline) + $"Flow delegate must return Flow, but '{method.Name}' returns {method.ReturnType.Name}.", + nameof(flow) ); } - // Build the resolver: for each parameter, determine how to resolve it at pipeline-build time + // Build the resolver: for each parameter, determine how to resolve it at flow-build time var resolvers = new Func[parameters.Length]; for (int i = 0; i < parameters.Length; i++) { @@ -232,7 +230,7 @@ public FlowthruServiceBuilder RegisterPipeline( if ( configurationSection != null - && !typeof(DataCatalogBase).IsAssignableFrom(paramType) + && !typeof(CatalogAbstract).IsAssignableFrom(paramType) && !paramType.IsInterface ) { @@ -240,7 +238,7 @@ public FlowthruServiceBuilder RegisterPipeline( if (_configuration == null) { throw new InvalidOperationException( - $"Pipeline '{label}' specifies a configuration section but UseConfiguration() has not been called." + $"Flow '{label}' specifies a configuration section but UseConfiguration() has not been called." ); } @@ -257,15 +255,15 @@ public FlowthruServiceBuilder RegisterPipeline( } } - var capturedDelegate = pipeline; - var entry = new PipelineRegistrationEntry( + var capturedDelegate = flow; + var entry = new FlowRegistrationEntry( label, sp => { var args = new object?[resolvers.Length]; for (int i = 0; i < resolvers.Length; i++) args[i] = resolvers[i](sp); - return (Pipelines.Pipeline)capturedDelegate.DynamicInvoke(args)!; + return (Flow)capturedDelegate.DynamicInvoke(args)!; } ); _registrations.Add(entry); @@ -274,16 +272,16 @@ public FlowthruServiceBuilder RegisterPipeline( } /// - /// Adds a description to the most recently registered pipeline. + /// Adds a description to the most recently registered flow. /// - /// Human-readable description of what the pipeline does + /// Human-readable description of what the Flow does /// This builder for method chaining public FlowthruServiceBuilder WithDescription(string description) { if (_lastRegistration == null) { throw new InvalidOperationException( - "WithDescription() can only be used after RegisterPipeline()." + "WithDescription() can only be used after RegisterFlow()." ); } @@ -448,7 +446,7 @@ Func strategyFactory /// This builder for method chaining /// /// - /// Metadata export is optional. If not configured, pipelines will execute + /// Metadata export is optional. If not configured, flows will execute /// without generating DAG diagrams or metadata files. /// /// @@ -556,10 +554,10 @@ private static void ApplyMetadataOptions(FlowthruMetadataBuilder builder, Metada }; mermaid.WithDirection(direction); - // Apply color configuration for active nodes and data - if (!string.IsNullOrEmpty(options.Mermaid.ActiveNodeColor)) + // Apply color configuration for active steps and data + if (!string.IsNullOrEmpty(options.Mermaid.ActiveStepColor)) { - mermaid.WithActiveNodeColor(options.Mermaid.ActiveNodeColor); + mermaid.WithActiveStepColor(options.Mermaid.ActiveStepColor); } if (!string.IsNullOrEmpty(options.Mermaid.ActiveDataColor)) { @@ -604,53 +602,53 @@ private static void ApplyFileConfiguration(TBuilder builder, MetadataO /// /// Internal method called by AddFlowthru to register the catalog collection and - /// pipeline dictionary into the DI container. Must be called after all UseCatalog - /// and RegisterPipeline calls have been made. + /// Flow dictionary into the DI container. Must be called after all RegisterCatalog + /// and RegisterFlow calls have been made. /// - internal void RegisterPipelineDictionary() + internal void RegisterFlowDictionary() { // Always register the catalog collection so FlowthruService can inject all catalogs. - // Merges both type-registered catalogs (UseCatalog) and dynamically constructed - // catalog collections (UseCatalogs). + // Merges both type-registered catalogs (RegisterCatalog) and dynamically constructed + // catalog collections (RegisterCatalogs). var catalogTypes = _registeredCatalogTypes.ToList(); var dynamicFactories = _dynamicCatalogFactories.ToList(); - _services.AddSingleton>(sp => + _services.AddSingleton>(sp => { - var typedCatalogs = catalogTypes.Select(t => (DataCatalogBase)sp.GetRequiredService(t)); + var typedCatalogs = catalogTypes.Select(t => (CatalogAbstract)sp.GetRequiredService(t)); var dynamicCatalogs = dynamicFactories.SelectMany(f => f(sp)); return typedCatalogs.Concat(dynamicCatalogs).ToList().AsReadOnly(); }); var snapshot = _registrations.ToList(); - var factory = _pipelineFactory; + var factory = _flowFactory; if (snapshot.Count == 0 && factory == null) { throw new InvalidOperationException( - "No pipelines were registered. Call RegisterPipeline() or UsePipelines() before building the service." + "No flows were registered. Call RegisterFlow() or RegisterFlows() before building the service." ); } - _services.AddSingleton>(sp => + _services.AddSingleton>(sp => { - var result = new Dictionary(); + var result = new Dictionary(); - // Source 1: UsePipelines factory (registered first so inline registrations can override) + // Source 1: RegisterFlows factory (registered first so inline registrations can override) if (factory != null) { - foreach (var (key, pipeline) in factory(sp)) + foreach (var (key, flow) in factory(sp)) { - result[key] = pipeline; + result[key] = flow; } } - // Source 2: RegisterPipeline calls (wins on key collision) + // Source 2: RegisterFlow calls (wins on key collision) foreach (var reg in snapshot) { - var pipeline = reg.Factory(sp); - pipeline.Name = reg.Label; - pipeline.Description = reg.Description; - result[reg.Label] = pipeline; + var flow = reg.Factory(sp); + flow.Name = reg.Label; + flow.Description = reg.Description; + result[reg.Label] = flow; } return result; diff --git a/src/core/Flowthru.Core/Services/FlowthruServiceCollectionExtensions.cs b/src/core/Flowthru.Core/Services/FlowthruServiceCollectionExtensions.cs index adefc0d0..a048871e 100644 --- a/src/core/Flowthru.Core/Services/FlowthruServiceCollectionExtensions.cs +++ b/src/core/Flowthru.Core/Services/FlowthruServiceCollectionExtensions.cs @@ -24,8 +24,8 @@ public static class FlowthruServiceCollectionExtensions /// // In Program.cs or Startup.cs /// services.AddFlowthru(flowthru => /// { - /// flowthru.UseCatalog<MyCatalog>(); - /// flowthru.UsePipelines(catalog => new Dictionary<string, Pipeline> + /// flowthru.RegisterCatalog<MyCatalog>(); + /// flowthru.RegisterPipelines(catalog => new Dictionary<string, Pipeline> /// { /// ["my_pipeline"] = MyPipeline.Create((MyCatalog)catalog) /// }); @@ -48,7 +48,7 @@ public static class FlowthruServiceCollectionExtensions /// /// public async Task<IActionResult> RunPipeline(string name) /// { - /// var request = new PipelineExecutionRequest { PipelineName = name }; + /// var request = new PipelineExecutionRequest { FlowName = name }; /// var result = await _flowthru.ExecutePipelineAsync(request); /// return Ok(result); /// } @@ -76,7 +76,7 @@ Action configure configure(builder); // Register pipeline dictionary factory if inline registrations exist - builder.RegisterPipelineDictionary(); + builder.RegisterFlowDictionary(); // Register core service services.AddSingleton(); diff --git a/src/core/Flowthru.Core/Services/IFlowthruService.cs b/src/core/Flowthru.Core/Services/IFlowthruService.cs index ce8ee930..8cb81969 100644 --- a/src/core/Flowthru.Core/Services/IFlowthruService.cs +++ b/src/core/Flowthru.Core/Services/IFlowthruService.cs @@ -1,13 +1,13 @@ using Flowthru.Data; using Flowthru.Data.Validation; +using Flowthru.Flows; using Flowthru.Meta.Models; -using Flowthru.Pipelines; using Flowthru.Services.Models; namespace Flowthru.Services; /// -/// Core service for executing Flowthru pipelines programmatically. +/// Core service for executing Flowthru flows programmatically. /// /// /// @@ -35,17 +35,17 @@ namespace Flowthru.Services; /// var options = new ExecutionOptions /// { /// DryRun = false, -/// SliceStrategy = new PipelineSliceStrategy +/// SliceStrategy = new FlowSliceStrategy /// { -/// Pipelines = new HashSet<string> { "data_processing" } +/// Flows = new HashSet<string> { "data_processing" } /// } /// }; /// -/// var result = await _flowthru.ExecutePipelineAsync(options); +/// var result = await _flowthru.ExecuteFlowAsync(options); /// /// if (result.Success) /// { -/// Console.WriteLine($"Processed {result.NodeResults.Count} nodes"); +/// Console.WriteLine($"Processed {result.StepResults.Count} flow"); /// } /// } /// } @@ -55,39 +55,39 @@ namespace Flowthru.Services; public interface IFlowthruService { /// - /// Gets the names of all registered pipelines. + /// Gets the names of all registered flows. /// - IReadOnlyCollection PipelineNames { get; } + IReadOnlyCollection FlowNames { get; } /// /// Gets all registered catalog instances. /// - IReadOnlyList Catalogs { get; } + IReadOnlyList Catalogs { get; } /// - /// Executes all registered pipelines, optionally sliced by criteria. + /// Executes all registered flows, optionally sliced by criteria. /// /// Execution options with optional slice strategy /// Whether to export DAG metadata /// Override for metadata output directory /// Cancellation token - /// Execution result with timing, node results, and status + /// Execution result with timing, step results, and status /// - /// This method always merges all registered pipelines into a single DAG, + /// This method always merges all registered flows into a single DAG, /// then applies optional slicing criteria from the execution options. - /// This enables cross-pipeline queries (e.g., --to-data across all pipelines). - /// To execute only specific pipelines, use SliceStrategy.Pipelines. + /// This enables cross-flow queries (e.g., --to-data across all flows). + /// To execute only specific flows, use SliceStrategy.Flows. /// /// The method performs: - /// 1. Pipeline merging into unified DAG + /// 1. Flow merging into unified DAG /// 2. Service injection /// 3. DAG building and slice application /// 4. Metadata export (if requested) /// 5. External input validation - /// 6. Pipeline execution (unless dry run) + /// 6. Flow execution (unless dry run) /// 7. Result formatting /// - Task ExecutePipelineAsync( + Task ExecuteFlowAsync( ExecutionOptions? options = null, bool exportMetadata = true, string? metadataOutputDirectory = null, @@ -95,74 +95,71 @@ Task ExecutePipelineAsync( ); /// - /// Gets metadata about a pipeline's structure. + /// Gets metadata about a Flow's structure. /// - /// Pipeline name - /// Pipeline metadata - /// Thrown if pipeline name not found + /// Flow name + /// Flow metadata + /// Thrown if Flow name not found /// - /// Returns structural information without executing the pipeline. - /// The pipeline must be built for accurate layer and input information. + /// Returns structural information without executing the flow. + /// The Flow must be built for accurate layer and input information. /// - PipelineMetadata GetPipelineMetadata(string pipelineName); + FlowMetadata GetFlowMetadata(string flowName); /// - /// Gets the full DAG metadata for pipeline introspection. + /// Gets the full DAG metadata for Flow introspection. /// - /// - /// Optional pipeline name to inspect a single pipeline. - /// When null, all registered pipelines are merged into a unified DAG. + /// + /// Optional Flow name to inspect a single flow. + /// When null, all registered flows are merged into a unified DAG. /// /// - /// Optional slice strategy to filter the DAG (e.g., from-node, to-data). + /// Optional slice strategy to filter the DAG (e.g., from-node). /// When provided, the returned metadata includes slice overlay information - /// (SlicedNodeIds and SlicedCatalogEntryKeys) identifying which nodes + /// (SlicedStepIds and SlicedItemKeys) identifying which nodes /// and data are in the active execution subset. /// /// - /// Full DAG metadata including nodes, catalog entries, edges, schemas, + /// Full DAG metadata including steps, catalog entries, edges, schemas, /// and producer-consumer relationships. /// /// - /// Thrown if is specified but not found. + /// Thrown if is specified but not found. /// /// - /// This method does not execute the pipeline. It returns structural metadata + /// This method does not execute the flow. It returns structural metadata /// useful for visualization, impact analysis, data lineage, and debugging. /// /// Examples: /// - /// // Inspect all pipelines merged + /// // Inspect all Flow merged /// var dag = flowthru.GetDagMetadata(); /// - /// // Inspect a single pipeline + /// // Inspect a single flow /// var dag = flowthru.GetDagMetadata("DataProcessing"); /// - /// // Inspect downstream of a specific node - /// var dag = flowthru.GetDagMetadata(sliceStrategy: new PipelineSliceStrategy + /// // Inspect downstream of a specific Flow node + /// var dag = flowthru.GetDagMetadata(sliceStrategy: new FlowSliceStrategy /// { - /// FromNodes = new HashSet<string> { "PreprocessCompanies" } + /// FromSteps = new HashSet<string> { "PreprocessCompanies" } /// }); /// /// - DagMetadata GetDagMetadata( - string? pipelineName = null, - PipelineSliceStrategy? sliceStrategy = null - ); + DagMetadata GetDagMetadata(string? flowName = null, FlowSliceStrategy? sliceStrategy = null); /// - /// Validates all external inputs (Layer 0) for a pipeline. + /// Validates all external inputs (Layer 0) for a flow. /// - /// Pipeline name + /// Flow name /// Cancellation token /// Validation result - /// Thrown if pipeline name not found + /// Thrown if Flow name not found /// - /// Checks accessibility of external data sources without executing the pipeline. + /// Checks accessibility of external data sources without executing the flow. /// Useful for pre-flight validation in CI/CD or scheduled jobs. /// - Task ValidatePipelineAsync( - string pipelineName, + Task ValidateFlowAsync( + string flowName, CancellationToken cancellationToken = default ); } diff --git a/src/core/Flowthru.Core/Services/Models/PipelineExecutionRequest.cs b/src/core/Flowthru.Core/Services/Models/PipelineExecutionRequest.cs deleted file mode 100644 index fd61dc51..00000000 --- a/src/core/Flowthru.Core/Services/Models/PipelineExecutionRequest.cs +++ /dev/null @@ -1,47 +0,0 @@ -using Flowthru.Pipelines; - -namespace Flowthru.Services.Models; - -/// -/// Request model for pipeline execution. -/// -/// -/// Encapsulates all configuration needed to execute a pipeline programmatically, -/// separate from CLI argument parsing. -/// -public record PipelineExecutionRequest -{ - /// - /// Name of the pipeline to execute. - /// - public required string PipelineName { get; init; } - - /// - /// Execution options (dry run, parallel execution, etc.). - /// - /// - /// If null, uses default execution options. - /// - public ExecutionOptions? Options { get; init; } - - /// - /// Optional slicing strategy to execute a subset of the pipeline. - /// - /// - /// If null or IsSliced=false, the entire pipeline executes. - /// - public PipelineSliceStrategy? SliceStrategy { get; init; } - - /// - /// Whether to export DAG metadata. - /// - /// - /// Defaults to true. Only applies if a metadata builder is configured. - /// - public bool ExportMetadata { get; init; } = true; - - /// - /// Output directory for metadata (if null, uses default from metadata builder). - /// - public string? MetadataOutputDirectory { get; init; } -} diff --git a/src/core/Flowthru.Core/Services/Models/PipelineMetadata.cs b/src/core/Flowthru.Core/Services/Models/PipelineMetadata.cs index 5c1980e8..dee4620c 100644 --- a/src/core/Flowthru.Core/Services/Models/PipelineMetadata.cs +++ b/src/core/Flowthru.Core/Services/Models/PipelineMetadata.cs @@ -1,31 +1,31 @@ namespace Flowthru.Services.Models; /// -/// Metadata about a pipeline's structure and configuration. +/// Metadata about a flow's structure and configuration. /// /// -/// Provides read-only information about a pipeline without executing it. +/// Provides read-only information about a Flow without executing it. /// Useful for discovery, validation, and UI generation. /// -public sealed record PipelineMetadata +public sealed record FlowMetadata { /// - /// The pipeline's registered name. + /// The flow's registered name. /// public required string Name { get; init; } /// - /// Optional description of the pipeline's purpose. + /// Optional description of the flow's purpose. /// public string? Description { get; init; } /// - /// Total number of nodes in the pipeline. + /// Total number of steps in the flow. /// - public required int NodeCount { get; init; } + public required int StepCount { get; init; } /// - /// Number of execution layers in the pipeline's DAG. + /// Number of execution layers in the flow's DAG. /// public required int LayerCount { get; init; } @@ -35,7 +35,7 @@ public sealed record PipelineMetadata public required IReadOnlyList ExternalInputs { get; init; } /// - /// Whether the pipeline has been built (DAG analyzed). + /// Whether the Flow has been built (DAG analyzed). /// public required bool IsBuilt { get; init; } } diff --git a/src/core/Flowthru.Core/Steps/Factory/NodeFactory.cs b/src/core/Flowthru.Core/Steps/Factory/NodeFactory.cs new file mode 100644 index 00000000..6807fa11 --- /dev/null +++ b/src/core/Flowthru.Core/Steps/Factory/NodeFactory.cs @@ -0,0 +1,42 @@ +namespace Flowthru.Steps.Factory; + +/// +/// Factory for creating step instances using TypeActivator. +/// +/// +/// +/// Design Pattern: Factory Pattern - provides a centralized location for +/// step instantiation logic. +/// +/// +/// This is a thin wrapper around TypeActivator, providing a domain-specific API for +/// creating steps. Could be extended in the future with: +/// - Step validation logic +/// - Pre/post-creation hooks +/// - Step decoration/wrapping +/// +/// +public static class StepFactory +{ + /// + /// Creates a new instance of the specified step type. + /// + /// The step type to instantiate + /// A new step instance + /// + /// + /// Requirements: + /// - TStep must inherit from StepBase<TInput, TOutput> + /// - TStep must have a parameterless constructor + /// + /// + /// These requirements are enforced at compile-time via generic constraints in + /// FlowBuilder.AddStep methods. + /// + /// + public static TStep Create() + where TStep : new() + { + return TypeActivator.Create(); + } +} diff --git a/src/core/Flowthru.Core/Nodes/Factory/TypeActivator.cs b/src/core/Flowthru.Core/Steps/Factory/TypeActivator.cs similarity index 98% rename from src/core/Flowthru.Core/Nodes/Factory/TypeActivator.cs rename to src/core/Flowthru.Core/Steps/Factory/TypeActivator.cs index a65759d9..e9a9f372 100644 --- a/src/core/Flowthru.Core/Nodes/Factory/TypeActivator.cs +++ b/src/core/Flowthru.Core/Steps/Factory/TypeActivator.cs @@ -1,7 +1,7 @@ using System.Collections.Concurrent; using System.Linq.Expressions; -namespace Flowthru.Nodes.Factory; +namespace Flowthru.Steps.Factory; /// /// Factory for creating instances of types using compiled expression trees for performance. @@ -18,7 +18,7 @@ namespace Flowthru.Nodes.Factory; /// - Significantly faster than Activator.CreateInstance<T>() for repeated calls /// /// -/// Inspiration: ChainSharp uses similar pattern for node instantiation. +/// Inspiration: ChainSharp uses similar pattern for step instantiation. /// /// /// Thread Safety: This class is thread-safe. Multiple threads can safely diff --git a/src/core/Flowthru.Core/Nodes/NoData.cs b/src/core/Flowthru.Core/Steps/NoData.cs similarity index 73% rename from src/core/Flowthru.Core/Nodes/NoData.cs rename to src/core/Flowthru.Core/Steps/NoData.cs index be04a88e..f8f67b2d 100644 --- a/src/core/Flowthru.Core/Nodes/NoData.cs +++ b/src/core/Flowthru.Core/Steps/NoData.cs @@ -1,8 +1,8 @@ -namespace Flowthru.Nodes; +namespace Flowthru.Steps; /// /// Marker type representing "no meaningful data" for nodes with side-effects or data generation. -/// Used as input/output type in NodeBase when a node doesn't consume or produce meaningful data. +/// Used as input/output type in StepBase when a step doesn't consume or produce meaningful data. /// /// /// @@ -18,8 +18,8 @@ namespace Flowthru.Nodes; /// Usage Examples: /// /// -/// // Node with no inputs (data generation) -/// public class GenerateDataNode : NodeBase<NoData, OutputSchema> +/// // Step with no inputs (data generation) +/// public class GenerateDataStep : StepBase<NoData, OutputSchema> /// { /// protected override Task<IEnumerable<OutputSchema>> Transform(IEnumerable<NoData> input) /// { @@ -28,8 +28,8 @@ namespace Flowthru.Nodes; /// } /// } /// -/// // Node with no outputs (side-effects only) -/// public class ValidateNode : NodeBase<InputSchema, NoData> +/// // Step with no outputs (side-effects only) +/// public class ValidateStep : StepBase<InputSchema, NoData> /// { /// protected override Task<IEnumerable<NoData>> Transform(IEnumerable<InputSchema> input) /// { @@ -44,12 +44,12 @@ namespace Flowthru.Nodes; /// /// /// // Simple syntax with automatic unique key generation -/// pipeline.AddNode<ValidationNode>( +/// pipeline.AddStep<ValidationStep>( /// input: catalog.InputData, /// output: NoData.Output // or just: NoData.Discard /// ); /// -/// pipeline.AddNode<GenerateDataNode>( +/// pipeline.AddStep<GenerateDataStep>( /// input: NoData.Input, // or just: NoData.None /// output: catalog.GeneratedData /// ); @@ -61,49 +61,43 @@ public sealed class NoData /// /// Singleton instance of NoData. - /// Use this value when returning NoData from node transformations. + /// Use this value when returning NoData from step transformations. /// public static readonly NoData Value = new(); /// - /// Creates a unique null catalog entry for use as a node input (no-input nodes). + /// Creates a unique null catalog entry for use as a step input (no-input nodes). /// Each call generates a new instance with a unique key to avoid DAG conflicts. /// /// /// Alias for readability in pipeline declarations where nodes don't consume external inputs. /// - public static Data.ICatalogEntry Input => - Data.CatalogEntries.Null( - $"_nodata_input_{Interlocked.Increment(ref _uniqueIdCounter)}" - ); + public static Data.IItem Input => + Data.ItemFactory.Null($"_nodata_input_{Interlocked.Increment(ref _uniqueIdCounter)}"); /// - /// Creates a unique null catalog entry for use as a node output (side-effect-only nodes). + /// Creates a unique null catalog entry for use as a step output (side-effect-only nodes). /// Each call generates a new instance with a unique key to avoid DAG conflicts. /// /// /// Alias for readability in pipeline declarations where nodes produce no meaningful output. /// - public static Data.ICatalogEntry Output => - Data.CatalogEntries.Null( - $"_nodata_output_{Interlocked.Increment(ref _uniqueIdCounter)}" - ); + public static Data.IItem Output => + Data.ItemFactory.Null($"_nodata_output_{Interlocked.Increment(ref _uniqueIdCounter)}"); /// - /// Creates a unique null catalog entry for use as a node output (side-effect-only nodes). + /// Creates a unique null catalog entry for use as a step output (side-effect-only nodes). /// Semantic alias for Output - use whichever reads better in context. /// - public static Data.ICatalogEntry Discard => - Data.CatalogEntries.Null( - $"_nodata_discard_{Interlocked.Increment(ref _uniqueIdCounter)}" - ); + public static Data.IItem Discard => + Data.ItemFactory.Null($"_nodata_discard_{Interlocked.Increment(ref _uniqueIdCounter)}"); /// - /// Creates a unique null catalog entry for use as a node input (no-input nodes). + /// Creates a unique null catalog entry for use as a step input (no-input nodes). /// Semantic alias for Input - use whichever reads better in context. /// - public static Data.ICatalogEntry None => - Data.CatalogEntries.Null($"_nodata_none_{Interlocked.Increment(ref _uniqueIdCounter)}"); + public static Data.IItem None => + Data.ItemFactory.Null($"_nodata_none_{Interlocked.Increment(ref _uniqueIdCounter)}"); /// /// Returns the standard NoData result for side-effect-only nodes. diff --git a/src/core/Flowthru.Core/Nodes/NoParams.cs b/src/core/Flowthru.Core/Steps/NoParams.cs similarity index 61% rename from src/core/Flowthru.Core/Nodes/NoParams.cs rename to src/core/Flowthru.Core/Steps/NoParams.cs index f367ec25..13a0cddf 100644 --- a/src/core/Flowthru.Core/Nodes/NoParams.cs +++ b/src/core/Flowthru.Core/Steps/NoParams.cs @@ -1,14 +1,14 @@ -namespace Flowthru.Nodes; +namespace Flowthru.Steps; /// /// Marker type for nodes that don't require parameters. -/// Used as the default TParameters type in NodeBase<TInput, TOutput, TParameters>. +/// Used as the default TParameters type in StepBase<TInput, TOutput, TParameters>. /// /// /// /// This is a simple empty class that serves as a shorthand for users and the library -/// when no parameters are needed for a node. Nodes that don't need configuration can -/// omit the third type parameter by using the two-parameter NodeBase<TInput, TOutput> +/// when no parameters are needed for a node. Steps that don't need configuration can +/// omit the third type parameter by using the two-parameter StepBase<TInput, TOutput> /// convenience base class. /// /// @@ -16,13 +16,13 @@ namespace Flowthru.Nodes; /// /// /// // Explicit NoParams (rarely needed) -/// public class MyNode : NodeBase<Input, Output, NoParams> { } +/// public class MyStep : StepBase<Input, Output, NoParams> { } /// /// // Recommended: Use two-parameter base class -/// public class MyNode : NodeBase<Input, Output> { } +/// public class MyStep : StepBase<Input, Output> { } /// /// // With parameters -/// public class ConfigurableNode : NodeBase<Input, Output, MyParameters> +/// public class ConfigurableStep : StepBase<Input, Output, MyParameters> /// { /// // Parameters property is automatically available /// } diff --git a/src/core/Flowthru.SourceGenerators/AnalyzerReleases.Unshipped.md b/src/core/Flowthru.SourceGenerators/AnalyzerReleases.Unshipped.md index 80204708..7fbb586c 100644 --- a/src/core/Flowthru.SourceGenerators/AnalyzerReleases.Unshipped.md +++ b/src/core/Flowthru.SourceGenerators/AnalyzerReleases.Unshipped.md @@ -7,7 +7,7 @@ | ------- | --------------------- | -------- | --------------------------------------------------------------------------- | | FT1001 | Flowthru.Schema | Error | FlowthruSchema type must be partial | | FT1002 | Flowthru.Schema | Warning | Conflicting manual schema interface | -| FT2001 | Flowthru.Registration | Error | Pipeline requires catalog not registered via UseCatalog | +| FT2001 | Flowthru.Registration | Error | Pipeline requires catalog not registered via RegisterCatalog | | FT2002 | Flowthru.Registration | Warning | Catalog registered but not referenced by any pipeline | | FT2003 | Flowthru.Registration | Warning | Concrete pipeline parameter resolved from DI; consider configurationSection | | FT2004 | Flowthru.Registration | Error | configurationSection specified but UseConfiguration() not called | diff --git a/src/core/Flowthru.SourceGenerators/FlowthruRegistrationAnalyzer.cs b/src/core/Flowthru.SourceGenerators/FlowthruRegistrationAnalyzer.cs index f35ebfbb..3e9426f9 100644 --- a/src/core/Flowthru.SourceGenerators/FlowthruRegistrationAnalyzer.cs +++ b/src/core/Flowthru.SourceGenerators/FlowthruRegistrationAnalyzer.cs @@ -7,74 +7,105 @@ namespace Flowthru.SourceGenerators; /// -/// Roslyn analyzer that cross-references UseCatalog and RegisterPipeline calls +/// Roslyn analyzer that cross-references RegisterCatalog and RegisterFlow calls /// within an AddFlowthru configuration block. /// /// Emits compile-time diagnostics when: -/// FT1001 — A pipeline delegate parameter extends DataCatalogBase but no -/// matching UseCatalog registration was found. -/// FT1002 — A UseCatalog registration is never referenced by any pipeline. +/// FT1001 — A Flow delegate parameter extends DataCatalogBase but no +/// matching RegisterCatalog registration was found. +/// FT1002 — A RegisterCatalog registration is never referenced by any flow. /// [DiagnosticAnalyzer(LanguageNames.CSharp)] public class FlowthruRegistrationAnalyzer : DiagnosticAnalyzer { + /// + /// No catalog registered for a Flow parameter that extends DataCatalogBase. + /// public const string MissingCatalogId = "FT2001"; + + /// + /// A catalog was registered via RegisterCatalog but is not referenced by any flow. + /// public const string UnusedCatalogId = "FT2002"; + + /// + /// A Flow has a concrete-class parameter that is not a catalog and is not covered by + /// configurationSection, meaning it will be resolved from DI. This may be intentional, + /// but if the parameter is a configuration POCO, the user should pass configurationSection + /// to RegisterFlow to bind it from appsettings instead. + /// public const string UnboundConcreteParamId = "FT2003"; + + /// + /// A Flow registration specifies a configurationSection but UseConfiguration() was never called + /// on the builder. The Flow will throw at pre-flight time when it attempts to resolve the + /// configurationSection. + /// public const string MissingUseConfigurationId = "FT2004"; - private static readonly DiagnosticDescriptor MissingCatalogRule = + private static readonly DiagnosticDescriptor _missingCatalogRule = new( MissingCatalogId, "Missing catalog registration", - "Pipeline '{0}' requires catalog '{1}' but no matching UseCatalog registration was found", + "Flow '{0}' requires catalog '{1}' but no matching RegisterCatalog registration was found", "Flowthru.Registration", DiagnosticSeverity.Error, isEnabledByDefault: true, - description: "Every DataCatalogBase-derived parameter in a RegisterPipeline delegate must have a corresponding UseCatalog registration." + description: "Every DataCatalogBase-derived parameter in a RegisterFlow delegate must have a corresponding RegisterCatalog registration." ); - private static readonly DiagnosticDescriptor UnusedCatalogRule = + private static readonly DiagnosticDescriptor _unusedCatalogRule = new( UnusedCatalogId, "Unused catalog registration", - "Catalog '{0}' is registered via UseCatalog but is not referenced by any pipeline", + "Catalog '{0}' is registered via RegisterCatalog but is not referenced by any flow", "Flowthru.Registration", DiagnosticSeverity.Warning, isEnabledByDefault: true, - description: "A catalog was registered but no pipeline references it. This may indicate dead configuration." + description: "A catalog was registered but no flow references it. This may indicate dead configuration." ); - private static readonly DiagnosticDescriptor UnboundConcreteParamRule = + private static readonly DiagnosticDescriptor _unboundConcreteParamRule = new( UnboundConcreteParamId, "Unbound concrete parameter", - "Pipeline '{0}' has parameter '{1}' of type '{2}' that will be resolved from DI. If it is a configuration object, pass configurationSection to RegisterPipeline.", + "Flow '{0}' has parameter '{1}' of type '{2}' that will be resolved from DI. If it is a configuration object, pass configurationSection to RegisterFlow.", "Flowthru.Registration", DiagnosticSeverity.Warning, isEnabledByDefault: true, - description: "A concrete-class pipeline parameter that is not a catalog will be resolved from DI at pipeline-build time. If it is a configuration POCO, pass configurationSection to RegisterPipeline to bind it from appsettings instead." + description: "A concrete-class flow parameter that is not a catalog will be resolved from DI at flow-build time. If it is a configuration POCO, pass configurationSection to RegisterFlow to bind it from appsettings instead." ); - private static readonly DiagnosticDescriptor MissingUseConfigurationRule = + private static readonly DiagnosticDescriptor _missingUseConfigurationRule = new( MissingUseConfigurationId, "Missing UseConfiguration call", - "Pipeline '{0}' specifies configurationSection '{1}' but UseConfiguration() has not been called", + "Flow '{0}' specifies configurationSection '{1}' but UseConfiguration() has not been called", "Flowthru.Registration", DiagnosticSeverity.Error, isEnabledByDefault: true, - description: "A RegisterPipeline call references a configurationSection, but UseConfiguration() was never called on the builder. The pipeline will throw at pre-flight time." + description: "A RegisterFlow call references a configurationSection, but UseConfiguration() was never called on the builder. The flow will throw at pre-flight time." ); + /// + /// Supported diagnostics for this analyzer. Each descriptor corresponds to a specific + /// registration issue. + /// public override ImmutableArray SupportedDiagnostics => ImmutableArray.Create( - MissingCatalogRule, - UnusedCatalogRule, - UnboundConcreteParamRule, - MissingUseConfigurationRule + _missingCatalogRule, + _unusedCatalogRule, + _unboundConcreteParamRule, + _missingUseConfigurationRule ); + /// + /// Initializes the analyzer by registering an operation block action that analyzes + /// the body of the lambda passed to AddFlowthru. We look for RegisterCatalog and + /// RegisterFlow calls, extract their relevant information, and cross-reference them to + /// emit diagnostics for any inconsistencies. + /// + /// public override void Initialize(AnalysisContext context) { context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None); @@ -85,24 +116,30 @@ public override void Initialize(AnalysisContext context) private static void AnalyzeOperationBlock(OperationBlockAnalysisContext context) { // We look for invocations of AddFlowthru that pass a lambda configuring the builder. - // Within that lambda, we collect UseCatalog and RegisterPipeline calls. + // Within that lambda, we collect RegisterCatalog and RegisterFlow calls. var dataCatalogBaseType = context.Compilation.GetTypeByMetadataName( "Flowthru.Data.DataCatalogBase" ); if (dataCatalogBaseType == null) + { return; + } foreach (var block in context.OperationBlocks) { foreach (var operation in block.DescendantsAndSelf()) { if (operation is not IInvocationOperation invocation) + { continue; + } // Match: services.AddFlowthru(flowthru => { ... }) if (invocation.TargetMethod.Name != "AddFlowthru") + { continue; + } // Find the lambda argument var lambdaArg = invocation @@ -113,7 +150,9 @@ private static void AnalyzeOperationBlock(OperationBlockAnalysisContext context) .FirstOrDefault(); if (lambdaArg?.Body == null) + { continue; + } AnalyzeFlowthruBlock(context, lambdaArg.Body, dataCatalogBaseType); } @@ -126,13 +165,13 @@ private static void AnalyzeFlowthruBlock( INamedTypeSymbol dataCatalogBaseType ) { - // Collect registered catalog types from UseCatalog calls + // Collect registered catalog types from RegisterCatalog calls var registeredCatalogs = new System.Collections.Generic.Dictionary< string, IInvocationOperation >(); - // Collect pipeline registrations with their required catalogs and any ambiguous concrete params - var pipelineRegistrations = new System.Collections.Generic.List<( + // Collect Flow registrations with their required catalogs and any ambiguous concrete params + var flowRegistrations = new System.Collections.Generic.List<( string Label, IInvocationOperation Invocation, System.Collections.Generic.List RequiredCatalogs, @@ -146,14 +185,16 @@ INamedTypeSymbol dataCatalogBaseType foreach (var descendant in body.DescendantsAndSelf()) { if (descendant is not IInvocationOperation call) + { continue; + } var methodName = call.TargetMethod.Name; - // ── UseCatalog ── - if (methodName == "UseCatalog") + // ── RegisterCatalog ── + if (methodName == "RegisterCatalog") { - var catalogType = ResolveCatalogTypeFromUseCatalog(call, dataCatalogBaseType); + var catalogType = ResolveCatalogTypeFromRegisterCatalog(call, dataCatalogBaseType); if (catalogType != null) { var key = catalogType.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); @@ -161,23 +202,23 @@ INamedTypeSymbol dataCatalogBaseType } } - // ── RegisterPipeline ── - if (methodName == "RegisterPipeline") + // ── RegisterFlow ── + if (methodName == "RegisterFlow") { - var (label, requiredCatalogs, ambiguousConcreteParams) = ResolvePipelineRequirements( + var (label, requiredCatalogs, ambiguousConcreteParams) = ResolveFlowRequirements( call, dataCatalogBaseType ); if (label != null) { - pipelineRegistrations.Add((label, call, requiredCatalogs, ambiguousConcreteParams)); + flowRegistrations.Add((label, call, requiredCatalogs, ambiguousConcreteParams)); } } } - // Cross-reference: FT2001 — pipeline requires catalog not registered + // Cross-reference: FT2001 — Flow requires catalog not registered var allReferencedCatalogs = new System.Collections.Generic.HashSet(); - foreach (var (label, invocation, requiredCatalogs, _) in pipelineRegistrations) + foreach (var (label, invocation, requiredCatalogs, _) in flowRegistrations) { foreach (var required in requiredCatalogs) { @@ -187,7 +228,7 @@ INamedTypeSymbol dataCatalogBaseType if (!registeredCatalogs.ContainsKey(key)) { var diagnostic = Diagnostic.Create( - MissingCatalogRule, + _missingCatalogRule, invocation.Syntax.GetLocation(), label, required.ToDisplayString(SymbolDisplayFormat.MinimallyQualifiedFormat) @@ -210,7 +251,7 @@ INamedTypeSymbol dataCatalogBaseType : "unknown"; var diagnostic = Diagnostic.Create( - UnusedCatalogRule, + _unusedCatalogRule, kvp.Value.Syntax.GetLocation(), catalogName ); @@ -219,12 +260,12 @@ INamedTypeSymbol dataCatalogBaseType } // FT2003 — concrete non-catalog parameter not bound from configuration - foreach (var (label, invocation, _, ambiguousConcreteParams) in pipelineRegistrations) + foreach (var (label, invocation, _, ambiguousConcreteParams) in flowRegistrations) { foreach (var (paramType, paramName) in ambiguousConcreteParams) { var diagnostic = Diagnostic.Create( - UnboundConcreteParamRule, + _unboundConcreteParamRule, invocation.Syntax.GetLocation(), label, paramName, @@ -237,7 +278,7 @@ INamedTypeSymbol dataCatalogBaseType // FT2004 — configurationSection supplied but UseConfiguration never called if (!hasUseConfiguration) { - foreach (var (label, invocation, _, _) in pipelineRegistrations) + foreach (var (label, invocation, _, _) in flowRegistrations) { string? sectionValue = null; foreach (var arg in invocation.Arguments) @@ -256,7 +297,7 @@ INamedTypeSymbol dataCatalogBaseType if (sectionValue != null) { var diagnostic = Diagnostic.Create( - MissingUseConfigurationRule, + _missingUseConfigurationRule, invocation.Syntax.GetLocation(), label, sectionValue @@ -268,25 +309,27 @@ INamedTypeSymbol dataCatalogBaseType } /// - /// Extracts the catalog type from a UseCatalog call. - /// Handles: UseCatalog<T>(), UseCatalog(instance), UseCatalog<T>(factory). + /// Extracts the catalog type from a RegisterCatalog call. + /// Handles: RegisterCatalog<T>(), RegisterCatalog(instance), RegisterCatalog<T>(factory). /// - private static ITypeSymbol? ResolveCatalogTypeFromUseCatalog( + private static ITypeSymbol? ResolveCatalogTypeFromRegisterCatalog( IInvocationOperation call, INamedTypeSymbol dataCatalogBaseType ) { var method = call.TargetMethod; - // Generic: UseCatalog() or UseCatalog(factory) + // Generic: RegisterCatalog() or RegisterCatalog(factory) if (method.TypeArguments.Length == 1) { var typeArg = method.TypeArguments[0]; if (InheritsFrom(typeArg, dataCatalogBaseType)) + { return typeArg; + } } - // Non-generic: UseCatalog(catalogInstance) — infer from argument type. + // Non-generic: RegisterCatalog(catalogInstance) — infer from argument type. // Strip through implicit conversions: passing UpstreamCatalog to a DataCatalogBase // parameter wraps the expression in IConversionOperation; the concrete type lives // on the innermost operand, not on the outer conversion. @@ -294,26 +337,33 @@ INamedTypeSymbol dataCatalogBaseType { IOperation argValue = call.Arguments[0].Value; while (argValue is IConversionOperation conv && conv.IsImplicit) + { argValue = conv.Operand; + } + var argType = argValue.Type; if (argType != null && InheritsFrom(argType, dataCatalogBaseType)) + { return argType; + } } - // Infer from lambda return type: UseCatalog(_ => new MyCatalog(...)) + // Infer from lambda return type: RegisterCatalog(_ => new MyCatalog(...)) if (method.TypeArguments.Length == 1) + { return method.TypeArguments[0]; + } return null; } /// - /// Extracts the pipeline label, required catalog types, and unbound concrete parameters - /// from a RegisterPipeline call. + /// Extracts the Flow label, required catalog types, and unbound concrete parameters + /// from a RegisterFlow call. /// /// Parameters are classified the same way the runtime resolver does: /// - /// Extends DataCatalogBase → required catalog (FT2001 if missing UseCatalog) + /// Extends DataCatalogBase → required catalog (FT2001 if missing RegisterCatalog) /// Interface → DI-resolved service (no warning — extension territory) /// Concrete class, not covered by configurationSection → ambiguous (FT2003) /// @@ -323,7 +373,7 @@ private static ( string? Label, System.Collections.Generic.List RequiredCatalogs, System.Collections.Generic.List<(ITypeSymbol Type, string ParamName)> AmbiguousConcreteParams - ) ResolvePipelineRequirements(IInvocationOperation call, INamedTypeSymbol dataCatalogBaseType) + ) ResolveFlowRequirements(IInvocationOperation call, INamedTypeSymbol dataCatalogBaseType) { string? label = null; var requiredCatalogs = new System.Collections.Generic.List(); @@ -359,42 +409,49 @@ string ParamName } } - // Resolve pipeline parameters via delegate signature or method group. - System.Collections.Generic.IEnumerable? pipelineParams = null; + // Resolve Flow parameters via delegate signature or method group. + System.Collections.Generic.IEnumerable? flowParams = null; foreach (var arg in call.Arguments) { - if (arg.Parameter?.Name != "pipeline") + if (arg.Parameter?.Name != "flow") + { continue; + } // Path 1: lambda / typed Func — read from the delegate's Invoke method var delegateType = ResolveMethodSignatureFromArgument(arg.Value); if (delegateType?.DelegateInvokeMethod != null) { - pipelineParams = delegateType.DelegateInvokeMethod.Parameters; + flowParams = delegateType.DelegateInvokeMethod.Parameters; break; } // Path 2: method group — unwrap any conversion wrapper and read Method.Parameters IOperation value = arg.Value; while (value is IConversionOperation conv) + { value = conv.Operand; + } + if (value is IMethodReferenceOperation methodRef) - pipelineParams = methodRef.Method.Parameters; + { + flowParams = methodRef.Method.Parameters; + } break; } - if (pipelineParams != null) + if (flowParams != null) { // The runtime consumes configurationSection on the first concrete non-catalog // non-interface param it encounters (left to right). Track that slot. bool configSectionConsumed = false; - foreach (var param in pipelineParams) + foreach (var param in flowParams) { if (InheritsFrom(param.Type, dataCatalogBaseType)) { - // Catalog — must be registered via UseCatalog. + // Catalog — must be registered via RegisterCatalog. requiredCatalogs.Add(param.Type); } else if (param.Type.TypeKind == TypeKind.Interface) @@ -425,7 +482,9 @@ string ParamName { // Unwrap conversions (method group → Delegate) while (value is IConversionOperation conversion) + { value = conversion.Operand; + } if (value is IDelegateCreationOperation delegateCreation) { @@ -438,13 +497,18 @@ string ParamName private static bool InheritsFrom(ITypeSymbol? type, INamedTypeSymbol baseType) { if (type == null) + { return false; + } var current = type.BaseType; while (current != null) { if (SymbolEqualityComparer.Default.Equals(current, baseType)) + { return true; + } + current = current.BaseType; } return false; diff --git a/src/core/Flowthru.SourceGenerators/PipelineBuilderGenerator.cs b/src/core/Flowthru.SourceGenerators/PipelineBuilderGenerator.cs index 7ebc4c5f..f3a1e9ce 100644 --- a/src/core/Flowthru.SourceGenerators/PipelineBuilderGenerator.cs +++ b/src/core/Flowthru.SourceGenerators/PipelineBuilderGenerator.cs @@ -7,10 +7,10 @@ namespace Flowthru.SourceGenerators; /// -/// Generates AddNode overloads for PipelineBuilder supporting up to 8 inputs and 8 outputs. +/// Generates AddStep overloads for FlowBuilder supporting up to 8 inputs and 8 outputs. /// [Generator] -public class PipelineBuilderGenerator : IIncrementalGenerator +public class FlowBuilderGenerator : IIncrementalGenerator { private const int MaxInputs = 8; private const int MaxOutputs = 8; @@ -27,12 +27,12 @@ public void Initialize(IncrementalGeneratorInitializationContext context) if (name != "Flowthru.Core") return; - GenerateAddNodeOverloads(ctx); + GenerateAddStepOverloads(ctx); } ); } - private static void GenerateAddNodeOverloads(SourceProductionContext context) + private static void GenerateAddStepOverloads(SourceProductionContext context) { var sb = new StringBuilder(); @@ -43,11 +43,11 @@ private static void GenerateAddNodeOverloads(SourceProductionContext context) #nullable enable using Flowthru.Data; - using Flowthru.Nodes; + using Flowthru.Steps; - namespace Flowthru.Pipelines; + namespace Flowthru.Flows; - public partial class PipelineBuilder + public partial class FlowBuilder { """ ); @@ -64,22 +64,19 @@ public partial class PipelineBuilder } // Generate both async and sync versions - GenerateAddNodeOverload(sb, inputs, outputs, isAsync: true); + GenerateAddStepOverload(sb, inputs, outputs, isAsync: true); sb.AppendLine(); - GenerateAddNodeOverload(sb, inputs, outputs, isAsync: false); + GenerateAddStepOverload(sb, inputs, outputs, isAsync: false); sb.AppendLine(); } } sb.AppendLine("}"); - context.AddSource( - "PipelineBuilder.Generated.cs", - SourceText.From(sb.ToString(), Encoding.UTF8) - ); + context.AddSource("FlowBuilder.Generated.cs", SourceText.From(sb.ToString(), Encoding.UTF8)); } - private static void GenerateAddNodeOverload( + private static void GenerateAddStepOverload( StringBuilder sb, int inputCount, int outputCount, @@ -97,7 +94,7 @@ bool isAsync sb.AppendLine( $$""" - public PipelineBuilder AddNode<{{typeParams}}>( + public FlowBuilder AddStep<{{typeParams}}>( string label, {{funcSignature}} transform, {{inputParam}} input, @@ -120,15 +117,15 @@ bool isAsync sb.AppendLine( $$""" - var pipelineNode = new PipelineNode( + var flowStep = new FlowStep( label: label, description: description, - node: transform, - inputs: new List { {{inputsList}} }, - outputs: new List { {{outputsList}} } + step: transform, + inputs: new List { {{inputsList}} }, + outputs: new List { {{outputsList}} } ); - _pipeline.AddNode(pipelineNode); + _flow.AddStep(flowStep); return this; } """ @@ -146,7 +143,7 @@ bool isAsync sb.AppendLine( $""" /// - /// Adds a node with {inputCount} input{( + /// Adds a step with {inputCount} input{( inputCount == 1 ? "" : "s" )} and {outputCount} output{(outputCount == 1 ? "" : "s")}{asyncDesc}. /// @@ -169,11 +166,11 @@ bool isAsync : "Synchronous transformation function"; sb.AppendLine( $""" - /// Unique identifier for this node + /// Unique identifier for this step /// {transformDesc} - /// Catalog entry or tuple of catalog entries providing input data - /// Catalog entry or tuple of catalog entries to store output data - /// Optional description of the node's purpose + /// Catalog item or tuple of catalog items providing input data + /// Catalog item or tuple of catalog items to store output data + /// Optional description of the step's purpose /// This builder for method chaining """ ); @@ -214,13 +211,10 @@ private static string GenerateInputParameter(int inputCount) { if (inputCount == 1) { - return "ICatalogEntry"; + return "IItem"; } - var types = string.Join( - ", ", - Enumerable.Range(1, inputCount).Select(i => $"ICatalogEntry") - ); + var types = string.Join(", ", Enumerable.Range(1, inputCount).Select(i => $"IItem")); return $"({types})"; } @@ -228,13 +222,10 @@ private static string GenerateOutputParameter(int outputCount) { if (outputCount == 1) { - return "ICatalogEntry"; + return "IItem"; } - var types = string.Join( - ", ", - Enumerable.Range(1, outputCount).Select(i => $"ICatalogEntry") - ); + var types = string.Join(", ", Enumerable.Range(1, outputCount).Select(i => $"IItem")); return $"({types})"; } diff --git a/src/extensions/Flowthru.Extensions.Csv/CsvCatalogEntryExtensions.cs b/src/extensions/Flowthru.Extensions.Csv/CsvCatalogEntryExtensions.cs index 7daa144c..05e76ad5 100644 --- a/src/extensions/Flowthru.Extensions.Csv/CsvCatalogEntryExtensions.cs +++ b/src/extensions/Flowthru.Extensions.Csv/CsvCatalogEntryExtensions.cs @@ -7,15 +7,15 @@ namespace Flowthru.Data; /// -/// Extension methods that add CSV support to . +/// Extension methods that add CSV support to . /// -public static class CsvCatalogEntryExtensions +public static class CsvItemExtensions { /// /// Creates a CSV file catalog entry with IEnumerable container. /// /// Row schema type (must be flat and text-serializable) - /// The enumerable catalog entries factory (from ) + /// The enumerable catalog entries factory (from ) /// Unique catalog label for DAG resolution /// Path to CSV file /// Catalog entry with file + CSV + IEnumerable composition @@ -35,8 +35,8 @@ public static class CsvCatalogEntryExtensions /// All other traits use filesystem baseline defaults /// /// - public static CatalogEntry> Csv( - this EnumerableCatalogEntries _, + public static Item> Csv( + this EnumerableItemFactory _, string label, string filePath ) @@ -47,7 +47,7 @@ string filePath var container = new EnumerableContainerAdapter(); var storage = new ComposedStorageAdapter, TRow>(medium, format, container); - return new CatalogEntry>(label, storage); + return new Item>(label, storage); } /// @@ -55,7 +55,7 @@ string filePath /// concatenates them into a single . /// /// Row schema type (must be flat and text-serializable) - /// The enumerable catalog entries factory (from ) + /// The enumerable catalog entries factory (from ) /// Unique catalog label for DAG resolution /// Path to the directory containing the CSV files /// Read-only catalog entry that concatenates every *.csv in the directory @@ -64,16 +64,13 @@ string filePath /// This entry is read-only — attempting to save will fail with /// . /// - public static CatalogEntry> CsvDirectory( - this EnumerableCatalogEntries _, + public static Item> CsvDirectory( + this EnumerableItemFactory _, string label, string directoryPath ) where TRow : notnull, IFlatSchema, ITextSerializable { - return new CatalogEntry>( - label, - new DirectoryCsvStorageAdapter(directoryPath) - ); + return new Item>(label, new DirectoryCsvStorageAdapter(directoryPath)); } } diff --git a/src/extensions/Flowthru.Extensions.Csv/Data/Storage/Format/CsvFormatSerializer.cs b/src/extensions/Flowthru.Extensions.Csv/Data/Storage/Format/CsvFormatSerializer.cs index 01216968..dffa576a 100644 --- a/src/extensions/Flowthru.Extensions.Csv/Data/Storage/Format/CsvFormatSerializer.cs +++ b/src/extensions/Flowthru.Extensions.Csv/Data/Storage/Format/CsvFormatSerializer.cs @@ -221,7 +221,7 @@ internal sealed class SerializedEnumCsvConverter public SerializedEnumCsvConverter() { - _metadata = Serialization.EnumMetadataRegistry.GetOrCreate(); + _metadata = Serialization.EnumMetadataRegistry.Create(); } public override object? ConvertFromString( diff --git a/src/extensions/Flowthru.Extensions.Csv/Data/Storage/Strategies/CsvStorageEntryFactory.cs b/src/extensions/Flowthru.Extensions.Csv/Data/Storage/Strategies/CsvStorageEntryFactory.cs index 3195aa4d..800b9e5b 100644 --- a/src/extensions/Flowthru.Extensions.Csv/Data/Storage/Strategies/CsvStorageEntryFactory.cs +++ b/src/extensions/Flowthru.Extensions.Csv/Data/Storage/Strategies/CsvStorageEntryFactory.cs @@ -52,22 +52,19 @@ public CsvStorageEntryFactory(string basePath) } /// - public ICatalogEntry> CreateEnumerable( - string label, - StorageOptions? options = null - ) + public IItem> CreateEnumerable(string label, StorageOptions? options = null) where T : notnull, IFlatSchema, ITextSerializable { var path = ResolvePath(label, options, ".csv"); - return CatalogEntries.Enumerable.Csv(label, path); + return ItemFactory.Enumerable.Csv(label, path); } /// - public ICatalogEntry CreateSingle(string label, StorageOptions? options = null) + public IItem CreateSingle(string label, StorageOptions? options = null) where T : IStructuredSerializable { var path = ResolvePath(label, options, ".json"); - return CatalogEntries.Single.Json(label, path); + return ItemFactory.Single.Json(label, path); } private string ResolvePath(string label, StorageOptions? options, string defaultExtension) diff --git a/src/extensions/Flowthru.Extensions.EFCore/Data/CatalogEntries.Enumerable.cs b/src/extensions/Flowthru.Extensions.EFCore/Data/CatalogEntries.Enumerable.cs index 8c6c5cf5..982b8427 100644 --- a/src/extensions/Flowthru.Extensions.EFCore/Data/CatalogEntries.Enumerable.cs +++ b/src/extensions/Flowthru.Extensions.EFCore/Data/CatalogEntries.Enumerable.cs @@ -5,7 +5,7 @@ namespace Flowthru.Extensions.EFCore.Data; -public static partial class EFCoreCatalogEntries +public static partial class EFCoreItemFactory { public static partial class Enumerable { @@ -41,19 +41,19 @@ public static partial class Enumerable /// // In catalog /// public static partial class DataCatalog /// { - /// public static ICatalogEntry<IEnumerable<Company>> Companies(DbContext db) => - /// CatalogEntries.Enumerable.EFCore<Company>("companies", db); + /// public static IItem<IEnumerable<Company>> Companies(DbContext db) => + /// ItemFactory.Enumerable.EFCore<Company>("companies", db); /// } /// /// // In pipeline - /// var pipeline = new PipelineBuilder("CompanyPipeline") - /// .AddNode("load_companies", catalog => new LoadCompaniesNode( + /// var pipeline = new FlowBuilder("CompanyPipeline") + /// .AddStep("load_companies", catalog => new LoadCompaniesStep( /// outputs: catalog.Companies(db) /// )) /// .Build(); /// /// - public static CatalogEntry> EFCore( + public static Item> EFCore( string label, DbContext context, bool allowEmptyData = false, @@ -63,7 +63,7 @@ public static CatalogEntry> EFCore( where T : class { var storage = new EFCoreStorageAdapter(context, allowEmptyData, queryCustomizer, saveFunc); - return new CatalogEntry>(label, storage); + return new Item>(label, storage); } /// @@ -98,12 +98,12 @@ public static CatalogEntry> EFCore( /// .UseSqlServer(connectionString) /// .Options); /// - /// public static ICatalogEntry<IEnumerable<Company>> Companies() => - /// CatalogEntries.Enumerable.EFCore<Company>("companies", CreateDbContext); + /// public static IItem<IEnumerable<Company>> Companies() => + /// ItemFactory.Enumerable.EFCore<Company>("companies", CreateDbContext); /// } /// /// - public static CatalogEntry> EFCore( + public static Item> EFCore( string label, Func contextFactory, bool allowEmptyData = false, @@ -118,7 +118,7 @@ public static CatalogEntry> EFCore( queryCustomizer, saveFunc ); - return new CatalogEntry>(label, storage); + return new Item>(label, storage); } /// @@ -135,7 +135,7 @@ public static CatalogEntry> EFCore( /// Optional save delegate receiving the concrete . /// Defaults to RemoveRange + AddRange when null. /// Catalog entry for EFCore database storage - public static CatalogEntry> EFCore( + public static Item> EFCore( string label, Func contextFactory, bool allowEmptyData = false, @@ -154,12 +154,12 @@ public static CatalogEntry> EFCore( queryCustomizer, baseSaveFunc ); - return new CatalogEntry>(label, storage); + return new Item>(label, storage); } /// /// Creates an EFCore catalog entry using — - /// the idiomatic EFCore pattern for per-operation context isolation and concurrent node safety. + /// the idiomatic EFCore pattern for per-operation context isolation and concurrent step safety. /// /// Entity type /// Concrete DbContext type @@ -170,7 +170,7 @@ public static CatalogEntry> EFCore( /// Optional save delegate receiving the concrete . /// Defaults to RemoveRange + AddRange when null. /// Catalog entry for EFCore database storage - public static CatalogEntry> EFCore( + public static Item> EFCore( string label, IDbContextFactory contextFactory, bool allowEmptyData = false, @@ -189,7 +189,7 @@ public static CatalogEntry> EFCore( queryCustomizer, baseSaveFunc ); - return new CatalogEntry>(label, storage); + return new Item>(label, storage); } } } diff --git a/src/extensions/Flowthru.Extensions.EFCore/Data/CatalogEntries.Single.cs b/src/extensions/Flowthru.Extensions.EFCore/Data/CatalogEntries.Single.cs index cb19eddd..fafac12c 100644 --- a/src/extensions/Flowthru.Extensions.EFCore/Data/CatalogEntries.Single.cs +++ b/src/extensions/Flowthru.Extensions.EFCore/Data/CatalogEntries.Single.cs @@ -5,7 +5,7 @@ namespace Flowthru.Extensions.EFCore.Data; -public static partial class EFCoreCatalogEntries +public static partial class EFCoreItemFactory { public static partial class Single { @@ -37,18 +37,18 @@ public static partial class Single /// /// /// // In catalog - /// public ICatalogEntry<ModelMetrics> Metrics(DbContext db) => - /// CatalogEntries.Single.EFCore<ModelMetrics>("metrics", db); + /// public IItem<ModelMetrics> Metrics(DbContext db) => + /// ItemFactory.Single.EFCore<ModelMetrics>("metrics", db); /// /// // In pipeline - /// var pipeline = new PipelineBuilder("MetricsPipeline") - /// .AddNode("save_metrics", catalog => new SaveMetricsNode( + /// var pipeline = new FlowBuilder("MetricsPipeline") + /// .AddStep("save_metrics", catalog => new SaveMetricsStep( /// outputs: catalog.Metrics(db) /// )) /// .Build(); /// /// - public static CatalogEntry EFCore( + public static Item EFCore( string label, DbContext context, bool allowEmptyData = false, @@ -64,7 +64,7 @@ public static CatalogEntry EFCore( queryCustomizer, saveFunc ); - return new CatalogEntry(label, adapter); + return new Item(label, adapter); } /// @@ -93,14 +93,14 @@ public static CatalogEntry EFCore( /// // In catalog with factory /// private readonly IServiceProvider _serviceProvider; /// - /// public ICatalogEntry<ModelMetrics> Metrics => - /// CatalogEntries.Single.EFCore<ModelMetrics>( + /// public IItem<ModelMetrics> Metrics => + /// ItemFactory.Single.EFCore<ModelMetrics>( /// "metrics", /// () => _serviceProvider.GetRequiredService<MyDbContext>() /// ); /// /// - public static CatalogEntry EFCore( + public static Item EFCore( string label, Func contextFactory, bool allowEmptyData = false, @@ -115,7 +115,7 @@ public static CatalogEntry EFCore( queryCustomizer, saveFunc ); - return new CatalogEntry(label, adapter); + return new Item(label, adapter); } /// @@ -132,7 +132,7 @@ public static CatalogEntry EFCore( /// Optional save delegate receiving the concrete . /// Defaults to clear-and-insert when null. /// Catalog entry for EFCore single entity storage - public static CatalogEntry EFCore( + public static Item EFCore( string label, Func contextFactory, bool allowEmptyData = false, @@ -151,7 +151,7 @@ public static CatalogEntry EFCore( queryCustomizer, baseSaveFunc ); - return new CatalogEntry(label, adapter); + return new Item(label, adapter); } /// @@ -166,7 +166,7 @@ public static CatalogEntry EFCore( /// Optional save delegate receiving the concrete . /// Defaults to clear-and-insert when null. /// Catalog entry for EFCore single entity storage - public static CatalogEntry EFCore( + public static Item EFCore( string label, IDbContextFactory contextFactory, bool allowEmptyData = false, @@ -185,7 +185,7 @@ public static CatalogEntry EFCore( queryCustomizer, baseSaveFunc ); - return new CatalogEntry(label, adapter); + return new Item(label, adapter); } } } diff --git a/src/extensions/Flowthru.Extensions.EFCore/Data/Storage/EFCoreSingleStorageAdapter.cs b/src/extensions/Flowthru.Extensions.EFCore/Data/Storage/EFCoreSingleStorageAdapter.cs index 0fd28e14..fa72c71d 100644 --- a/src/extensions/Flowthru.Extensions.EFCore/Data/Storage/EFCoreSingleStorageAdapter.cs +++ b/src/extensions/Flowthru.Extensions.EFCore/Data/Storage/EFCoreSingleStorageAdapter.cs @@ -309,7 +309,7 @@ private DbContext GetContext() /// /// Validates that the entity type T is properly configured in the DbContext. /// Forces EF Core model building to catch configuration errors during catalog initialization - /// (pre-flight phase) rather than at runtime during node execution. + /// (pre-flight phase) rather than at runtime during step execution. /// /// DbContext to validate against /// diff --git a/src/extensions/Flowthru.Extensions.EFCore/Data/Storage/EFCoreStorageAdapter.cs b/src/extensions/Flowthru.Extensions.EFCore/Data/Storage/EFCoreStorageAdapter.cs index 0849d90a..e8cf839c 100644 --- a/src/extensions/Flowthru.Extensions.EFCore/Data/Storage/EFCoreStorageAdapter.cs +++ b/src/extensions/Flowthru.Extensions.EFCore/Data/Storage/EFCoreStorageAdapter.cs @@ -68,14 +68,14 @@ namespace Flowthru.Data.Storage; /// /// // Injected DbContext (from DI container) /// var adapter = new EFCoreStorageAdapter<Company>(dbContext); -/// var entry = new CatalogEntry<IEnumerable<Company>>("companies", adapter); +/// var entry = new Item<IEnumerable<Company>>("companies", adapter); /// /// // Factory-based DbContext (created per operation) /// var adapter = new EFCoreStorageAdapter<Company>(() => new AppDbContext(options)); -/// var entry = new CatalogEntry<IEnumerable<Company>>("companies", adapter); +/// var entry = new Item<IEnumerable<Company>>("companies", adapter); /// /// // Read-only mode (apply constraint at catalog level) -/// var entry = new CatalogEntry<IEnumerable<Company>>("companies", adapter) +/// var entry = new Item<IEnumerable<Company>>("companies", adapter) /// .Constrain(traits => traits with { CanWrite = false }); /// /// // Allow empty tables during validation @@ -164,7 +164,7 @@ public EFCoreStorageAdapter( /// /// Validates that the entity type T is properly configured in the DbContext. /// Forces EF Core model building to catch configuration errors during catalog initialization - /// (pre-flight phase) rather than at runtime during node execution. + /// (pre-flight phase) rather than at runtime during step execution. /// /// DbContext to validate against /// diff --git a/src/extensions/Flowthru.Extensions.EFCore/README.md b/src/extensions/Flowthru.Extensions.EFCore/README.md index 33b524fc..ba94821e 100644 --- a/src/extensions/Flowthru.Extensions.EFCore/README.md +++ b/src/extensions/Flowthru.Extensions.EFCore/README.md @@ -45,9 +45,9 @@ public partial class Catalog : DataCatalogBase InitializeCatalogProperties(); } - public ICatalogEntry> Companies => + public IItem> Companies => GetOrCreateEntry(() => - EFCoreCatalogEntries.Enumerable.EFCore( + EFCoreItems.Enumerable.EFCore( label: "Companies", contextFactory: _contextFactory ) @@ -79,13 +79,13 @@ Use `EFCore` to preserve the concrete context type all the way to s ```csharp // IDbContextFactory overload — recommended for concurrent pipelines -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItems.Enumerable.EFCore( label: "Companies", contextFactory: dbContextFactory // IDbContextFactory ) // Func overload — useful when constructing contexts manually -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItems.Enumerable.EFCore( label: "Companies", contextFactory: () => new AppDbContext(options) ) @@ -96,13 +96,13 @@ EFCoreCatalogEntries.Enumerable.EFCore( The `queryCustomizer` parameter shapes the query before execution. Use it for navigation property includes, filtering, or ordering: ```csharp -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItems.Enumerable.EFCore( label: "Persons", contextFactory: _contextFactory, queryCustomizer: q => q.Include(p => p.Address).AsNoTracking() ) -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItems.Enumerable.EFCore( label: "Shuttles", contextFactory: _contextFactory, queryCustomizer: q => q.OrderBy(s => s.Id) @@ -114,7 +114,7 @@ EFCoreCatalogEntries.Enumerable.EFCore( Override the default `RemoveRange + AddRange` write strategy via `saveFunc`. The typed context is passed directly — no cast needed: ```csharp -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItems.Enumerable.EFCore( label: "Companies", contextFactory: _contextFactory, saveFunc: async (ctx, data, ct) => @@ -142,7 +142,7 @@ saveFunc: async (ctx, data, ct) => For tables that store exactly one row (trained models, configuration records, aggregated metrics): ```csharp -EFCoreCatalogEntries.Single.EFCore( +EFCoreItems.Single.EFCore( label: "ModelMetrics", contextFactory: _contextFactory ) @@ -155,7 +155,7 @@ The adapter validates "exactly one row" during pre-flight. Use `allowEmptyData: Use `.Constrain()` to prevent writes. The pipeline fails at build time — not at runtime — if a node attempts to write to a constrained entry: ```csharp -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItems.Enumerable.EFCore( label: "SourceData", contextFactory: _sourceFactory) .Constrain(traits => traits with { CanWrite = false }) @@ -166,7 +166,7 @@ EFCoreCatalogEntries.Enumerable.EFCore( By default, an empty table fails pre-flight validation. Set `allowEmptyData: true` for tables that are legitimately empty on first run: ```csharp -EFCoreCatalogEntries.Enumerable.EFCore( +EFCoreItems.Enumerable.EFCore( label: "AuditEvents", contextFactory: _contextFactory, allowEmptyData: true @@ -227,9 +227,9 @@ await db.Database.MigrateAsync(); await pipeline.ExecuteAsync(); // Option 2: Migration node in pipeline -var pipeline = new PipelineBuilder("Setup") - .AddNode("migrate", catalog => new MigrationNode(db)) - .AddNode("load_data", catalog => new LoadNode(...)) +var pipeline = new FlowBuilder("Setup") + .AddStep("migrate", catalog => new MigrationNode(db)) + .AddStep("load_data", catalog => new LoadNode(...)) .Build(); ``` diff --git a/src/extensions/Flowthru.Extensions.EFCore/Tests/CompilationTest.cs b/src/extensions/Flowthru.Extensions.EFCore/Tests/CompilationTest.cs index 6a1d9000..d91bf456 100644 --- a/src/extensions/Flowthru.Extensions.EFCore/Tests/CompilationTest.cs +++ b/src/extensions/Flowthru.Extensions.EFCore/Tests/CompilationTest.cs @@ -15,10 +15,10 @@ public void PartialClassExtensionWorks() DbContext? context = null; // Extension method from Flowthru.Extensions.EFCore - var entry = EFCoreCatalogEntries.Enumerable.EFCore("test", context!); + var entry = EFCoreItemFactory.Enumerable.EFCore("test", context!); // Verify it returns the correct type - var _ = entry as ICatalogEntry>; + var _ = entry as IItem>; } public void TypedContextFactoryOverloadsWork() @@ -26,21 +26,21 @@ public void TypedContextFactoryOverloadsWork() Func typedFactory = () => null!; // Func overload — no cast of DbContext in save delegate - var entryTypedFactory = EFCoreCatalogEntries.Enumerable.EFCore( + var entryTypedFactory = EFCoreItemFactory.Enumerable.EFCore( "test", typedFactory ); - var _ = entryTypedFactory as ICatalogEntry>; + var _ = entryTypedFactory as IItem>; // Func with typed save delegate — TContext flows to delegate, no cast needed - var entryWithSaveFunc = EFCoreCatalogEntries.Enumerable.EFCore( + var entryWithSaveFunc = EFCoreItemFactory.Enumerable.EFCore( "test", typedFactory, saveFunc: (db, data, ct) => Task.CompletedTask ); // Func with query customizer - var entryWithCustomizer = EFCoreCatalogEntries.Enumerable.EFCore( + var entryWithCustomizer = EFCoreItemFactory.Enumerable.EFCore( "test", typedFactory, queryCustomizer: q => q.Where(e => e.Id > 0) @@ -52,14 +52,14 @@ public void DbContextFactoryOverloadsWork() IDbContextFactory factory = new TestDbContextFactory(); // IDbContextFactory overload — idiomatic EFCore concurrency pattern - var entryFactory = EFCoreCatalogEntries.Enumerable.EFCore( + var entryFactory = EFCoreItemFactory.Enumerable.EFCore( "test", factory ); - var _ = entryFactory as ICatalogEntry>; + var _ = entryFactory as IItem>; // IDbContextFactory with typed save delegate - var entryWithSaveFunc = EFCoreCatalogEntries.Enumerable.EFCore( + var entryWithSaveFunc = EFCoreItemFactory.Enumerable.EFCore( "test", factory, saveFunc: (db, data, ct) => Task.CompletedTask @@ -72,18 +72,15 @@ public void SingleTypedContextOverloadsWork() IDbContextFactory factory = new TestDbContextFactory(); // Single Func with typed save delegate - var entryTyped = EFCoreCatalogEntries.Single.EFCore( + var entryTyped = EFCoreItemFactory.Single.EFCore( "test", typedFactory, saveFunc: (db, data, ct) => Task.CompletedTask ); - var _ = entryTyped as ICatalogEntry; + var _ = entryTyped as IItem; // Single IDbContextFactory - var entryFactory = EFCoreCatalogEntries.Single.EFCore( - "test", - factory - ); + var entryFactory = EFCoreItemFactory.Single.EFCore("test", factory); } private class TestEntity diff --git a/src/extensions/Flowthru.Extensions.Excel/ExcelCatalogEntryExtensions.cs b/src/extensions/Flowthru.Extensions.Excel/ExcelCatalogEntryExtensions.cs index 44e0241a..01547118 100644 --- a/src/extensions/Flowthru.Extensions.Excel/ExcelCatalogEntryExtensions.cs +++ b/src/extensions/Flowthru.Extensions.Excel/ExcelCatalogEntryExtensions.cs @@ -7,15 +7,15 @@ namespace Flowthru.Data; /// -/// Extension methods that add Excel support to . +/// Extension methods that add Excel support to . /// -public static class ExcelCatalogEntryExtensions +public static class ExcelItemExtensions { /// /// Creates a read-only Excel file catalog entry with IEnumerable container. /// /// Row schema type (must be flat and text-serializable) - /// The enumerable catalog entries factory (from ) + /// The enumerable catalog entries factory (from ) /// Unique catalog label for DAG resolution /// Path to Excel file (.xlsx) /// Name of the sheet to read @@ -39,8 +39,8 @@ public static class ExcelCatalogEntryExtensions /// CanWrite: false (Excel adapter is read-only via ExcelDataReader) /// /// - public static CatalogEntry> Excel( - this EnumerableCatalogEntries _, + public static Item> Excel( + this EnumerableItemFactory _, string label, string filePath, string sheetName @@ -52,6 +52,6 @@ string sheetName var container = new EnumerableContainerAdapter(); var storage = new ComposedStorageAdapter, TRow>(medium, format, container); - return new CatalogEntry>(label, storage); + return new Item>(label, storage); } } diff --git a/src/extensions/Flowthru.Extensions.MLNet/CatalogEntries.MLNet.cs b/src/extensions/Flowthru.Extensions.MLNet/CatalogEntries.MLNet.cs index 7a95a216..239e73b0 100644 --- a/src/extensions/Flowthru.Extensions.MLNet/CatalogEntries.MLNet.cs +++ b/src/extensions/Flowthru.Extensions.MLNet/CatalogEntries.MLNet.cs @@ -8,7 +8,7 @@ namespace Flowthru.Extensions.MLNet; /// /// /// -/// Discovery Pattern: Import this class alongside +/// Discovery Pattern: Import this class alongside /// for ML.NET-specific catalog entry factory methods. /// /// @@ -24,15 +24,15 @@ namespace Flowthru.Extensions.MLNet; /// /// using Flowthru.Data; /// using Flowthru.Extensions.MLNet; -/// +/// /// // Core entries -/// var csvEntry = CatalogEntries.Enumerable.Csv<MySchema>("data", "data.csv"); -/// +/// var csvEntry = ItemFactory.Enumerable.Csv<MySchema>("data", "data.csv"); +/// /// // MLNet entries -/// var modelEntry = CatalogEntriesMLNet.OnnxModel("model", "model.onnx"); +/// var modelEntry = ItemFactoryMLNet.OnnxModel("model", "model.onnx"); /// /// -public static class CatalogEntriesMLNet +public static class ItemFactoryMLNet { /// /// Creates a catalog entry for an ONNX model file. @@ -42,14 +42,14 @@ public static class CatalogEntriesMLNet /// A catalog entry wrapping an ONNX model storage adapter /// /// - /// var entry = CatalogEntriesMLNet.OnnxModel( + /// var entry = ItemFactoryMLNet.OnnxModel( /// label: "BertModel", /// filePath: "models/bert-base.onnx" /// ); /// /// - public static ICatalogEntry OnnxModel(string label, string filePath) + public static IItem OnnxModel(string label, string filePath) { - return new CatalogEntry(label, new OnnxModelStorageAdapter(filePath)); + return new Item(label, new OnnxModelStorageAdapter(filePath)); } } diff --git a/src/extensions/Flowthru.Extensions.Parquet/ParquetCatalogEntryExtensions.cs b/src/extensions/Flowthru.Extensions.Parquet/ParquetCatalogEntryExtensions.cs index b977dd4b..42391f7a 100644 --- a/src/extensions/Flowthru.Extensions.Parquet/ParquetCatalogEntryExtensions.cs +++ b/src/extensions/Flowthru.Extensions.Parquet/ParquetCatalogEntryExtensions.cs @@ -7,15 +7,15 @@ namespace Flowthru.Data; /// -/// Extension methods that add Parquet support to . +/// Extension methods that add Parquet support to . /// -public static class ParquetCatalogEntryExtensions +public static class ParquetItemExtensions { /// /// Creates a Parquet file catalog entry with IEnumerable container. /// /// Row schema type (must be flat and binary-serializable) - /// The enumerable catalog entries factory (from ) + /// The enumerable catalog entries factory (from ) /// Unique catalog label for DAG resolution /// Path to Parquet file /// Catalog entry with file + Parquet + IEnumerable composition @@ -31,8 +31,8 @@ public static class ParquetCatalogEntryExtensions /// Performance: Optimized for large datasets with columnar storage. /// /// - public static CatalogEntry> Parquet( - this EnumerableCatalogEntries _, + public static Item> Parquet( + this EnumerableItemFactory _, string label, string filePath ) @@ -43,6 +43,6 @@ string filePath var container = new EnumerableContainerAdapter(); var storage = new ComposedStorageAdapter, TRow>(medium, format, container); - return new CatalogEntry>(label, storage); + return new Item>(label, storage); } } diff --git a/src/extensions/Flowthru.Extensions.Python.SourceGenerators/PythonNodeFactoryGenerator.cs b/src/extensions/Flowthru.Extensions.Python.SourceGenerators/PythonNodeFactoryGenerator.cs index 8421a2da..35109a5d 100644 --- a/src/extensions/Flowthru.Extensions.Python.SourceGenerators/PythonNodeFactoryGenerator.cs +++ b/src/extensions/Flowthru.Extensions.Python.SourceGenerators/PythonNodeFactoryGenerator.cs @@ -9,29 +9,34 @@ namespace Flowthru.Extensions.Python.SourceGenerators; /// -/// Generates strongly-typed factory methods from Python @node decorators. +/// Generates strongly-typed factory methods from Python @step decorators. /// /// /// -/// Discovers Python files with @node decorators at build time and generates +/// Discovers Python files with @step decorators at build time and generates /// factory methods that return Func<(inputs), (outputs)> delegates for use -/// with the standard AddNode pipeline builder interface. +/// with the standard AddStep Flow builder interface. /// /// -/// This moves Python node registration from runtime (stringly-typed module/function names) +/// This moves Python step registration from runtime (stringly-typed module/function names) /// to build-time (strongly-typed factory methods with tuple signatures). /// /// [Generator] -public class PythonNodeFactoryGenerator : IIncrementalGenerator +public class PythonStepFactoryGenerator : IIncrementalGenerator { + /// + /// Initializes the source generator by registering a Flow that discovers Python steps + /// and generates factory methods for them. + /// + /// The incremental generator initialization context public void Initialize(IncrementalGeneratorInitializationContext context) { // For debugging: always generate a diagnostic file to confirm generator is running context.RegisterPostInitializationOutput(ctx => { ctx.AddSource( - "_PythonNodeFactoryGenerator.Diagnostic.g.cs", + "_PythonStepFactoryGenerator.Diagnostic.g.cs", SourceText.From("// Generator is running", Encoding.UTF8) ); }); @@ -39,8 +44,8 @@ public void Initialize(IncrementalGeneratorInitializationContext context) // Find all Python files marked as AdditionalFiles var pythonFiles = context.AdditionalTextsProvider.Where(file => file.Path.EndsWith(".py")); - // Parse each Python file to extract @node decorators - var nodeDeclarations = pythonFiles + // Parse each Python file to extract @step decorators + var stepDeclarations = pythonFiles .Select( (file, ct) => { @@ -48,43 +53,43 @@ public void Initialize(IncrementalGeneratorInitializationContext context) if (content == null) return null; - return ParsePythonNode(file.Path, content); + return ParsePythonStep(file.Path, content); } ) - .Where(node => node != null) + .Where(step => step != null) .Collect(); - // Generate factory class from all discovered nodes + // Generate factory class from all discovered steps context.RegisterSourceOutput( - nodeDeclarations, - (ctx, nodes) => + stepDeclarations, + (ctx, steps) => { - if (nodes.Length == 0) + if (steps.Length == 0) { - // Add diagnostic output showing we got no nodes + // Add diagnostic output showing we got no steps ctx.AddSource( - "_PythonNodeFactoryGenerator.NoNodes.g.cs", - SourceText.From("// No Python nodes discovered", Encoding.UTF8) + "_PythonStepFactoryGenerator.NoSteps.g.cs", + SourceText.From("// No Python steps discovered", Encoding.UTF8) ); return; } - var validNodes = nodes.Where(n => n != null).Select(n => n!).ToList(); - if (validNodes.Count == 0) + var validSteps = steps.Where(n => n != null).Select(n => n!).ToList(); + if (validSteps.Count == 0) return; - var source = GeneratePythonNodeFactories(validNodes); - ctx.AddSource("PythonNodes.g.cs", SourceText.From(source, Encoding.UTF8)); + var source = GeneratePythonStepFactories(validSteps); + ctx.AddSource("PythonSteps.g.cs", SourceText.From(source, Encoding.UTF8)); } ); } - private static PythonNodeInfo? ParsePythonNode(string filePath, string content) + private static PythonStepInfo? ParsePythonStep(string filePath, string content) { - // Regex to match @node decorator with inputs/outputs - // Handles: @node(inputs=["Schema1", "Schema2"], outputs=["Schema3"]) + // Regex to match @step decorator with inputs/outputs + // Handles: @step(inputs=["Schema1", "Schema2"], outputs=["Schema3"]) var decoratorPattern = - @"@node\s*\(\s*inputs\s*=\s*\[([^\]]*)\]\s*,\s*outputs\s*=\s*(\[[^\]]*\]|None)\s*\)"; + @"@step\s*\(\s*inputs\s*=\s*\[([^\]]*)\]\s*,\s*outputs\s*=\s*(\[[^\]]*\]|None)\s*\)"; var functionPattern = @"def\s+(\w+)\s*\("; var decoratorMatch = Regex.Match(content, decoratorPattern); @@ -107,10 +112,10 @@ public void Initialize(IncrementalGeneratorInitializationContext context) var outputs = outputsRaw == "None" ? new List() : ParseSchemaList(outputsRaw); // Derive module path from file path - // e.g., Pipelines/DataScience/Nodes/split_data.py → Pipelines.DataScience.Nodes.split_data + // e.g., Flows/DataScience/Steps/split_data.py → Flows.DataScience.Steps.split_data var modulePath = DeriveModulePath(filePath); - return new PythonNodeInfo( + return new PythonStepInfo( functionName: functionName, modulePath: modulePath, inputs: inputs, @@ -143,14 +148,14 @@ private static List ParseSchemaList(string raw) private static string DeriveModulePath(string filePath) { // Extract the relative path from the project and convert to Python module notation - // Look for common markers like "Pipelines" or "Nodes" + // Look for common markers like "Flows" or "Steps" var parts = filePath.Replace('\\', '/').Split('/'); var relevantParts = new List(); var startCapturing = false; foreach (var part in parts) { - if (part == "Pipelines" || startCapturing) + if (part == "Flows" || startCapturing) { startCapturing = true; if (part.EndsWith(".py")) @@ -167,7 +172,7 @@ private static string DeriveModulePath(string filePath) return string.Join(".", relevantParts); } - private static string GeneratePythonNodeFactories(List nodes) + private static string GeneratePythonStepFactories(List steps) { var sb = new StringBuilder(); @@ -177,19 +182,19 @@ private static string GeneratePythonNodeFactories(List nodes) sb.AppendLine("using System;"); sb.AppendLine("using System.Collections.Generic;"); sb.AppendLine("using Flowthru.Extensions.Python.Execution;"); - sb.AppendLine("using Flowthru.Extensions.Python.Nodes;"); + sb.AppendLine("using Flowthru.Extensions.Python.Steps;"); sb.AppendLine(); sb.AppendLine("namespace Flowthru.Extensions.Python.Generated;"); sb.AppendLine(); sb.AppendLine("/// "); - sb.AppendLine("/// Strongly-typed factory methods for Python nodes discovered at build time."); + sb.AppendLine("/// Strongly-typed factory methods for Python steps discovered at build time."); sb.AppendLine("/// "); - sb.AppendLine("public static class PythonNodes"); + sb.AppendLine("public static class PythonSteps"); sb.AppendLine("{"); - foreach (var node in nodes) + foreach (var step in steps) { - GenerateFactoryMethod(sb, node); + GenerateFactoryMethod(sb, step); sb.AppendLine(); } @@ -198,11 +203,11 @@ private static string GeneratePythonNodeFactories(List nodes) return sb.ToString(); } - private static void GenerateFactoryMethod(StringBuilder sb, PythonNodeInfo node) + private static void GenerateFactoryMethod(StringBuilder sb, PythonStepInfo step) { // Generate type parameters - var inputTypes = node.Inputs.Select(MapSchemaToType).ToList(); - var outputTypes = node.Outputs.Select(MapSchemaToType).ToList(); + var inputTypes = step.Inputs.Select(MapSchemaToType).ToList(); + var outputTypes = step.Outputs.Select(MapSchemaToType).ToList(); // Determine input/output tuple structures var inputTupleType = inputTypes.Count switch @@ -221,27 +226,27 @@ private static void GenerateFactoryMethod(StringBuilder sb, PythonNodeInfo node) // Generate XML documentation sb.AppendLine($" /// "); - sb.AppendLine($" /// Creates a Python node for {node.FunctionName}."); + sb.AppendLine($" /// Creates a Python step for {step.FunctionName}."); sb.AppendLine($" /// "); sb.AppendLine($" /// Python executor instance."); sb.AppendLine($" /// "); sb.AppendLine( - $" /// A function that invokes the Python node with the specified inputs and outputs." + $" /// A function that invokes the Python step with the specified inputs and outputs." ); sb.AppendLine($" /// "); // Generate method signature - var methodName = ToPascalCase(node.FunctionName); + var methodName = ToPascalCase(step.FunctionName); sb.AppendLine($" public static Func<{inputTupleType}, {outputTupleType}> {methodName}("); sb.AppendLine($" IPythonExecutor executor"); sb.AppendLine($" )"); sb.AppendLine(" {"); // Create wrapper - sb.AppendLine($" var wrapper = new PythonNodeWrapper<{inputTupleType}, {outputTupleType}>("); + sb.AppendLine($" var wrapper = new PythonStepWrapper<{inputTupleType}, {outputTupleType}>("); sb.AppendLine($" executor,"); - sb.AppendLine($" module: \"{node.ModulePath}\","); - sb.AppendLine($" function: \"{node.FunctionName}\""); + sb.AppendLine($" module: \"{step.ModulePath}\","); + sb.AppendLine($" function: \"{step.FunctionName}\""); sb.AppendLine($" );"); sb.AppendLine(); sb.AppendLine($" return wrapper.GetTransform();"); @@ -272,16 +277,16 @@ private static string ToPascalCase(string snakeCase) } /// -/// Information about a Python node parsed from source. +/// Information about a Python step parsed from source. /// -internal class PythonNodeInfo +internal class PythonStepInfo { public string FunctionName { get; } public string ModulePath { get; } public List Inputs { get; } public List Outputs { get; } - public PythonNodeInfo( + public PythonStepInfo( string functionName, string modulePath, List inputs, diff --git a/src/extensions/Flowthru.Extensions.Python.SourceGenerators/PythonNodeGenerator.cs b/src/extensions/Flowthru.Extensions.Python.SourceGenerators/PythonNodeGenerator.cs index 121da7b4..f1fe26ac 100644 --- a/src/extensions/Flowthru.Extensions.Python.SourceGenerators/PythonNodeGenerator.cs +++ b/src/extensions/Flowthru.Extensions.Python.SourceGenerators/PythonNodeGenerator.cs @@ -7,20 +7,20 @@ namespace Flowthru.Extensions.Python.SourceGenerators; /// -/// Generates AddPythonNode overloads for PipelineBuilder supporting up to 8 inputs and 8 outputs. +/// Generates AddPythonStep overloads for FlowBuilder supporting up to 8 inputs and 8 outputs. /// /// /// /// Phase 5 implementation: -/// Mirrors PipelineBuilderGenerator's combinatorial approach but for Python nodes. -/// Each overload accepts executor and runtime parameters and delegates to PythonNodeWrapper. +/// Mirrors FlowBuilderGenerator's combinatorial approach but for Python steps. +/// Each overload accepts executor and runtime parameters and delegates to PythonStepWrapper. /// /// /// Sync-only in Phase 5 — async support deferred to Phase 6. /// /// [Generator] -public class PythonNodeGenerator : IIncrementalGenerator +public class PythonStepGenerator : IIncrementalGenerator { private const int MaxInputs = 8; private const int MaxOutputs = 8; @@ -28,11 +28,11 @@ public class PythonNodeGenerator : IIncrementalGenerator public void Initialize(IncrementalGeneratorInitializationContext context) { // Run for any project that references Flowthru.Extensions.Python - // (unlike PipelineBuilderGenerator which only runs for the core Flowthru assembly) - context.RegisterPostInitializationOutput(ctx => GenerateAddPythonNode(ctx)); + // (unlike FlowBuilderGenerator which only runs for the core Flowthru assembly) + context.RegisterPostInitializationOutput(ctx => GenerateAddPythonStep(ctx)); } - private static void GenerateAddPythonNode(IncrementalGeneratorPostInitializationContext context) + private static void GenerateAddPythonStep(IncrementalGeneratorPostInitializationContext context) { var sb = new StringBuilder(); @@ -44,13 +44,13 @@ private static void GenerateAddPythonNode(IncrementalGeneratorPostInitialization using Flowthru.Data; using Flowthru.Extensions.Python.Execution; - using Flowthru.Extensions.Python.Nodes; + using Flowthru.Extensions.Python.Steps; using Flowthru.Extensions.Python.Runtime; - using Flowthru.Pipelines; + using Flowthru.Flows; - namespace Flowthru.Extensions.Python.Nodes; + namespace Flowthru.Extensions.Python.Steps; - public static partial class PythonNodeFactory + public static partial class PythonStepFactory { """ ); @@ -67,7 +67,7 @@ public static partial class PythonNodeFactory } // Generate sync version only (Phase 5) - GenerateAddPythonNode(sb, inputs, outputs); + GenerateAddPythonStep(sb, inputs, outputs); sb.AppendLine(); } } @@ -75,12 +75,12 @@ public static partial class PythonNodeFactory sb.AppendLine("}"); context.AddSource( - "PythonNodeFactory.Generated.cs", + "PythonStepFactory.Generated.cs", SourceText.From(sb.ToString(), Encoding.UTF8) ); } - private static void GenerateAddPythonNode(StringBuilder sb, int inputCount, int outputCount) + private static void GenerateAddPythonStep(StringBuilder sb, int inputCount, int outputCount) { // Generate XML documentation GenerateXmlDoc(sb, inputCount, outputCount); @@ -96,8 +96,8 @@ private static void GenerateAddPythonNode(StringBuilder sb, int inputCount, int sb.AppendLine( $$""" - public static PipelineBuilder AddPythonNode<{{typeParams}}>( - this PipelineBuilder builder, + public static FlowBuilder AddPythonStep<{{typeParams}}>( + this FlowBuilder builder, string label, string module, string function, @@ -116,7 +116,7 @@ private static void GenerateAddPythonNode(StringBuilder sb, int inputCount, int // Generate output deconstruction if needed GenerateOutputDeconstruction(sb, outputCount); - // Generate inputs and outputs lists for AddNode + // Generate inputs and outputs lists for AddStep var inputsList = GenerateInputsList(inputCount); var outputsList = GenerateOutputsList(outputCount); @@ -154,11 +154,11 @@ private static void GenerateAddPythonNode(StringBuilder sb, int inputCount, int // Create wrapper with tuple types sb.AppendLine( $$""" - var wrapper = new PythonNodeWrapper<{{inputTupleType}}, {{outputTupleType}}>(executor, module, function); - executor.ValidateNode(module, function); + var wrapper = new PythonStepWrapper<{{inputTupleType}}, {{outputTupleType}}>(executor, module, function); + executor.ValidateStep(module, function); - // Delegate to existing AddNode infrastructure - return builder.AddNode( + // Delegate to existing AddStep infrastructure + return builder.AddStep( label: label, transform: wrapper.GetTransform(), input: input, @@ -175,7 +175,7 @@ private static void GenerateXmlDoc(StringBuilder sb, int inputCount, int outputC sb.AppendLine( $""" /// - /// Adds a Python node with {inputCount} input{( + /// Adds a Python step with {inputCount} input{( inputCount == 1 ? "" : "s" )} and {outputCount} output{(outputCount == 1 ? "" : "s")}. /// @@ -195,14 +195,14 @@ private static void GenerateXmlDoc(StringBuilder sb, int inputCount, int outputC // Parameter documentation sb.AppendLine( """ - /// Pipeline builder instance - /// Unique identifier for this node - /// Dotted Python module name (e.g., "Pipelines.DataScience.train_model") + /// Flow builder instance + /// Unique identifier for this step + /// Dotted Python module name (e.g., "Flows.DataScience.train_model") /// Python function name within the module - /// Catalog entry or tuple of catalog entries providing input data - /// Catalog entry or tuple of catalog entries to store output data + /// Catalog item or tuple of catalog items providing input data + /// Catalog item or tuple of catalog items to store output data /// Python executor for invoking the function - /// Optional node description + /// Optional step description /// This builder for method chaining """ ); @@ -228,13 +228,10 @@ private static string GenerateInputParameter(int inputCount) { if (inputCount == 1) { - return "ICatalogEntry"; + return "IItem"; } - var types = string.Join( - ", ", - Enumerable.Range(1, inputCount).Select(i => $"ICatalogEntry") - ); + var types = string.Join(", ", Enumerable.Range(1, inputCount).Select(i => $"IItem")); return $"({types})"; } @@ -242,13 +239,10 @@ private static string GenerateOutputParameter(int outputCount) { if (outputCount == 1) { - return "ICatalogEntry"; + return "IItem"; } - var types = string.Join( - ", ", - Enumerable.Range(1, outputCount).Select(i => $"ICatalogEntry") - ); + var types = string.Join(", ", Enumerable.Range(1, outputCount).Select(i => $"IItem")); return $"({types})"; } diff --git a/src/extensions/Flowthru.Extensions.Python/Execution/IPythonExecutor.cs b/src/extensions/Flowthru.Extensions.Python/Execution/IPythonExecutor.cs index 83724f59..2ca34e2f 100644 --- a/src/extensions/Flowthru.Extensions.Python/Execution/IPythonExecutor.cs +++ b/src/extensions/Flowthru.Extensions.Python/Execution/IPythonExecutor.cs @@ -5,7 +5,7 @@ namespace Flowthru.Extensions.Python.Execution; /// /// /// -/// Decouples the execution strategy from Python node wiring. +/// Decouples the execution strategy from Python step wiring. /// Two implementations ship out of the box: /// /// — in-process via Python.NET (opt-in) @@ -41,7 +41,7 @@ public interface IPythonExecutor /// C# output type. Same range as . /// /// - /// Dotted module name (e.g., "Pipelines.DataScience.train_model"). + /// Dotted module name (e.g., "Flows.DataScience.train_model"). /// Must be resolvable via the executor's configured sys.path. /// /// Python function name within the module. @@ -54,13 +54,13 @@ public interface IPythonExecutor TOutput Invoke(string moduleName, string functionName, TInput input); /// - /// Validates that a Python node exists and satisfies Flowthru's @node contract. + /// Validates that a Python step exists and satisfies Flowthru's @step contract. /// /// Dotted Python module name. /// Python function name within the module. /// /// Thrown if the module is not importable, the function is missing, or the - /// @node decorator is absent. + /// @step decorator is absent. /// - void ValidateNode(string moduleName, string functionName); + void ValidateStep(string moduleName, string functionName); } diff --git a/src/extensions/Flowthru.Extensions.Python/Execution/PythonNetExecutor.cs b/src/extensions/Flowthru.Extensions.Python/Execution/PythonNetExecutor.cs index c1e836d0..bf632ed8 100644 --- a/src/extensions/Flowthru.Extensions.Python/Execution/PythonNetExecutor.cs +++ b/src/extensions/Flowthru.Extensions.Python/Execution/PythonNetExecutor.cs @@ -97,10 +97,10 @@ public TOutput Invoke(string moduleName, string functionName, T } /// - public void ValidateNode(string moduleName, string functionName) + public void ValidateStep(string moduleName, string functionName) { _runtime.Initialize(); - PythonNodeRegistrationValidator.ValidateRegistration(_runtime, moduleName, functionName); + PythonStepRegistrationValidator.ValidateRegistration(_runtime, moduleName, functionName); } // ── Scalar path ────────────────────────────────────────────────────────────────────── diff --git a/src/extensions/Flowthru.Extensions.Python/Execution/SubprocessPythonExecutor.cs b/src/extensions/Flowthru.Extensions.Python/Execution/SubprocessPythonExecutor.cs index 3c439564..bc855a67 100644 --- a/src/extensions/Flowthru.Extensions.Python/Execution/SubprocessPythonExecutor.cs +++ b/src/extensions/Flowthru.Extensions.Python/Execution/SubprocessPythonExecutor.cs @@ -94,7 +94,7 @@ public TOutput Invoke(string moduleName, string functionName, T } /// - public void ValidateNode(string moduleName, string functionName) + public void ValidateStep(string moduleName, string functionName) { EnsureStarted(); @@ -111,7 +111,7 @@ public void ValidateNode(string moduleName, string functionName) { var msg = resp["message"]?.GetValue() ?? "Unknown error"; throw new InvalidOperationException( - $"Python node validation failed for {moduleName}.{functionName}: {msg}" + $"Python step validation failed for {moduleName}.{functionName}: {msg}" ); } } diff --git a/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.csproj b/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.csproj index e2fecd1e..8fa9e6b6 100644 --- a/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.csproj +++ b/src/extensions/Flowthru.Extensions.Python/Flowthru.Extensions.Python.csproj @@ -55,15 +55,15 @@ Pack="true" PackagePath="build\" /> - + - diff --git a/src/extensions/Flowthru.Extensions.Python/README.md b/src/extensions/Flowthru.Extensions.Python/README.md index adc5e01a..0d1bfa96 100644 --- a/src/extensions/Flowthru.Extensions.Python/README.md +++ b/src/extensions/Flowthru.Extensions.Python/README.md @@ -19,13 +19,13 @@ dotnet add package Flowthru.Extensions.Python services.AddFlowthru(flowthru => { flowthru - .UseCatalog() + .RegisterCatalog() .UsePython(python => { python.VenvPath = ".venv"; python.ModuleSearchPaths.Add("Pipelines"); }) - .UsePipelines(catalog => new Dictionary + .RegisterPipelines(catalog => new Dictionary { ["my_pipeline"] = MyPipeline.Create(catalog) }); diff --git a/src/extensions/Flowthru.Extensions.Python/Runtime/PythonRuntimeOptions.cs b/src/extensions/Flowthru.Extensions.Python/Runtime/PythonRuntimeOptions.cs index 685160a7..fcb78021 100644 --- a/src/extensions/Flowthru.Extensions.Python/Runtime/PythonRuntimeOptions.cs +++ b/src/extensions/Flowthru.Extensions.Python/Runtime/PythonRuntimeOptions.cs @@ -1,19 +1,21 @@ +using System.Collections.Concurrent; + namespace Flowthru.Extensions.Python.Runtime; /// -/// Controls how Python node execution is isolated between FlowthruService instances. +/// Controls how Python step execution is isolated between FlowthruService instances. /// public enum PythonExecutionMode { /// - /// Executes Python nodes in the same process via Python.NET. + /// Executes Python steps in the same process via Python.NET. /// Fast (no IPC overhead), but all services share one Python interpreter, - /// sys.modules, and GIL. Use when co-hosted pipelines are known to be compatible. + /// sys.modules, and GIL. Use when co-hosted flows are known to be compatible. /// InProcess, /// - /// Executes Python nodes in an isolated child process per FlowthruService. + /// Executes Python steps in an isolated child process per FlowthruService. /// Each service gets its own Python interpreter, venv, sys.path, and module cache. /// Default for multi-service scenarios. /// @@ -104,7 +106,7 @@ public sealed class PythonRuntimeOptions public string UvPath { get; set; } = "uv"; /// - /// Controls whether Python nodes run in the same process or an isolated child process. + /// Controls whether Python steps run in the same process or an isolated child process. /// Defaults to for per-service isolation. /// Set to to opt in to shared-interpreter mode. /// @@ -122,8 +124,8 @@ public sealed class PythonRuntimeOptions /// /// /// - /// Python nodes at Pipelines/DataScience/Nodes/train_model.py are referenced as - /// "Pipelines.DataScience.Nodes.train_model" when the project root is in sys.path. + /// Python steps at Flows/DataScience/Steps/train_model.py are referenced as + /// "Flows.DataScience.Steps.train_model" when the project root is in sys.path. /// /// public List ModuleSearchPaths { get; set; } = new(); @@ -267,12 +269,20 @@ public string GetResolvedPythonDll() /// public string GetResolvedPythonExe() { - // 1. Explicit VenvPath + // 1. Explicit VenvPath — try as existing .venv, then as project dir for uv sync if (!string.IsNullOrWhiteSpace(VenvPath)) { var exe = FindPythonExeInVenv(VenvPath); if (exe != null) return exe; + + var uvVenvPath = EnsureVenvViaUv(VenvPath); + if (uvVenvPath != null) + { + exe = FindPythonExeInVenv(uvVenvPath); + if (exe != null) + return exe; + } } // 2. Auto-init .venv via uv sync in output directory (same trigger as GetResolvedPythonDll) @@ -424,6 +434,10 @@ public List GetResolvedModuleSearchPaths() /// Returns the .venv/ path if successful, or null if files are missing or sync fails. /// /// + // One semaphore per directory prevents concurrent uv sync calls from racing on the same .venv. + private static readonly ConcurrentDictionary _uvSyncLocks = + new(StringComparer.OrdinalIgnoreCase); + private string? EnsureVenvViaUv(string directory) { var pyprojectPath = Path.Combine(directory, "pyproject.toml"); @@ -437,59 +451,81 @@ public List GetResolvedModuleSearchPaths() return null; // Not a uv-managed project } - // If .venv exists but lacks pyvenv.cfg, it's corrupt — delete and recreate - if (Directory.Exists(venvPath)) + // Fast path: venv already exists and is valid — no lock needed. + if (File.Exists(pyvenvCfg)) { - try - { - Directory.Delete(venvPath, recursive: true); - } - catch - { - // Deletion failed (permissions, locked files, etc.) — let uv sync handle it - } + return venvPath; } - // Run uv sync to materialize .venv + // Slow path: venv is missing or corrupt. One caller creates it; others wait and reuse. + var semaphore = _uvSyncLocks.GetOrAdd(directory, _ => new SemaphoreSlim(1, 1)); + semaphore.Wait(); try { - var startInfo = new System.Diagnostics.ProcessStartInfo + // Re-check under the lock — another thread may have created it while we were waiting. + if (File.Exists(pyvenvCfg)) { - FileName = UvPath, - Arguments = "sync --frozen --python-preference only-managed", - WorkingDirectory = directory, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - }; - - using var process = System.Diagnostics.Process.Start(startInfo); - if (process == null) - { - return null; // Failed to start process + return venvPath; } - process.WaitForExit(); - - // Check if .venv was created successfully - if (process.ExitCode == 0 && File.Exists(pyvenvCfg)) + // If .venv exists but lacks pyvenv.cfg, it's corrupt — delete and recreate + if (Directory.Exists(venvPath)) { - return venvPath; + try + { + Directory.Delete(venvPath, recursive: true); + } + catch + { + // Deletion failed (permissions, locked files, etc.) — let uv sync handle it + } } - // Sync failed — log stderr if available for diagnostics - var stderr = process.StandardError.ReadToEnd(); - if (!string.IsNullOrWhiteSpace(stderr)) + // Run uv sync to materialize .venv + try + { + var startInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = UvPath, + Arguments = "sync --frozen --python-preference only-managed", + WorkingDirectory = directory, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + + using var process = System.Diagnostics.Process.Start(startInfo); + if (process == null) + { + return null; // Failed to start process + } + + process.WaitForExit(); + + // Check if .venv was created successfully + if (process.ExitCode == 0 && File.Exists(pyvenvCfg)) + { + return venvPath; + } + + // Sync failed — log stderr if available for diagnostics + var stderr = process.StandardError.ReadToEnd(); + if (!string.IsNullOrWhiteSpace(stderr)) + { + Console.Error.WriteLine($"uv sync failed: {stderr}"); + } + return null; + } + catch { - Console.Error.WriteLine($"uv sync failed: {stderr}"); + // uv not found, permission denied, etc. — fall through to other resolution methods + return null; } - return null; } - catch + finally { - // uv not found, permission denied, etc. — fall through to other resolution methods - return null; + semaphore.Release(); } } } diff --git a/src/extensions/Flowthru.Extensions.Python/Services/FlowthruServiceBuilderExtensions.cs b/src/extensions/Flowthru.Extensions.Python/Services/FlowthruServiceBuilderExtensions.cs index dd8ff5c2..4eabd207 100644 --- a/src/extensions/Flowthru.Extensions.Python/Services/FlowthruServiceBuilderExtensions.cs +++ b/src/extensions/Flowthru.Extensions.Python/Services/FlowthruServiceBuilderExtensions.cs @@ -31,7 +31,7 @@ public static class FlowthruServiceBuilderExtensions /// services.AddFlowthru(flowthru => /// { /// flowthru - /// .UseCatalog<MyCatalog>() + /// .RegisterCatalog<MyCatalog>() /// .UsePython(); // Auto-detects .venv/, project root, etc. /// }); /// @@ -64,12 +64,12 @@ public static FlowthruServiceBuilder UsePython(this FlowthruServiceBuilder build /// services.AddFlowthru(flowthru => /// { /// flowthru - /// .UseCatalog<MyCatalog>() + /// .RegisterCatalog<MyCatalog>() /// .UsePython(python => /// { /// python.PythonDll = "/usr/lib/x86_64-linux-gnu/libpython3.12.so"; - /// python.ModuleSearchPaths.Add("Pipelines"); - /// python.ModuleSearchPaths.Add("SharedNodes"); + /// python.ModuleSearchPaths.Add("Flows"); + /// python.ModuleSearchPaths.Add("SharedSteps"); /// }); /// }); /// @@ -80,7 +80,7 @@ public static FlowthruServiceBuilder UsePython(this FlowthruServiceBuilder build /// services.AddFlowthru(flowthru => /// { /// flowthru - /// .UseCatalog<MyCatalog>() + /// .RegisterCatalog<MyCatalog>() /// .UsePython(python => /// { /// // Reads PYTHONNET_PYDLL, FLOWTHRU_PYTHON_VENV, FLOWTHRU_PYTHON_PATH @@ -124,7 +124,7 @@ Action configure // Register pre-flight decorator + dry-run dtype validation (in-process only) builder .Services() - .AddSingleton(); + .AddSingleton(); } else { diff --git a/src/extensions/Flowthru.Extensions.Python/Nodes/PythonNodeFactory.cs b/src/extensions/Flowthru.Extensions.Python/Steps/PythonNodeFactory.cs similarity index 69% rename from src/extensions/Flowthru.Extensions.Python/Nodes/PythonNodeFactory.cs rename to src/extensions/Flowthru.Extensions.Python/Steps/PythonNodeFactory.cs index 47e170f9..cf741e81 100644 --- a/src/extensions/Flowthru.Extensions.Python/Nodes/PythonNodeFactory.cs +++ b/src/extensions/Flowthru.Extensions.Python/Steps/PythonNodeFactory.cs @@ -2,33 +2,33 @@ using Flowthru.Extensions.Python.Execution; using Flowthru.Extensions.Python.Runtime; using Flowthru.Extensions.Python.Validation; -using Flowthru.Pipelines; +using Flowthru.Flows; -namespace Flowthru.Extensions.Python.Nodes; +namespace Flowthru.Extensions.Python.Steps; /// -/// Extension methods for adding Python nodes to pipelines. +/// Extension methods for adding Python steps to flows. /// /// /// /// Phases 2-5 implementation: -/// Hand-written 1×1 AddPythonNode(Phase 2-4). +/// Hand-written 1×1 AddPythonStep(Phase 2-4). /// Source-generated N×M overloads for multi-I/O support (Phase 5). /// /// /// Usage: /// /// -/// public static Pipeline Create( +/// public static Flow Create( /// Catalog catalog, /// IPythonExecutor executor, /// PythonRuntime runtime) /// { -/// return PipelineBuilder.CreatePipeline(pipeline => +/// return FlowBuilder.CreateFlow(flow => /// { -/// pipeline.AddPythonNode( +/// flow.AddPythonStep( /// label: "Transform", -/// module: "my_nodes.transform", +/// module: "my_steps.transform", /// function: "process", /// input: catalog.RawData, /// output: catalog.ProcessedData, @@ -45,30 +45,30 @@ namespace Flowthru.Extensions.Python.Nodes; /// /// /// -public static partial class PythonNodeFactory +public static partial class PythonStepFactory { /// - /// Adds a Python node with single input and single output. + /// Adds a Python step with single input and single output. /// - /// Input type (must match catalog entry type). - /// Output type (must match catalog entry type). - /// Pipeline builder instance. - /// Unique identifier for this node. + /// Input type (must match catalog item type). + /// Output type (must match catalog item type). + /// Flow builder instance. + /// Unique identifier for this step. /// - /// Dotted Python module name (e.g., "Pipelines.DataScience.train_model"). + /// Dotted Python module name (e.g., "Flows.DataScience.train_model"). /// Must be resolvable via sys.path. /// /// Python function name within the module. - /// Catalog entry providing input data. - /// Catalog entry to store output data. + /// Catalog item providing input data. + /// Catalog item to store output data. /// Python executor for invoking the function. /// Python runtime for GIL management. - /// Optional node description. + /// Optional step description. /// This builder for method chaining. /// /// /// Compile-time type safety: - /// Generic type parameters are inferred from catalog entries. + /// Generic type parameters are inferred from catalog items. /// Mismatched types produce compiler errors. /// /// @@ -76,7 +76,7 @@ public static partial class PythonNodeFactory /// /// Module is importable (exists, no syntax errors) /// Function exists in module - /// @node decorator is present + /// @step decorator is present /// /// /// @@ -88,13 +88,13 @@ public static partial class PythonNodeFactory /// /// /// - public static PipelineBuilder AddPythonNode( - this PipelineBuilder builder, + public static FlowBuilder AddPythonStep( + this FlowBuilder builder, string label, string module, string function, - ICatalogEntry input, - ICatalogEntry output, + IItem input, + IItem output, IPythonExecutor executor, string description = "" ) @@ -135,14 +135,14 @@ public static PipelineBuilder AddPythonNode( } // Phase 4: Registration-time validation via executor - executor.ValidateNode(module, function); + executor.ValidateStep(module, function); - var wrapper = new PythonNodeWrapper(executor, module, function); + var wrapper = new PythonStepWrapper(executor, module, function); - // Delegate to the existing AddNode infrastructure + // Delegate to the existing AddStep infrastructure // This ensures DAG scheduling, dependency analysis, and all other - // pipeline mechanics work identically for Python and C# nodes - return builder.AddNode( + // Flow mechanics work identically for Python and C# steps + return builder.AddStep( label: label, transform: wrapper.GetTransform(), input: input, diff --git a/src/extensions/Flowthru.Extensions.Python/Nodes/PythonNodeWrapper.cs b/src/extensions/Flowthru.Extensions.Python/Steps/PythonNodeWrapper.cs similarity index 65% rename from src/extensions/Flowthru.Extensions.Python/Nodes/PythonNodeWrapper.cs rename to src/extensions/Flowthru.Extensions.Python/Steps/PythonNodeWrapper.cs index d98e24cb..0f6a68ea 100644 --- a/src/extensions/Flowthru.Extensions.Python/Nodes/PythonNodeWrapper.cs +++ b/src/extensions/Flowthru.Extensions.Python/Steps/PythonNodeWrapper.cs @@ -1,30 +1,37 @@ using Flowthru.Extensions.Python.Execution; -namespace Flowthru.Extensions.Python.Nodes; +namespace Flowthru.Extensions.Python.Steps; /// /// Thin wrapper that binds an to a specific module/function pair, -/// exposing it as a typed Func<TInput, TOutput> for use with the pipeline builder. +/// exposing it as a typed Func<TInput, TOutput> for use with the Flow builder. /// /// /// All marshalling (scalar, tabular, bytes, multi-I/O tuples) is delegated to the executor. /// -public sealed class PythonNodeWrapper +public sealed class PythonStepWrapper { private readonly IPythonExecutor _executor; private readonly string _moduleName; private readonly string _functionName; - public PythonNodeWrapper(IPythonExecutor executor, string moduleName, string functionName) + /// + /// Initializes the wrapper with the executor and step information. + /// + /// + /// + /// + /// + public PythonStepWrapper(IPythonExecutor executor, string moduleName, string functionName) { _executor = executor ?? throw new ArgumentNullException(nameof(executor)); _moduleName = moduleName ?? throw new ArgumentNullException(nameof(moduleName)); _functionName = functionName ?? throw new ArgumentNullException(nameof(functionName)); - _executor.ValidateNode(_moduleName, _functionName); + _executor.ValidateStep(_moduleName, _functionName); } /// - /// Gets the transformation function that invokes the Python node. + /// Gets the transformation function that invokes the Python step. /// /// /// A function that takes and returns . diff --git a/src/extensions/Flowthru.Extensions.Python/Validation/PythonNodeRegistrationValidator.cs b/src/extensions/Flowthru.Extensions.Python/Validation/PythonNodeRegistrationValidator.cs index 25243643..d8598c74 100644 --- a/src/extensions/Flowthru.Extensions.Python/Validation/PythonNodeRegistrationValidator.cs +++ b/src/extensions/Flowthru.Extensions.Python/Validation/PythonNodeRegistrationValidator.cs @@ -5,20 +5,20 @@ namespace Flowthru.Extensions.Python.Validation; /// -/// Performs basic validation checks during Python node registration. +/// Performs basic validation checks during Python step registration. /// /// /// /// Phase 4 registration-time validation: -/// Validates that the Python module, function, and @node decorator exist before -/// creating the node wrapper. This provides fast feedback during pipeline definition. +/// Validates that the Python module, function, and @step decorator exist before +/// creating the step wrapper. This provides fast feedback during Flow definition. /// /// /// Checks performed: /// /// Module is importable (exists, no syntax errors) /// Function exists in module -/// Function has @node decorator with metadata +/// Function has @step decorator with metadata /// /// /// @@ -30,14 +30,14 @@ namespace Flowthru.Extensions.Python.Validation; /// /// /// -internal static class PythonNodeRegistrationValidator +internal static class PythonStepRegistrationValidator { /// - /// Validates that a Python node can be registered. + /// Validates that a Python step can be registered. /// /// Python executor for module/function inspection /// Python runtime for GIL management - /// Python module name (e.g., "pipelines.nodes.transform") + /// Python module name (e.g., "flows.steps.transform") /// Python function name (e.g., "encode_species") /// /// Thrown if module, function, or decorator is missing or invalid @@ -62,7 +62,7 @@ string functionName catch (PythonException ex) { throw new InvalidOperationException( - $"Python node registration failed: Module '{moduleName}' not found in sys.path\n\n" + $"Python step registration failed: Module '{moduleName}' not found in sys.path\n\n" + $"Error: {ex.Message}\n\n" + "Check:\n" + " - Module name spelling\n" @@ -90,7 +90,7 @@ string functionName } throw new InvalidOperationException( - $"Python node registration failed: Function '{functionName}' not found in module '{moduleName}'\n\n" + $"Python step registration failed: Function '{functionName}' not found in module '{moduleName}'\n\n" + $"Available functions: {string.Join(", ", availableFunctions)}\n\n" + "Check:\n" + " - Function name spelling\n" @@ -101,20 +101,20 @@ string functionName PyObject function = module.GetAttr(functionName); - // Check 3: @node decorator is present + // Check 3: @step decorator is present if (!function.HasAttr("__flowthru_inputs__") || !function.HasAttr("__flowthru_outputs__")) { throw new InvalidOperationException( - $"Python node registration failed: Function '{functionName}' in module '{moduleName}' " - + "is missing required @node decorator.\n\n" + $"Python step registration failed: Function '{functionName}' in module '{moduleName}' " + + "is missing required @step decorator.\n\n" + "Add decorator to declare schema contract:\n\n" - + " from flowthru import node\n" + + " from flowthru import step\n" + " from flowthru_schemas import InputSchema, OutputSchema\n" + " \n" - + $" @node(inputs=[InputSchema], outputs=[OutputSchema])\n" + + $" @step(inputs=[InputSchema], outputs=[OutputSchema])\n" + $" def {functionName}(...):\n" + " ...\n\n" - + "The @node decorator is required for all Python nodes in Flowthru pipelines." + + "The @step decorator is required for all Python steps in Flows." ); } diff --git a/src/extensions/Flowthru.Extensions.Python/Validation/PythonNodeValidator.cs b/src/extensions/Flowthru.Extensions.Python/Validation/PythonNodeValidator.cs index db1fc770..fbc0d973 100644 --- a/src/extensions/Flowthru.Extensions.Python/Validation/PythonNodeValidator.cs +++ b/src/extensions/Flowthru.Extensions.Python/Validation/PythonNodeValidator.cs @@ -3,86 +3,83 @@ using Flowthru.Data.Validation; using Flowthru.Extensions.Python.Execution; using Flowthru.Extensions.Python.Marshalling; -using Flowthru.Extensions.Python.Nodes; using Flowthru.Extensions.Python.Runtime; -using Flowthru.Pipelines; -using Flowthru.Pipelines.Validation; +using Flowthru.Extensions.Python.Steps; +using Flowthru.Flows; +using Flowthru.Flows.Validation; using Python.Runtime; namespace Flowthru.Extensions.Python.Validation; /// -/// Validation hook for Python nodes. +/// Validation hook for Python steps. /// /// /// /// Phase 4 pre-flight validation: -/// Validates Python nodes during pipeline pre-flight to catch schema mismatches, +/// Validates Python steps during Flow pre-flight to catch schema mismatches, /// incorrect function signatures, and structural errors before execution. /// /// /// Checks performed: /// -/// @node decorator schemas match C# generic type parameters +/// @step decorator schemas match C# generic type parameters /// Function signature arity is correct for input count /// Dry-run with 0-row data validates output structure /// /// /// /// Integration: -/// Register this hook via Pipeline.ValidationHooks during pipeline setup. -/// The hook is automatically invoked during Pipeline.ValidateExternalInputsAsync(). +/// Register this hook via Flow.ValidationHooks during Flow setup. +/// The hook is automatically invoked during Flow.ValidateExternalInputsAsync(). /// /// -public class PythonNodeValidator : IPipelineValidationHook +public class PythonStepValidator : IFlowValidationHook { private readonly IPythonExecutor _executor; private readonly PythonRuntime _runtime; /// - /// Initializes a new instance of . + /// Initializes a new instance of . /// /// Python executor for function inspection /// Python runtime for GIL management - public PythonNodeValidator(IPythonExecutor executor, PythonRuntime runtime) + public PythonStepValidator(IPythonExecutor executor, PythonRuntime runtime) { _executor = executor ?? throw new ArgumentNullException(nameof(executor)); _runtime = runtime ?? throw new ArgumentNullException(nameof(runtime)); } /// - public async Task ValidateAsync( - Pipeline pipeline, - CancellationToken cancellationToken - ) + public async Task ValidateAsync(Flow flow, CancellationToken cancellationToken) { var errors = new List(); - foreach (var node in pipeline.Nodes) + foreach (var step in flow.Steps) { - // Check if this is a Python node by examining the transform function - if (!IsPythonNode(node, out var pythonNodeInfo)) + // Check if this is a Python step by examining the transform function + if (!IsPythonStep(step, out var pythonStepInfo)) { - continue; // Skip non-Python nodes + continue; // Skip non-Python steps } // Validate decorator and schema compatibility - var nodeResult = await ValidatePythonNode(pythonNodeInfo!, cancellationToken); - errors.AddRange(nodeResult.Errors); + var stepResult = await ValidatePythonStep(pythonStepInfo!, cancellationToken); + errors.AddRange(stepResult.Errors); } return new ValidationResult(errors); } /// - /// Checks if a pipeline node is a Python node. + /// Checks if a Flow step is a Python step. /// - private bool IsPythonNode(Flowthru.Pipelines.PipelineNode node, out PythonNodeInfo? info) + private bool IsPythonStep(FlowStep step, out PythonStepInfo? info) { info = null; - // Python nodes use PythonNodeWrapper as the transform target - var transform = node.TransformFunction; + // Python steps use PythonStepWrapper as the transform target + var transform = step.TransformFunction; if (transform.Target == null) { return false; @@ -90,14 +87,14 @@ private bool IsPythonNode(Flowthru.Pipelines.PipelineNode node, out PythonNodeIn var targetType = transform.Target.GetType(); - // Check if target is PythonNodeWrapper<,> + // Check if target is PythonStepWrapper<,> if (!targetType.IsGenericType) { return false; } var genericTypeDef = targetType.GetGenericTypeDefinition(); - if (genericTypeDef != typeof(PythonNodeWrapper<,>)) + if (genericTypeDef != typeof(PythonStepWrapper<,>)) { return false; } @@ -132,9 +129,9 @@ private bool IsPythonNode(Flowthru.Pipelines.PipelineNode node, out PythonNodeIn return false; } - info = new PythonNodeInfo + info = new PythonStepInfo { - Label = node.Label, + Label = step.Label, ModuleName = moduleName, FunctionName = functionName, InputType = genericArgs[0], @@ -145,10 +142,10 @@ private bool IsPythonNode(Flowthru.Pipelines.PipelineNode node, out PythonNodeIn } /// - /// Validates a single Python node. + /// Validates a single Python step. /// - private async Task ValidatePythonNode( - PythonNodeInfo nodeInfo, + private async Task ValidatePythonStep( + PythonStepInfo stepInfo, CancellationToken cancellationToken ) { @@ -164,15 +161,15 @@ CancellationToken cancellationToken { using (Py.GIL()) { - module = Py.Import(nodeInfo.ModuleName); - function = module.GetAttr(nodeInfo.FunctionName); + module = Py.Import(stepInfo.ModuleName); + function = module.GetAttr(stepInfo.FunctionName); } } catch (PythonException ex) { errors.Add( new ValidationError( - nodeInfo.Label, + stepInfo.Label, ValidationErrorType.InvalidFormat, $"Failed to import Python function: {ex.Message}", ex.ToString() @@ -182,7 +179,7 @@ CancellationToken cancellationToken } // Check decorator metadata - var decoratorResult = ValidateDecorator(nodeInfo, function); + var decoratorResult = ValidateDecorator(stepInfo, function); errors.AddRange(decoratorResult.Errors); if (!decoratorResult.IsValid) @@ -191,9 +188,9 @@ CancellationToken cancellationToken } // Dry-run dtype validation for tabular outputs - if (IsEnumerableSchema(nodeInfo.OutputType)) + if (IsEnumerableSchema(stepInfo.OutputType)) { - var dryRunResult = ValidateDryRunDtypes(nodeInfo, function); + var dryRunResult = ValidateDryRunDtypes(stepInfo, function); errors.AddRange(dryRunResult.Errors); } } @@ -210,7 +207,7 @@ CancellationToken cancellationToken /// passes it through the Python function, and checks if the output dtypes /// can be safely coerced to the C# schema types. /// - private ValidationResult ValidateDryRunDtypes(PythonNodeInfo nodeInfo, PyObject function) + private ValidationResult ValidateDryRunDtypes(PythonStepInfo stepInfo, PyObject function) { var errors = new List(); @@ -218,7 +215,7 @@ private ValidationResult ValidateDryRunDtypes(PythonNodeInfo nodeInfo, PyObject { using (Py.GIL()) { - var schemaType = nodeInfo.OutputType.GetGenericArguments()[0]; + var schemaType = stepInfo.OutputType.GetGenericArguments()[0]; // Build expected dtype spec from C# schema var buildDtypeSpecMethod = typeof(ArrowSchemaMapper) @@ -248,14 +245,14 @@ private ValidationResult ValidateDryRunDtypes(PythonNodeInfo nodeInfo, PyObject ); // Note: A full dry-run would require creating 0-row input DataFrames for all inputs, - // invoking the function, and checking output dtypes. This is complex for multi-input nodes + // invoking the function, and checking output dtypes. This is complex for multi-input step // and may have side effects (imports, setup code). // // Instead, we rely on: - // 1. Registration-time validation (@node decorator exists) + // 1. Registration-time validation (@step decorator exists) // 2. Runtime automatic coercion in _flowthru_arrow.py (df_to_ipc with dtype_spec) // - // If a Python node returns incompatible dtypes, the runtime coercion will raise + // If a Python step returns incompatible dtypes, the runtime coercion will raise // detailed TypeError/OverflowError with fix guidance. // // Pre-flight dry-run validation would be added here as a Phase 5 enhancement. @@ -265,7 +262,7 @@ private ValidationResult ValidateDryRunDtypes(PythonNodeInfo nodeInfo, PyObject { errors.Add( new ValidationError( - nodeInfo.Label, + stepInfo.Label, ValidationErrorType.InspectionFailure, $"Dry-run dtype validation failed: {ex.Message}", "This is a framework error, not a user error. Report this issue." @@ -291,9 +288,9 @@ private static bool IsEnumerableSchema(Type type) } /// - /// Validates that the @node decorator metadata matches C# type parameters. + /// Validates that the @step decorator metadata matches C# type parameters. /// - private ValidationResult ValidateDecorator(PythonNodeInfo nodeInfo, PyObject function) + private ValidationResult ValidateDecorator(PythonStepInfo stepInfo, PyObject function) { var errors = new List(); @@ -302,9 +299,9 @@ private ValidationResult ValidateDecorator(PythonNodeInfo nodeInfo, PyObject fun { errors.Add( new ValidationError( - nodeInfo.Label, + stepInfo.Label, ValidationErrorType.SchemaMismatch, - $"Function '{nodeInfo.FunctionName}' is missing @node decorator metadata", + $"Function '{stepInfo.FunctionName}' is missing @step decorator metadata", "This should have been caught during registration-time validation" ) ); @@ -340,15 +337,15 @@ private ValidationResult ValidateDecorator(PythonNodeInfo nodeInfo, PyObject fun } // Extract C# schema names from type parameters (Phase 5: supports tuples) - var csharpInputSchemas = ExtractSchemaNames(nodeInfo.InputType); - var csharpOutputSchemas = ExtractSchemaNames(nodeInfo.OutputType); + var csharpInputSchemas = ExtractSchemaNames(stepInfo.InputType); + var csharpOutputSchemas = ExtractSchemaNames(stepInfo.OutputType); // Validate input schema count if (decoratorInputs.Count != csharpInputSchemas.Count) { errors.Add( new ValidationError( - nodeInfo.Label, + stepInfo.Label, ValidationErrorType.SchemaMismatch, $"Input schema count mismatch: C# expects {csharpInputSchemas.Count} input(s), decorator declares {decoratorInputs.Count}", $"C# inputs: [{string.Join(", ", csharpInputSchemas)}]\nDecorator inputs: [{string.Join(", ", decoratorInputs)}]" @@ -361,7 +358,7 @@ private ValidationResult ValidateDecorator(PythonNodeInfo nodeInfo, PyObject fun { errors.Add( new ValidationError( - nodeInfo.Label, + stepInfo.Label, ValidationErrorType.SchemaMismatch, $"Output schema count mismatch: C# expects {csharpOutputSchemas.Count} output(s), decorator declares {decoratorOutputs.Count}", $"C# outputs: [{string.Join(", ", csharpOutputSchemas)}]\nDecorator outputs: [{string.Join(", ", decoratorOutputs)}]" @@ -376,10 +373,10 @@ private ValidationResult ValidateDecorator(PythonNodeInfo nodeInfo, PyObject fun { errors.Add( new ValidationError( - nodeInfo.Label, + stepInfo.Label, ValidationErrorType.SchemaMismatch, $"Input schema mismatch at position {i + 1}:\n C# registration declares: {csharpInputSchemas[i]}\n Python decorator declares: {decoratorInputs[i]}", - "The @node decorator must match the C# generic type parameters" + "The @step decorator must match the C# generic type parameters" ) ); } @@ -392,17 +389,17 @@ private ValidationResult ValidateDecorator(PythonNodeInfo nodeInfo, PyObject fun { errors.Add( new ValidationError( - nodeInfo.Label, + stepInfo.Label, ValidationErrorType.SchemaMismatch, $"Output schema mismatch at position {i + 1}:\n C# registration declares: {csharpOutputSchemas[i]}\n Python decorator declares: {decoratorOutputs[i]}", - "The @node decorator must match the C# generic type parameters" + "The @step decorator must match the C# generic type parameters" ) ); } } // Phase 5: Validate function arity using inspect.signature - var arityResult = ValidateFunctionArity(nodeInfo, function, csharpInputSchemas.Count); + var arityResult = ValidateFunctionArity(stepInfo, function, csharpInputSchemas.Count); errors.AddRange(arityResult.Errors); return new ValidationResult(errors); @@ -412,7 +409,7 @@ private ValidationResult ValidateDecorator(PythonNodeInfo nodeInfo, PyObject fun /// Validates that the Python function's parameter count matches expected input count (Phase 5). /// private ValidationResult ValidateFunctionArity( - PythonNodeInfo nodeInfo, + PythonStepInfo stepInfo, PyObject function, int expectedParamCount ) @@ -433,9 +430,9 @@ int expectedParamCount { errors.Add( new ValidationError( - nodeInfo.Label, + stepInfo.Label, ValidationErrorType.SchemaMismatch, - $"Function parameter count mismatch: Python function '{nodeInfo.FunctionName}' has {actualParamCount} parameter(s), but C# registration expects {expectedParamCount} input(s)", + $"Function parameter count mismatch: Python function '{stepInfo.FunctionName}' has {actualParamCount} parameter(s), but C# registration expects {expectedParamCount} input(s)", $"Update the Python function signature to accept {expectedParamCount} parameter(s)" ) ); @@ -446,7 +443,7 @@ int expectedParamCount { errors.Add( new ValidationError( - nodeInfo.Label, + stepInfo.Label, ValidationErrorType.InvalidFormat, $"Failed to inspect function signature: {ex.Message}", "Ensure the Python function is properly defined" @@ -520,9 +517,9 @@ private bool IsValueTuple(Type type) } /// - /// Information about a Python node extracted from the pipeline. + /// Information about a Python step extracted from the flow. /// - private class PythonNodeInfo + private class PythonStepInfo { public required string Label { get; init; } public required string ModuleName { get; init; } diff --git a/src/extensions/Flowthru.Extensions.Python/build/Flowthru.Extensions.Python.targets b/src/extensions/Flowthru.Extensions.Python/build/Flowthru.Extensions.Python.targets index d44f3120..163994a0 100644 --- a/src/extensions/Flowthru.Extensions.Python/build/Flowthru.Extensions.Python.targets +++ b/src/extensions/Flowthru.Extensions.Python/build/Flowthru.Extensions.Python.targets @@ -68,7 +68,7 @@ PackagePath="build" Visible="false" /> - + - + Link="flowthru\step.py" />