diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json
index b908b557..02960c4e 100644
--- a/.config/dotnet-tools.json
+++ b/.config/dotnet-tools.json
@@ -16,6 +16,13 @@
"dotnet-script"
],
"rollForward": false
+ },
+ "docfx": {
+ "version": "2.78.5",
+ "commands": [
+ "docfx"
+ ],
+ "rollForward": false
}
}
}
\ No newline at end of file
diff --git a/.vscode/flowthru.code-snippets b/.vscode/flowthru.code-snippets
index 1354eadf..59afcb87 100644
--- a/.vscode/flowthru.code-snippets
+++ b/.vscode/flowthru.code-snippets
@@ -10,16 +10,16 @@
"scope": "csharp",
"prefix": "_pipeline",
"body": [
- "using Flowthru.Pipelines;",
+ "using Flowthru.Flows;",
"using $1",
"",
"public static class ${RELATIVE_FILEPATH/.*?(\\w+).cs$/$1/}",
"{",
" public static Pipeline Create(Catalog catalog)",
" {",
- " return PipelineBuilder.CreatePipeline(pipeline =>",
+ " return FlowBuilder.CreateFlow(pipeline =>",
" {",
- " pipeline.AddNode(",
+ " pipeline.AddStep(",
" label: \"$2\",",
" description: \"\"\"",
" $3",
@@ -34,7 +34,7 @@
],
"description": "Creates a basic Flowthru pipeline structure."
},
- "FlowthruNode": {
+ "FlowthruStep": {
"scope": "csharp",
"prefix": "_node",
"body": [
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4f84f98b..d4293e5b 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -74,8 +74,8 @@ Flowthru handles these through an **effect type** called `FlowIO`. If you're
The key runtime guarantees:
- **All I/O is lazy and explicit.** Side effects cannot be accidentally dropped or silently ignored.
-- **Errors are captured, never swallowed.** Node failures propagate to structured pipeline results. Silent `catch {}` blocks are a bug.
-- **Nodes are isolated.** A failing node halts execution and reports which node failed and why — partial silent failures are not possible.
+- **Errors are captured, never swallowed.** Step failures propagate to structured pipeline results. Silent `catch {}` blocks are a bug.
+- **Steps are isolated.** A failing node halts execution and reports which node failed and why — partial silent failures are not possible.
## Decision Rules for Contributors
diff --git a/Directory.Build.props b/Directory.Build.props
index ae821490..b57dec72 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -16,6 +16,11 @@
false
+
+
+ true
+
+
0.1.33
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 9d82936b..5c144be5 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -27,7 +27,7 @@ Consider these examples:
- **"How do I start writing my first pipeline?"** → This is someone learning. They need a tutorial.
- **"How can I read data from a database instead of local files?"** → This is someone working with a specific need. They need a guide.
- **"Why does Flowthru use so many types?"** → This is someone studying the framework's design. They need an explanation.
-- **"What parameters does `CatalogEntry` accept?"** → This is someone looking up technical details. They need reference documentation.
+- **"What parameters does `Item` accept?"** → This is someone looking up technical details. They need reference documentation.
The same topic can yield different documentation depending on the question:
@@ -156,7 +156,7 @@ Let's say you want to document "catalog entries." Start by listing questions:
- "How do I create my first catalog entry?" → Tutorial
- "How do I configure a catalog entry for Parquet files?" → Guide
- "Why are catalog entries properties rather than string keys?" → Explanation
-- "What methods are available on `ICatalogEntry`?" → Reference
+- "What methods are available on `IItem`?" → Reference
Each question becomes a separate piece of documentation in its appropriate category. Together, they serve users at every stage of their journey with catalog entries.
diff --git a/docs/explanation/advanced/storage-composition.md b/docs/explanation/advanced/storage-composition.md
index 0979521a..be92b825 100644
--- a/docs/explanation/advanced/storage-composition.md
+++ b/docs/explanation/advanced/storage-composition.md
@@ -140,11 +140,11 @@ public class EFCoreStorageAdapter : IStorageAdapter>
## Catalog-Level Constraint Narrowing
-After adapter creation, pipeline authors can further constrain catalog entries using `CatalogEntry.Constrain()`:
+After adapter creation, pipeline authors can further constrain catalog entries using `Item.Constrain()`:
```csharp
-public ICatalogEntry> Companies => GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Csv("companies", "data/companies.csv")
+public IItem> Companies => GetOrCreateEntry(() =>
+ ItemFactory.Enumerable.Csv("companies", "data/companies.csv")
.Constrain(traits => traits with { CanWrite = false })
);
```
@@ -201,7 +201,7 @@ The trait system solves all three:
- **Composable**: Eight independent boolean flags
- **Queryable**: `adapter.Traits.CanWrite` checked before operation
-- **Fail-fast**: `CatalogEntry.Constrain()` validates at initialization
+- **Fail-fast**: `Item.Constrain()` validates at initialization
The old interfaces remain as `[Obsolete]` for backward compatibility but have no effect on pipeline behavior.
@@ -211,9 +211,9 @@ The storage architecture is implemented in:
- `src/core/Flowthru/Data/Capabilities/StorageTraits.cs` — the trait record
- `src/core/Flowthru/Data/Storage/ComposedStorageAdapter.cs` — trait merging logic
-- `src/core/Flowthru/Data/CatalogEntry.cs` — `Constrain()` with ratchet validation
+- `src/core/Flowthru/Data/Item.cs` — `Constrain()` with ratchet validation
- `src/core/Flowthru/Data/Storage/Medium/` — medium implementations with traits
- `src/core/Flowthru/Data/Storage/Format/` — format serializers with traits
-Catalog entry factories (`CatalogEntries.Enumerable.Csv(...)`) construct composed adapters with sensible defaults. Extension authors creating custom adapters should declare traits that accurately reflect their capabilities.
+Catalog entry factories (`ItemFactory.Enumerable.Csv(...)`) construct composed adapters with sensible defaults. Extension authors creating custom adapters should declare traits that accurately reflect their capabilities.
diff --git a/docs/explanation/anatomy-of-a-pipeline.md b/docs/explanation/anatomy-of-a-pipeline.md
index ce07a87b..2f197142 100644
--- a/docs/explanation/anatomy-of-a-pipeline.md
+++ b/docs/explanation/anatomy-of-a-pipeline.md
@@ -75,9 +75,9 @@ In this example:
// Data/_01_Raw/Catalog.Raw.cs
public partial class Catalog
{
- public ICatalogEntry> IrisRaw =>
+ public IItem> IrisRaw =>
GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Csv(
+ ItemFactory.Enumerable.Csv(
label: "IrisRaw",
filePath: $"{_basePath}/_01_Raw/Datasets/iris.csv"
)
@@ -95,9 +95,9 @@ In this example:
// Data/_04_Feature/Catalog.Feature.cs
public partial class Catalog
{
- public ICatalogEntry> IrisFeatures =>
+ public IItem> IrisFeatures =>
GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Parquet(
+ ItemFactory.Enumerable.Parquet(
label: "IrisFeatures",
filePath: $"{_basePath}/_04_Feature/Datasets/iris_features.parquet"
)
@@ -169,13 +169,13 @@ Pipelines are made up of **nodes** — transformation functions that are built t
1. When writing a node, you don't need to worry about how it's connecting to other nodes — you just need to make sure it inputs the schema, and outputs the schema
-### Nodes
+### Steps
-Nodes are simply functions. Easy! The **only** purpose of a node is to take in data that has one schema, and convert it to data that has another schema.
+Steps are simply functions. Easy! The **only** purpose of a node is to take in data that has one schema, and convert it to data that has another schema.
```csharp
-// Pipelines/DataEngineering/Nodes/SplitAndEncodeNode.cs
-public static class SplitAndEncodeNode
+// Pipelines/DataEngineering/Steps/SplitAndEncodeStep.cs
+public static class SplitAndEncodeStep
{
public static Func<
IEnumerable, // Input Schema
@@ -194,8 +194,8 @@ public static class SplitAndEncodeNode
Key points:
-- Nodes are a *contract*: that data for the node will **always** come in as the input schemas, and **always** come out as the output schemas.
-- Nodes can have any number of inputs, and any number of outputs — as long as they're defined in the input and output schemas, you're not limited to just one-in, one-out.
+- Steps are a *contract*: that data for the node will **always** come in as the input schemas, and **always** come out as the output schemas.
+- Steps can have any number of inputs, and any number of outputs — as long as they're defined in the input and output schemas, you're not limited to just one-in, one-out.
### Pipelines
@@ -214,11 +214,11 @@ public static class DataEngineeringPipeline
{
public static Pipeline Create(Catalog catalog, Params parameters)
{
- return PipelineBuilder.CreatePipeline(pipeline =>
+ return FlowBuilder.CreateFlow(pipeline =>
{
- pipeline.AddNode(
+ pipeline.AddStep(
label: "SplitAndEncode", // Unique label for this node in the pipeline
- transform: SplitAndEncodeNode.Create(),
+ transform: SplitAndEncodeStep.Create(),
input: catalog.IrisRaw,
output: catalog.IrisFeatures
);
@@ -229,7 +229,7 @@ public static class DataEngineeringPipeline
Key points:
-- **Nodes are never directly connected to each other**: Nodes always take in Catalog entries, and output Catalog entries — *never* directly to each other.
+- **Steps are never directly connected to each other**: Steps always take in Catalog entries, and output Catalog entries — *never* directly to each other.
- **Order doesn't matter.** A node in the pipeline is **only** ever concerned about its input data and output data. Flowthru handles the order when you run the pipeline: as long as the data is available, or generated by another node, your pipeline will run.
## Entry Point
@@ -241,7 +241,7 @@ private static void ConfigureServices(IServiceCollection services, string basePa
{
services.AddFlowthru(flowthru =>
{
- flowthru.UseCatalog(_ => new Catalog(basePath: "Data")));
+ flowthru.RegisterCatalog(_ => new Catalog(basePath: "Data")));
flowthru.RegisterPipeline(
label: "DataEngineering",
@@ -256,4 +256,4 @@ private static void ConfigureServices(IServiceCollection services, string basePa
}
```
-And that's it! This finishes the pipeline anatomy. At this point, you have Catalog entries, connected with Nodes in your Pipelines — everything you need to find new and creative ways to organize, analyze, and report on your data!
+And that's it! This finishes the pipeline anatomy. At this point, you have Catalog entries, connected with Steps in your Pipelines — everything you need to find new and creative ways to organize, analyze, and report on your data!
diff --git a/docs/guides/advanced/container-deployment.md b/docs/guides/advanced/container-deployment.md
index 730615e5..0dcf9c75 100644
--- a/docs/guides/advanced/container-deployment.md
+++ b/docs/guides/advanced/container-deployment.md
@@ -13,7 +13,7 @@ Deploy a Flowthru pipeline as a standalone container image for execution in envi
A containerized pipeline replaces `FlowthruCli` with a minimal `Program.cs` that owns its own DI container. The key difference: no CLI argument parsing, no filesystem-based configuration.
```csharp
-using Flowthru.Pipelines;
+using Flowthru.Flows;
using Flowthru.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
@@ -26,7 +26,7 @@ services.AddLogging(logging =>
services.AddFlowthru(flowthru =>
{
flowthru
- .UseCatalog()
+ .RegisterCatalog()
.RegisterPipeline("Ingest", catalog => IngestPipeline.Create(catalog))
.RegisterPipeline("Transform", catalog => TransformPipeline.Create(catalog));
@@ -50,19 +50,19 @@ return result.Success ? 0 : 1;
// --- helpers ---
-static PipelineSliceStrategy BuildSliceStrategy()
+static FlowSliceStrategy BuildSliceStrategy()
{
static HashSet? ParseCsv(string? value) =>
string.IsNullOrWhiteSpace(value) ? null : new(value.Split(',', StringSplitOptions.RemoveEmptyEntries));
- return new PipelineSliceStrategy
+ return new FlowSliceStrategy
{
Pipelines = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_PIPELINES")),
- FromNodes = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_FROM_NODES")),
- ToNodes = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_TO_NODES")),
+ FromSteps = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_FROM_NODES")),
+ ToSteps = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_TO_NODES")),
FromData = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_FROM_DATA")),
ToData = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_TO_DATA")),
- OnlyNodes = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_ONLY_NODES")),
+ OnlySteps = ParseCsv(Environment.GetEnvironmentVariable("FLOWTHRU_ONLY_NODES")),
};
}
```
@@ -87,7 +87,7 @@ services.AddFlowthru(flowthru =>
{
// No UseConfiguration() call — the IConfiguration above is already registered
flowthru
- .UseCatalog()
+ .RegisterCatalog()
.RegisterPipeline("Ingest", catalog => IngestPipeline.Create(catalog));
});
```
@@ -152,7 +152,7 @@ docker run \
## Inspecting Results
-`ExecutePipelineAsync` returns a `PipelineResult` — a structured value object with `Success`, `ExecutionTime`, per-node `NodeResults` (including individual timing, I/O counts, and exceptions), and an optional top-level `Exception`. Serialize it to structured output for your runtime's observability:
+`ExecutePipelineAsync` returns a `FlowResult` — a structured value object with `Success`, `ExecutionTime`, per-node `StepResults` (including individual timing, I/O counts, and exceptions), and an optional top-level `Exception`. Serialize it to structured output for your runtime's observability:
```csharp
var result = await flowthru.ExecutePipelineAsync(options, cancellationToken: cts.Token);
@@ -163,9 +163,9 @@ if (!result.Success)
result.ExecutionTime.TotalSeconds,
result.Exception?.Message);
- foreach (var (name, node) in result.NodeResults.Where(n => !n.Value.Success))
+ foreach (var (name, node) in result.StepResults.Where(n => !n.Value.Success))
{
- logger.LogError(" Node {Node} failed: {Error}", name, node.Exception?.Message);
+ logger.LogError(" Step {Step} failed: {Error}", name, node.Exception?.Message);
}
}
```
diff --git a/docs/guides/advanced/metadata-providers.md b/docs/guides/advanced/metadata-providers.md
index 817a0cd7..67896dbe 100644
--- a/docs/guides/advanced/metadata-providers.md
+++ b/docs/guides/advanced/metadata-providers.md
@@ -40,8 +40,8 @@ public class DashboardMetadataProvider : IMetadataProvider
{
var payload = new
{
- PipelineName = dag.PipelineName,
- NodeCount = dag.Nodes.Count,
+ FlowName = dag.FlowName,
+ StepCount = dag.Steps.Count,
EdgeCount = dag.Edges.Count,
Timestamp = DateTime.UtcNow
};
@@ -85,14 +85,14 @@ using Flowthru.Meta.Providers;
services.AddFlowthru(flowthru =>
{
- flowthru.UseCatalog(_ => new MyCatalog());
- flowthru.UsePipelines(_ => myPipelines);
+ flowthru.RegisterCatalog(_ => new MyCatalog());
+ flowthru.RegisterPipelines(_ => myPipelines);
flowthru.ConfigureMetadata(meta =>
{
meta.AddProvider(json => json
.WithOutputDirectory("metadata")
- .WithFilenameTemplate("dag-{PipelineName}-{Timestamp}")
+ .WithFilenameTemplate("dag-{FlowName}-{Timestamp}")
.WithTimestamp("yyyyMMdd-HHmmss")
.UseCompactFormat());
@@ -143,9 +143,9 @@ var singlePipelineDag = service.GetDagMetadata(pipelineName: "DataEngineering");
// Get DAG with slicing applied
var slicedDag = service.GetDagMetadata(
- sliceStrategy: new PipelineSliceStrategy
+ sliceStrategy: new FlowSliceStrategy
{
- ToNodes = new HashSet { "TransformNode" }
+ ToSteps = new HashSet { "TransformStep" }
}
);
```
@@ -159,7 +159,7 @@ Useful for tooling, tests, or debugging pipeline structure before execution.
```csharp
meta.AddProvider(json => json
.WithOutputDirectory("metadata")
- .WithFilenameTemplate("dag-{PipelineName}-{Timestamp}")
+ .WithFilenameTemplate("dag-{FlowName}-{Timestamp}")
.WithTimestamp("yyyyMMdd-HHmmss")
.UseCompactFormat());
```
@@ -170,6 +170,6 @@ meta.AddProvider(json => json
meta.AddProvider(mermaid => mermaid
.WithOutputDirectory("metadata")
.WithDirection(MermaidFlowchartDirection.LeftToRight)
- .WithActiveNodeColor("#90EE90")
+ .WithActiveStepColor("#90EE90")
.WithActiveDataColor("#ADD8E6"));
```
diff --git a/docs/guides/advanced/service-integration.md b/docs/guides/advanced/service-integration.md
index 9162654d..757e5826 100644
--- a/docs/guides/advanced/service-integration.md
+++ b/docs/guides/advanced/service-integration.md
@@ -18,7 +18,7 @@ var builder = WebApplication.CreateBuilder(args);
builder.Services.AddFlowthru(flowthru =>
{
flowthru
- .UseCatalog()
+ .RegisterCatalog()
.RegisterPipeline("Ingest", c => IngestPipeline.Create(c))
.RegisterPipeline("Transform", c => TransformPipeline.Create(c))
.UseStorageStrategy();
@@ -42,7 +42,7 @@ var myParams = builder.Configuration
builder.Services.AddFlowthru(flowthru =>
{
flowthru
- .UseCatalog()
+ .RegisterCatalog()
.RegisterPipeline(
"Transform",
(catalog, p) => TransformPipeline.Create(catalog, p),
@@ -57,7 +57,7 @@ builder.Services.AddFlowthru(flowthru =>
{
flowthru
.UseConfiguration(opts => opts.BasePath = "/app/pipeline-config")
- .UseCatalog()
+ .RegisterCatalog()
.RegisterPipelineWithConfiguration(
"Transform",
(catalog, p) => TransformPipeline.Create(catalog, p),
@@ -79,7 +79,7 @@ app.MapPost("/pipelines/run", async (
{
var options = new ExecutionOptions
{
- SliceStrategy = new PipelineSliceStrategy
+ SliceStrategy = new FlowSliceStrategy
{
Pipelines = request.Pipelines is { Count: > 0 }
? request.Pipelines.ToHashSet()
@@ -115,25 +115,25 @@ public record PipelineRunRequest
public List? Pipelines { get; init; }
public List? ToData { get; init; }
public List? FromData { get; init; }
- public List? OnlyNodes { get; init; }
+ public List? OnlySteps { get; init; }
}
```
-`PipelineSliceStrategy` properties are `IReadOnlySet?` — convert from lists at the boundary.
+`FlowSliceStrategy` properties are `IReadOnlySet?` — convert from lists at the boundary.
## Handling Results
-`PipelineResult` carries structured execution data. Map it to your application's response model rather than returning it directly:
+`FlowResult` carries structured execution data. Map it to your application's response model rather than returning it directly:
```csharp
-static PipelineRunResponse MapToResponse(PipelineResult result)
+static PipelineRunResponse MapToResponse(FlowResult result)
{
return new PipelineRunResponse
{
Success = result.Success,
DurationMs = result.ExecutionTime.TotalMilliseconds,
- NodesExecuted = result.NodeResults.Count,
- NodeSummaries = result.NodeResults.Select(kvp => new NodeSummary
+ StepsExecuted = result.StepResults.Count,
+ StepSummaries = result.StepResults.Select(kvp => new StepSummary
{
Name = kvp.Key,
Success = kvp.Value.Success,
@@ -168,8 +168,8 @@ public class PipelineBackgroundService : BackgroundService
if (result.Success)
{
_logger.LogInformation(
- "Pipeline completed: {NodeCount} nodes in {Duration}s",
- result.NodeResults.Count,
+ "Pipeline completed: {StepCount} nodes in {Duration}s",
+ result.StepResults.Count,
result.ExecutionTime.TotalSeconds);
}
else
diff --git a/docs/guides/constraining-catalog-entries.md b/docs/guides/constraining-catalog-entries.md
index 4e08eec3..427749a8 100644
--- a/docs/guides/constraining-catalog-entries.md
+++ b/docs/guides/constraining-catalog-entries.md
@@ -22,8 +22,8 @@ Call `.Constrain()` on any catalog entry to narrow its capabilities:
```csharp
public class DataCatalog : DataCatalogBase
{
- public ICatalogEntry> Companies => GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Csv("companies", "data/companies.csv")
+ public IItem> Companies => GetOrCreateEntry(() =>
+ ItemFactory.Enumerable.Csv("companies", "data/companies.csv")
.Constrain(traits => traits with { CanWrite = false })
);
}
@@ -39,20 +39,20 @@ Mark external data sources as read-only to prevent accidental modifications:
```csharp
// API endpoint that only supports GET requests
-public ICatalogEntry WeatherFeed => GetOrCreateEntry(() =>
- CatalogEntries.Single.Http("weather", "https://api.weather.com/current")
+public IItem WeatherFeed => GetOrCreateEntry(() =>
+ ItemFactory.Single.Http("weather", "https://api.weather.com/current")
.Constrain(traits => traits with { CanWrite = false })
);
// Production database view used for reporting
-public ICatalogEntry> ProductionSales => GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.EFCore("sales", dbContext)
+public IItem> ProductionSales => GetOrCreateEntry(() =>
+ ItemFactory.Enumerable.EFCore("sales", dbContext)
.Constrain(traits => traits with { CanWrite = false })
);
// Archived historical data that must not change
-public ICatalogEntry> HistoricalTransactions => GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Parquet("archive", "data/2024/transactions.parquet")
+public IItem> HistoricalTransactions => GetOrCreateEntry(() =>
+ ItemFactory.Enumerable.Parquet("archive", "data/2024/transactions.parquet")
.Constrain(traits => traits with { CanWrite = false })
);
```
@@ -62,8 +62,8 @@ public ICatalogEntry> HistoricalTransactions => GetOrCr
Mark catalog entries that require network connectivity to prevent offline execution attempts:
```csharp
-public ICatalogEntry> RemoteUsers => GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Http("users", "https://internal.api/users")
+public IItem> RemoteUsers => GetOrCreateEntry(() =>
+ ItemFactory.Enumerable.Http("users", "https://internal.api/users")
.Constrain(traits => traits with
{
RequiresNetwork = true,
@@ -77,8 +77,8 @@ public ICatalogEntry> RemoteUsers => GetOrCreateEntry(() =>
Mark in-memory or cache-based entries as non-persistent:
```csharp
-public ICatalogEntry Cache => GetOrCreateEntry(() =>
- CatalogEntries.Single.Memory("cache")
+public IItem Cache => GetOrCreateEntry(() =>
+ ItemFactory.Single.Memory("cache")
// Memory adapter already sets IsPersistent = false
);
```
@@ -89,7 +89,7 @@ If you try to use a constrained entry incorrectly, the pipeline fails during con
```csharp
// ❌ This fails at pipeline.Build() — before any data is processed
-pipeline.AddNode(
+pipeline.AddStep(
name: "WriteToReadOnly",
transform: node,
input: catalog.InputData,
@@ -125,8 +125,8 @@ Attempts to relax constraints throw `InvalidOperationException` with a detailed
Apply multiple constraints in a single call:
```csharp
-public ICatalogEntry> AuditLogs => GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Csv("audit", "logs/audit.csv")
+public IItem> AuditLogs => GetOrCreateEntry(() =>
+ ItemFactory.Enumerable.Csv("audit", "logs/audit.csv")
.Constrain(traits => traits with
{
CanWrite = false,
@@ -139,8 +139,8 @@ public ICatalogEntry> AuditLogs => GetOrCreateEntry(() =>
Or chain multiple constraints for readability:
```csharp
-public ICatalogEntry> Customers => GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Csv("customers", "data/customers.csv")
+public IItem> Customers => GetOrCreateEntry(() =>
+ ItemFactory.Enumerable.Csv("customers", "data/customers.csv")
.Constrain(traits => traits with { CanWrite = false })
.Constrain(traits => traits with { RequiresNetwork = false })
);
@@ -170,13 +170,13 @@ Default values represent **filesystem-file** semantics — the common case for l
Prevent pipelines from modifying reference data used across multiple projects:
```csharp
-public ICatalogEntry> Countries => GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Csv("countries", "reference/countries.csv")
+public IItem> Countries => GetOrCreateEntry(() =>
+ ItemFactory.Enumerable.Csv("countries", "reference/countries.csv")
.Constrain(traits => traits with { CanWrite = false })
);
-public ICatalogEntry> Currencies => GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Json("currencies", "reference/currencies.json")
+public IItem> Currencies => GetOrCreateEntry(() =>
+ ItemFactory.Enumerable.Json("currencies", "reference/currencies.json")
.Constrain(traits => traits with { CanWrite = false })
);
```
@@ -186,9 +186,9 @@ public ICatalogEntry> Currencies => GetOrCreateEntry(() =>
Use configuration to apply different constraints in different environments:
```csharp
-public ICatalogEntry> Orders => GetOrCreateEntry(() =>
+public IItem> Orders => GetOrCreateEntry(() =>
{
- var entry = CatalogEntries.Enumerable.EFCore("orders", dbContext, readOnly: false);
+ var entry = ItemFactory.Enumerable.EFCore("orders", dbContext, readOnly: false);
// In production, make orders read-only for reporting pipelines
if (_environment.IsProduction)
@@ -206,13 +206,13 @@ In multi-stage pipelines where early stages produce data for later stages, preve
```csharp
// Stage 1: Data ingestion (writes allowed)
-public ICatalogEntry> RawData => GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Csv("raw", "data/raw.csv")
+public IItem> RawData => GetOrCreateEntry(() =>
+ ItemFactory.Enumerable.Csv("raw", "data/raw.csv")
);
// Stage 2: Used as read-only input to transformations
-public ICatalogEntry> RawDataReadOnly => GetOrCreateEntry(() =>
- CatalogEntries.Enumerable.Csv("raw_readonly", "data/raw.csv")
+public IItem> RawDataReadOnly => GetOrCreateEntry(() =>
+ ItemFactory.Enumerable.Csv("raw_readonly", "data/raw.csv")
.Constrain(traits => traits with { CanWrite = false })
);
```
diff --git a/docs/guides/misc/AGENTS.md b/docs/guides/misc/AGENTS.md
index 40a61b43..9b205757 100644
--- a/docs/guides/misc/AGENTS.md
+++ b/docs/guides/misc/AGENTS.md
@@ -26,9 +26,9 @@ Data/ # Everything related to data definitions
└── _08_Reporting/ # Descriptive and summary outputs for business consumption
Pipelines/ # All pipelines
-└── /
- ├── Pipeline.cs # Nodes wired to input/output catalog entries
- └── Nodes/
+└── /
+ ├── Pipeline.cs # Steps wired to input/output catalog entries
+ └── Steps/
└── *.cs # Individual transformation nodes
```
@@ -49,7 +49,7 @@ Follow these steps in order:
2. **Plan intermediate schemas.** Identify what data shapes are needed between each transformation step.
3. **Write schemas.** Create schema classes for input, output, and all intermediary data in the appropriate `Data//Schemas/` directories.
4. **Create catalog entries.** Add catalog entries for each schema in the corresponding `Catalog..cs` file. Each entry declares its schema type and storage strategy.
-5. **Write nodes.** Implement the transformation logic in `Pipelines//Nodes/`. Each node declares typed inputs and outputs matching the schemas from step 3.
-6. **Wire the pipeline.** Connect nodes to their catalog entries in `Pipelines//Pipeline.cs`.
+5. **Write nodes.** Implement the transformation logic in `Pipelines//Steps/`. Each node declares typed inputs and outputs matching the schemas from step 3.
+6. **Wire the pipeline.** Connect nodes to their catalog entries in `Pipelines//Pipeline.cs`.
7. **Register the pipeline.** Add the pipeline to `Program.cs` so the CLI runner can discover it.
8. **Run and confirm.** Execute the pipeline with `dotnet run` and verify the output.
diff --git a/docs/guides/slicing-pipelines.md b/docs/guides/slicing-pipelines.md
index 3eaddb45..6e8648d2 100644
--- a/docs/guides/slicing-pipelines.md
+++ b/docs/guides/slicing-pipelines.md
@@ -100,11 +100,11 @@ using Flowthru.Services.Models;
var options = new ExecutionOptions
{
- SliceStrategy = new PipelineSliceStrategy
+ SliceStrategy = new FlowSliceStrategy
{
Pipelines = new HashSet { "DataScience" },
- FromNodes = new HashSet { "PreprocessCompanies", "PreprocessShuttles" },
- ToNodes = new HashSet { "CreateModelInput" }
+ FromSteps = new HashSet { "PreprocessCompanies", "PreprocessShuttles" },
+ ToSteps = new HashSet { "CreateModelInput" }
}
};
@@ -114,14 +114,14 @@ var result = await service.ExecutePipelineAsync(options, exportMetadata: true, m
### All Strategy Properties
```csharp
-var strategy = new PipelineSliceStrategy
+var strategy = new FlowSliceStrategy
{
Pipelines = new HashSet { "DataScience", "Reporting" }, // filter by pipeline name
- FromNodes = new HashSet { "NodeA", "NodeB" }, // + upstream
- ToNodes = new HashSet { "NodeC" }, // + downstream
+ FromSteps = new HashSet { "StepA", "StepB" }, // + upstream
+ ToSteps = new HashSet { "StepC" }, // + downstream
FromData = new HashSet { "model_input" }, // consumers + downstream
ToData = new HashSet { "raw_data" }, // producers + upstream
- OnlyNodes = new HashSet { "NodeD", "NodeE" } // explicit allowlist
+ OnlySteps = new HashSet { "StepD", "StepE" } // explicit allowlist
};
```
@@ -135,12 +135,12 @@ var options = new ExecutionOptions(); // No slicing, all pipelines execute
Slicing guarantees runnability — the result is always a valid sub-DAG:
-1. **Pipelines** filters nodes by pipeline name prefix (e.g., "DataScience.NodeName")
-2. **FromNodes** includes all upstream dependencies (transitive closure)
-3. **ToNodes** includes all downstream dependents (transitive closure)
+1. **Pipelines** filters nodes by pipeline name prefix (e.g., "DataScience.StepName")
+2. **FromSteps** includes all upstream dependencies (transitive closure)
+3. **ToSteps** includes all downstream dependents (transitive closure)
4. **FromData** resolves to consumer nodes, then includes downstream (transitive closure)
5. **ToData** resolves to producer nodes, then includes upstream (transitive closure)
-6. **OnlyNodes** automatically includes required dependencies
+6. **OnlySteps** automatically includes required dependencies
7. Multiple strategies intersect — each narrows the result set
**Slicing is additive only.** There is no `--except` flag because subtractive operations break the runnability guarantee.
@@ -169,7 +169,7 @@ Available pipelines: DataScience, DataProcessing, Reporting
```
```
-✗ FromNodes references non-existent node: 'InvalidNode'
+✗ FromSteps references non-existent node: 'InvalidStep'
Available nodes: PreprocessCompanies, PreprocessShuttles, ...
```
diff --git a/docs/guides/using-efcore-catalog-entries.md b/docs/guides/using-efcore-catalog-entries.md
index 38e2e31d..34b33790 100644
--- a/docs/guides/using-efcore-catalog-entries.md
+++ b/docs/guides/using-efcore-catalog-entries.md
@@ -8,7 +8,7 @@ This guide is for teams who already have a working Flowthru pipeline backed by f
dotnet add package Flowthru.Extensions.EFCore
```
-This package provides `EFCoreCatalogEntries`, the primary namespace for creating EFCore-backed catalog entries.
+This package provides `EFCoreItemFactory`, the primary namespace for creating EFCore-backed catalog entries.
## What your schemas need
@@ -87,7 +87,7 @@ Wire the factory from your DI container:
```csharp
// Program.cs
-flowthru.UseCatalog(sp => new Catalog(
+flowthru.RegisterCatalog(sp => new Catalog(
basePath: Path.Combine(basePath, "Data"),
contextFactory: sp.GetRequiredService>()
));
@@ -103,9 +103,9 @@ using Flowthru.Extensions.EFCore.Data;
public partial class Catalog
{
- public ICatalogEntry> PreprocessedCompanies =>
+ public IItem> PreprocessedCompanies =>
GetOrCreateEntry(() =>
- EFCoreCatalogEntries.Enumerable.EFCore(
+ EFCoreItemFactory.Enumerable.EFCore(
label: "PreprocessedCompanies",
contextFactory: _contextFactory
)
@@ -123,9 +123,9 @@ For tables that store exactly one row — a trained model, a configuration recor
// Data/_06_Models/Catalog.Models.cs
public partial class Catalog
{
- public ICatalogEntry Regressor =>
+ public IItem Regressor =>
GetOrCreateEntry(() =>
- EFCoreCatalogEntries.Single.EFCore(
+ EFCoreItemFactory.Single.EFCore(
label: "Regressor",
contextFactory: _contextFactory
)
@@ -141,21 +141,21 @@ The `queryCustomizer` parameter receives the raw `IQueryable` before the adap
```csharp
// Order for deterministic output
-EFCoreCatalogEntries.Enumerable.EFCore(
+EFCoreItemFactory.Enumerable.EFCore(
label: "Shuttles",
contextFactory: _contextFactory,
queryCustomizer: q => q.OrderBy(s => s.Id)
)
// Load with a navigation property
-EFCoreCatalogEntries.Enumerable.EFCore(
+EFCoreItemFactory.Enumerable.EFCore(
label: "Persons",
contextFactory: _contextFactory,
queryCustomizer: q => q.Include(p => p.Address).AsNoTracking()
)
// Filter to a subset
-EFCoreCatalogEntries.Enumerable.EFCore(
+EFCoreItemFactory.Enumerable.EFCore(
label: "ActiveRecords",
contextFactory: _contextFactory,
queryCustomizer: q => q.Where(r => r.IsActive)
@@ -170,7 +170,7 @@ The default save strategy is `RemoveRange` + `AddRange` (full replace semantics,
```csharp
// Upsert instead of replace
-EFCoreCatalogEntries.Enumerable.EFCore(
+EFCoreItemFactory.Enumerable.EFCore(
label: "Companies",
contextFactory: _contextFactory,
saveFunc: async (ctx, data, ct) =>
@@ -206,7 +206,7 @@ internal static class MyProjectSaveFuncs
}
// In your catalog entry:
-EFCoreCatalogEntries.Enumerable.EFCore(
+EFCoreItemFactory.Enumerable.EFCore(
label: "Companies",
contextFactory: _contextFactory,
saveFunc: MyProjectSaveFuncs.ReplaceCompanies
@@ -228,7 +228,7 @@ saveFunc: async (ctx, data, ct) =>
Use `.Constrain()` to mark source or reference tables as read-only. Flowthru will fail at pipeline build time — before any data moves — if a node attempts to write to this entry:
```csharp
-EFCoreCatalogEntries.Enumerable.EFCore(
+EFCoreItemFactory.Enumerable.EFCore(
label: "SourceRecords",
contextFactory: _sourceFactory)
.Constrain(traits => traits with { CanWrite = false })
@@ -241,7 +241,7 @@ See [Constraining Catalog Entries](constraining-catalog-entries.md) for the full
By default, an empty table fails pre-flight validation. The adapter expects each catalog entry to have data before the pipeline runs. For tables that are legitimately empty on first run — audit logs, optional output tables, or incremental pipelines — set `allowEmptyData: true`:
```csharp
-EFCoreCatalogEntries.Enumerable.EFCore(
+EFCoreItemFactory.Enumerable.EFCore(
label: "AuditEvents",
contextFactory: _contextFactory,
allowEmptyData: true
diff --git a/docs/project.json b/docs/project.json
new file mode 100644
index 00000000..292c1716
--- /dev/null
+++ b/docs/project.json
@@ -0,0 +1,19 @@
+{
+ "name": "docs",
+ "$schema": "../node_modules/nx/schemas/project-schema.json",
+ "projectType": "library",
+ "sourceRoot": "./docs",
+ "targets": {
+ "build:reference": {
+ "//": "Generates markdown API reference from C# docstrings into docs/reference/src/",
+ "executor": "nx:run-commands",
+ "dependsOn": [ "flowthru:restore" ],
+ "options": {
+ "commands": [
+ "./scripts/docfx-metadata.sh --projects src"
+ ],
+ "parallel": false
+ }
+ }
+ }
+}
diff --git a/docs/reference/.gitkeep b/docs/reference/.gitkeep
deleted file mode 100644
index e69de29b..00000000
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.FlowthruSchemaAttribute.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.FlowthruSchemaAttribute.md
new file mode 100644
index 00000000..cc8cbee1
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.FlowthruSchemaAttribute.md
@@ -0,0 +1,71 @@
+# Class FlowthruSchemaAttribute
+
+Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md)
+Assembly: Flowthru.Core.dll
+
+Marks a schema type for automatic interface generation. The source generator
+will analyze the type's properties and emit the appropriate marker interfaces:
+
+Purpose: Indicates a schema contains only primitive properties,
+with no collections or nested objects.
+
+
+Flat Structure Requirements:
+
+
All properties are primitives (int, long, double, decimal, bool, string, DateTime, etc.)
Nullable primitives (int?, double?, bool?, etc.) are allowed
Enums and nullable enums are allowed
Value types that serialize to single values (Guid, TimeSpan, etc.) are allowed
+
+Not Flat (Use INestedSchema instead):
+
+
Collections: List<T>, Array, IEnumerable<T>, etc.
Nested objects: Custom class/record properties
Dictionaries or complex structures
+
+Design Rationale:
+
+
+Separating structural properties (flat vs nested) from serialization capabilities
+(text/structured/binary) enables:
+- Explicit schema documentation
+- Compile-time format validation
+- Format serializers can enforce appropriate constraints
+- Clear intent about data structure complexity
+
+
+Relationship with Serialization Markers:
+
+
+A flat schema should also implement one or more serialization capability markers:
+- - Can serialize to CSV/TSV
+- - Can serialize to JSON/XML
+- - Can serialize to Parquet/Avro
+
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IFlatSerializable.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IFlatSerializable.md
new file mode 100644
index 00000000..0ea58d3c
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IFlatSerializable.md
@@ -0,0 +1,70 @@
+# Interface IFlatSerializable
+
+Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md)
+Assembly: Flowthru.Core.dll
+
+Marker interface for schema types that contain only flat, primitive data.
+
+```csharp
+public interface IFlatSerializable
+```
+
+## Examples
+
+
// ✅ Flat schema - CSV compatible
+public record CompanySchema : IFlatSerializable
+{
+ public string Id { get; init; } = null!;
+ public string Name { get; init; } = null!;
+ public decimal Rating { get; init; }
+ public int FoundedYear { get; init; }
+ public bool IataApproved { get; init; }
+}
+
+// ✅ Compiles successfully - flat schema with flat storage
+catalog.Companies = CreateCsvDataset<CompanySchema>("companies", "companies.csv");
+
+// ❌ Nested schema - requires JSON or Parquet
+public record CrossValidationResults : INestedSerializable
+{
+ public List<FoldMetric> FoldMetrics { get; init; } = new();
+ public double MeanR2Score { get; init; }
+}
+
+// ❌ Compile error: CrossValidationResults does not implement IFlatSerializable
+catalog.Results = CreateCsvDataset<CrossValidationResults>("results", "results.csv");
+
+// ✅ Correct: Use JSON for nested data
+catalog.Results = CreateJsonObject<CrossValidationResults>("results", "results.json");
JSON files (JsonCatalogDataset<T> - also supports nested)
Parquet files (ParquetCatalogDataset<T> - also supports nested)
Database tables (flat row structure)
+
+Design Rationale:
+
+
+This marker interface follows Flowthru's philosophy of "fail at compile-time, not runtime."
+By requiring flat-only storage formats to constrain their generic type parameter with
+where T : IFlatSerializable, we catch schema-format mismatches during compilation
+rather than discovering silent data loss or runtime serialization errors during execution.
+
Intent Declaration: Distinguishes "happens to be flat" from "designed to be flat"
Migration Safety: Identifies schemas that cannot use flat formats
Future Validation: Enables build-time checks via analyzers
+
+Relationship with IFlatSchema:
+
+
+These interfaces are mutually exclusive. A schema should implement exactly one:
+
+
- All primitive properties
- Contains collections or nested objects
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.INestedSerializable.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.INestedSerializable.md
new file mode 100644
index 00000000..779d3d4c
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.INestedSerializable.md
@@ -0,0 +1,78 @@
+# Interface INestedSerializable
+
+Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md)
+Assembly: Flowthru.Core.dll
+
+Marker interface for schema types that contain nested structures or collections.
+
+```csharp
+public interface INestedSerializable
+```
+
+## Examples
+
+
// ✅ Nested schema - requires hierarchical storage
+public record CrossValidationResults : INestedSerializable
+{
+ // Collection of nested objects
+ public List<FoldMetric> FoldMetrics { get; init; } = new();
+
+ // Flat properties are fine in nested schemas
+ public double MeanR2Score { get; init; }
+ public double StdDevR2Score { get; init; }
+ public int NumFolds { get; init; }
+}
+
+public record FoldMetric : IFlatSerializable // Individual fold is flat
+{
+ public int FoldNumber { get; init; }
+ public double R2Score { get; init; }
+ public double MeanAbsoluteError { get; init; }
+}
+
+// ✅ Correct: Use JSON for nested data
+catalog.CrossValidationResults = CreateJsonObject<CrossValidationResults>(
+ "cross_validation_results",
+ "model_output/cross_validation_results.json");
+
+// ❌ Compile error if attempted with CSV (prevented by IFlatSerializable constraint)
+// catalog.CrossValidationResults = CreateCsvDataset<CrossValidationResults>(...);
+
+## Remarks
+
+
+Purpose: Documents that a schema contains hierarchical data and
+requires storage formats that support nested structures (JSON, Parquet, XML, etc.).
+
+
+What qualifies as "nested"?
+
+
+A schema is considered nested if it contains one or more of:
+
❌ Relational database tables - requires denormalization or separate tables
+
+Design Rationale:
+
+
+While this interface is not enforced by generic constraints (since nested-compatible
+formats like JSON can also handle flat data), it serves important documentation and
+validation purposes:
+
+
Self-Documentation: Clearly signals schema complexity to developers
Future Analyzer Support: Enables build-time validation via Roslyn analyzers
Intent Declaration: Distinguishes "happens to be flat" from "designed to be flat"
Migration Safety: Helps identify schemas that cannot be migrated to flat formats
+
+Relationship with IFlatSerializable:
+
+
+These interfaces are mutually exclusive. A schema should implement exactly one:
+
+
- All primitive properties, CSV-compatible
- Contains collections or nested objects
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IScalar.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IScalar.md
new file mode 100644
index 00000000..cd7e7808
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IScalar.md
@@ -0,0 +1,67 @@
+# Interface IScalar
+
+Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md)
+Assembly: Flowthru.Core.dll
+
+Marker interface for types that serialize to a single primitive value —
+i.e., they produce "key": value in JSON, not "key": {...} or "key": [...].
+
+```csharp
+public interface IScalar
+```
+
+## Examples
+
+
// ✅ Single-value wrapper — safe to implement IScalar
+public readonly record struct CustomerId(string Value) : IScalar;
+
+// ✅ Schema using the NewType is classified as flat
+[FlowthruSchema]
+public partial record OrderSchema
+{
+ public required CustomerId Id { get; init; }
+ public required string Name { get; init; }
+}
+
+// ❌ Multi-property struct — NOT a scalar, do not implement IScalar
+public readonly record struct Address(string Street, string City) /* : IScalar — wrong */;
+
+## Remarks
+
+
+Purpose: Enables user-defined NewTypes and value-object wrappers to participate
+in flat schema classification. Without this interface, the source generator cannot distinguish
+a CustomerId wrapping a string from a nested object requiring {...}
+serialization.
+
+
+The JSON Test:
+
+
+The definitive question when implementing this interface is: does your type serialize to a
+single JSON value? A type is a flat scalar if and only if it produces one of:
+
+
"key": 42 — numeric
"key": "abc" — string
"key": true — boolean
"key": null — null
+
+If your type requires "key": { "inner": ... } or "key": [...], it is NOT a
+scalar and implementing this interface would misrepresent its structure, causing silent data
+loss or serialization failures in flat formats like CSV.
+
+
+Typical Use Cases:
+
+
NewType / value-object wrappers: record CustomerId(string Value) : IScalar
Strong-typed identifiers backed by a primitive: record OrderRef(Guid Id) : IScalar
Domain primitives that round-trip through a single string or numeric field
+
+What this is NOT for:
+
+
Types with multiple public properties — those are nested objects
Collections or dictionaries
BCL types like Guid, DateTime, TimeSpan — those are recognized
+automatically by the source generator as known flat scalars
+
+Relationship with :
+
+
+ marks a row type — a schema whose properties are all
+scalars. marks a property type — a single value that can
+appear as a column in a flat row. Nesting a flat row inside another row is still nesting.
+
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IStructuredSerializable.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IStructuredSerializable.md
new file mode 100644
index 00000000..5024d787
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.IStructuredSerializable.md
@@ -0,0 +1,66 @@
+# Interface IStructuredSerializable
+
+Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md)
+Assembly: Flowthru.Core.dll
+
+Marker interface for schema types that can be serialized to structured formats (JSON, XML).
+
+```csharp
+public interface IStructuredSerializable
+```
+
+## Examples
+
+
+Structured serialization is the most flexible format capability:
+- Can represent any schema structure (flat or nested)
+- Human-readable and widely supported
+- Preserves type information and hierarchy
+- Suitable for configuration, results, and complex data
+
+
+Typical Usage Patterns:
+
+
Nested schemas: Must use structured serialization (JSON/XML only option)
Flat schemas: Can use structured serialization when human-readability matters
Configuration objects: Often use JSON for flexibility
Model metadata: JSON captures complex metrics and parameters
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.ITextSerializable.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.ITextSerializable.md
new file mode 100644
index 00000000..add434c6
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.ITextSerializable.md
@@ -0,0 +1,67 @@
+# Interface ITextSerializable
+
+Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md)
+Assembly: Flowthru.Core.dll
+
+Marker interface for schema types that can be serialized to text-based formats (CSV, TSV).
+
+```csharp
+public interface ITextSerializable
+```
+
+## Examples
+
+
// ✅ Flat schema with text serialization
+public sealed record CompanySchema(
+ int Id,
+ string Name,
+ float Rating
+) : IFlatSchema, ITextSerializable;
+
+// ✅ Flat schema with multiple serialization capabilities
+public sealed record DataRow(
+ DateTime Timestamp,
+ double Value,
+ string Category
+) : IFlatSchema, ITextSerializable, IBinarySerializable;
+
+// ❌ Cannot be text serialized - nested structure
+public sealed record NestedData(
+ string Id,
+ List<string> Tags // Collection!
+) : INestedSchema, IStructuredSerializable; // Not ITextSerializable
+
+## Remarks
+
+
+Purpose: Indicates a schema is compatible with text-based, flat file
+formats that represent data in rows and columns.
+
+
+Compatible Formats:
+
+
CSV (Comma-Separated Values)
TSV (Tab-Separated Values)
Other delimited text formats
+
+Requirements:
+
+
+Schemas implementing this interface must:
+
+
Also implement (no nested structures)
Have properties that can be converted to/from string representation
Support single-value serialization per field
+
+Design Rationale:
+
+
+Separating serialization capability from structure allows:
+- Compile-time enforcement: CsvFormatSerializer<T> where T : IFlatSchema, ITextSerializable
+- Multiple serialization targets: A flat schema can be both text and binary serializable
+- Clear documentation of supported formats
+
+
+Typical Usage:
+
+
+Most flat schemas should implement both and
+ to support maximum format flexibility.
+
// Example: Magic: The Gathering color enum
+public enum MtgColor
+{
+ [SerializedEnum("W")]
+ White,
+
+ [SerializedEnum("U")]
+ Blue,
+
+ [SerializedEnum("B")]
+ Black,
+
+ [SerializedEnum("R")]
+ Red,
+
+ [SerializedEnum("G")]
+ Green
+}
+
+// Example: Rarity enum with lowercase convention
+public enum Rarity
+{
+ [SerializedEnum("common")]
+ Common,
+
+ [SerializedEnum("uncommon")]
+ Uncommon,
+
+ [SerializedEnum("rare")]
+ Rare,
+
+ [SerializedEnum("mythic")]
+ Mythic
+}
+
+// Usage in schema - no additional configuration needed
+public record Card(
+ string Name,
+ MtgColor Color, // Automatically serializes using [SerializedEnum] mappings
+ Rarity Rarity // Works across all formats (JSON, CSV, Excel, Parquet)
+) : IFlatSchema, ITextSerializable;
+
+## Remarks
+
+
+Purpose: Provides explicit string mappings for enum values across all storage formats.
+
+
+Format Agnostic: Works uniformly across CSV, Excel, JSON, Parquet, and future formats.
+The underlying serialization mechanism is abstracted away.
+
+
+Required for All Enum Members: Every enum member must have this attribute when used
+in schemas that will be serialized. This ensures explicit, documented mappings and prevents
+accidental mismatches between C# names and external data formats.
+
+
+Validation: The Flowthru pipeline validates at build time that all enum members
+used in schemas have this attribute applied.
+
+
+Common Use Cases:
+
+
Abbreviated values: [SerializedEnum("W")] White
Lowercase conventions: [SerializedEnum("common")] Common
Legacy formats: [SerializedEnum("STATUS_ACTIVE")] Active
+
+## Constructors
+
+### SerializedEnumAttribute\(string\)
+
+Initializes a new instance of the class.
+
+```csharp
+public SerializedEnumAttribute(string value)
+```
+
+#### Parameters
+
+`value` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+The string value to use when serializing this enum member.
+
+#### Exceptions
+
+ [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception)
+
+Thrown when value is null.
+
+ [ArgumentException](https://learn.microsoft.com/dotnet/api/system.argumentexception)
+
+Thrown when value is empty or whitespace.
+
+## Properties
+
+### Value
+
+Gets the serialized string value for this enum member.
+
+```csharp
+public string Value { get; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.SerializedLabelAttribute.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.SerializedLabelAttribute.md
new file mode 100644
index 00000000..383b534e
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.SerializedLabelAttribute.md
@@ -0,0 +1,143 @@
+# Class SerializedLabelAttribute
+
+Namespace: [Flowthru.Abstractions](Flowthru.Abstractions.md)
+Assembly: Flowthru.Core.dll
+
+Specifies the external field name for a property when serialized to/from storage.
+
+```csharp
+[AttributeUsage(AttributeTargets.Property, AllowMultiple = false, Inherited = true)]
+public sealed class SerializedLabelAttribute : Attribute
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[Attribute](https://learn.microsoft.com/dotnet/api/system.attribute) ←
+[SerializedLabelAttribute](Flowthru.Abstractions.SerializedLabelAttribute.md)
+
+#### Inherited Members
+
+[Attribute.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.attribute.equals),
+[Attribute.GetCustomAttribute\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-assembly\-system\-type\)),
+[Attribute.GetCustomAttribute\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-assembly\-system\-type\-system\-boolean\)),
+[Attribute.GetCustomAttribute\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-memberinfo\-system\-type\)),
+[Attribute.GetCustomAttribute\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)),
+[Attribute.GetCustomAttribute\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-module\-system\-type\)),
+[Attribute.GetCustomAttribute\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-module\-system\-type\-system\-boolean\)),
+[Attribute.GetCustomAttribute\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-parameterinfo\-system\-type\)),
+[Attribute.GetCustomAttribute\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattribute\#system\-attribute\-getcustomattribute\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)),
+[Attribute.GetCustomAttributes\(Assembly\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\)),
+[Attribute.GetCustomAttributes\(Assembly, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-boolean\)),
+[Attribute.GetCustomAttributes\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-type\)),
+[Attribute.GetCustomAttributes\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-assembly\-system\-type\-system\-boolean\)),
+[Attribute.GetCustomAttributes\(MemberInfo\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\)),
+[Attribute.GetCustomAttributes\(MemberInfo, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-boolean\)),
+[Attribute.GetCustomAttributes\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-type\)),
+[Attribute.GetCustomAttributes\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)),
+[Attribute.GetCustomAttributes\(Module\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\)),
+[Attribute.GetCustomAttributes\(Module, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-boolean\)),
+[Attribute.GetCustomAttributes\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-type\)),
+[Attribute.GetCustomAttributes\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-module\-system\-type\-system\-boolean\)),
+[Attribute.GetCustomAttributes\(ParameterInfo\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\)),
+[Attribute.GetCustomAttributes\(ParameterInfo, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-boolean\)),
+[Attribute.GetCustomAttributes\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-type\)),
+[Attribute.GetCustomAttributes\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.getcustomattributes\#system\-attribute\-getcustomattributes\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)),
+[Attribute.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.attribute.gethashcode),
+[Attribute.IsDefaultAttribute\(\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefaultattribute),
+[Attribute.IsDefined\(Assembly, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-assembly\-system\-type\)),
+[Attribute.IsDefined\(Assembly, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-assembly\-system\-type\-system\-boolean\)),
+[Attribute.IsDefined\(MemberInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-memberinfo\-system\-type\)),
+[Attribute.IsDefined\(MemberInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-memberinfo\-system\-type\-system\-boolean\)),
+[Attribute.IsDefined\(Module, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-module\-system\-type\)),
+[Attribute.IsDefined\(Module, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-module\-system\-type\-system\-boolean\)),
+[Attribute.IsDefined\(ParameterInfo, Type\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-parameterinfo\-system\-type\)),
+[Attribute.IsDefined\(ParameterInfo, Type, bool\)](https://learn.microsoft.com/dotnet/api/system.attribute.isdefined\#system\-attribute\-isdefined\(system\-reflection\-parameterinfo\-system\-type\-system\-boolean\)),
+[Attribute.Match\(object?\)](https://learn.microsoft.com/dotnet/api/system.attribute.match),
+[Attribute.TypeId](https://learn.microsoft.com/dotnet/api/system.attribute.typeid),
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Examples
+
+
// Tier 1: No annotation - property name matches external field name
+public record SimpleSchema(
+ int Id, // Looks for "Id" in external data
+ string Name // Looks for "Name" in external data
+) : IFlatSchema, ITextSerializable;
+
+// Tier 2: Explicit annotation - handle naming mismatches
+public record ShuttleSchema(
+ string Id, // Looks for "Id"
+ [SerializedLabel("shuttle_location")] // Looks for "shuttle_location"
+ string ShuttleLocation,
+
+ [SerializedLabel("d_check_complete")] // Looks for "d_check_complete"
+ bool DCheckComplete,
+
+ [SerializedLabel("company id")] // Handles space-separated names
+ int CompanyId
+) : IFlatSchema, ITextSerializable;
+
+## Remarks
+
+
+Purpose: Bridges C# property names with external data field names across all storage formats.
+
+
+Format Agnostic: Works uniformly across CSV, Excel, JSON, Parquet, and future formats.
+The serialization mechanism (CsvHelper, ExcelDataReader, System.Text.Json, etc.) is abstracted away.
+
+
+Opt-in Override: If not present, the property name is used as-is.
+Use this attribute when the external field name differs from the C# property name.
+
+
+Common Use Cases:
+
+
snake_case data sources: [SerializedLabel("company_id")] int CompanyId
kebab-case data sources: [SerializedLabel("company-id")] int CompanyId
Space-separated names: [SerializedLabel("Company ID")] int CompanyId
Legacy column names: [SerializedLabel("CMPNY_ID")] int CompanyId
+
+## Constructors
+
+### SerializedLabelAttribute\(string\)
+
+Initializes a new instance of the class.
+
+```csharp
+public SerializedLabelAttribute(string label)
+```
+
+#### Parameters
+
+`label` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+The external field name in the serialized data
+
+#### Exceptions
+
+ [ArgumentNullException](https://learn.microsoft.com/dotnet/api/system.argumentnullexception)
+
+Thrown when label is null
+
+ [ArgumentException](https://learn.microsoft.com/dotnet/api/system.argumentexception)
+
+Thrown when label is empty or whitespace
+
+## Properties
+
+### Label
+
+Gets the external field name for serialization.
+
+```csharp
+public string Label { get; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.md
new file mode 100644
index 00000000..92f38f24
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Abstractions.md
@@ -0,0 +1,53 @@
+# Namespace Flowthru.Abstractions
+
+### Classes
+
+ [FlowthruSchemaAttribute](Flowthru.Abstractions.FlowthruSchemaAttribute.md)
+
+Marks a schema type for automatic interface generation. The source generator
+will analyze the type's properties and emit the appropriate marker interfaces:
+
or based on property types
for flat schemas (CSV/TSV compatible)
for flat schemas (Parquet compatible)
for all schemas (JSON/XML compatible)
+
+ [SerializedEnumAttribute](Flowthru.Abstractions.SerializedEnumAttribute.md)
+
+Specifies the serialized string value for an enum member when written to or read from storage.
+
+ [SerializedLabelAttribute](Flowthru.Abstractions.SerializedLabelAttribute.md)
+
+Specifies the external field name for a property when serialized to/from storage.
+
+### Interfaces
+
+ [IBinarySerializable](Flowthru.Abstractions.IBinarySerializable.md)
+
+Marker interface for schema types that can be serialized to columnar binary formats (Parquet, Avro).
+
+ [IFlatSchema](Flowthru.Abstractions.IFlatSchema.md)
+
+Marker interface for schema types with flat (non-nested) structure.
+
+ [IFlatSerializable](Flowthru.Abstractions.IFlatSerializable.md)
+
+Marker interface for schema types that contain only flat, primitive data.
+
+ [INestedSchema](Flowthru.Abstractions.INestedSchema.md)
+
+Marker interface for schema types with nested structure (collections or nested objects).
+
+ [INestedSerializable](Flowthru.Abstractions.INestedSerializable.md)
+
+Marker interface for schema types that contain nested structures or collections.
+
+ [IScalar](Flowthru.Abstractions.IScalar.md)
+
+Marker interface for types that serialize to a single primitive value —
+i.e., they produce "key": value in JSON, not "key": {...} or "key": [...].
+
+ [IStructuredSerializable](Flowthru.Abstractions.IStructuredSerializable.md)
+
+Marker interface for schema types that can be serialized to structured formats (JSON, XML).
+
+ [ITextSerializable](Flowthru.Abstractions.ITextSerializable.md)
+
+Marker interface for schema types that can be serialized to text-based formats (CSV, TSV).
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.FlowthruCli.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.FlowthruCli.md
new file mode 100644
index 00000000..2b200f12
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.FlowthruCli.md
@@ -0,0 +1,130 @@
+# Class FlowthruCli
+
+Namespace: [Flowthru.Cli](Flowthru.Cli.md)
+Assembly: Flowthru.Core.dll
+
+Command-line interface wrapper for IFlowthruService.
+
+```csharp
+public sealed class FlowthruCli
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[FlowthruCli](Flowthru.Cli.FlowthruCli.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Remarks
+
+
+FlowthruCli provides a thin CLI layer over the core IFlowthruService.
+It handles:
+- Command-line argument parsing
+- Help/version display
+- Result formatting
+- Exit code generation
+
+
+The CLI delegates all business logic to IFlowthruService, making the
+service layer testable and reusable in non-CLI scenarios.
+
+This is the recommended entry point for standalone console applications using Flowthru.
+It manages the ServiceProvider lifecycle automatically, ensuring proper disposal of
+logging providers and other resources so the process exits cleanly after pipeline completion.
+
+
+For applications that integrate Flowthru into an existing DI container (e.g., ASP.NET Core),
+use the standard constructor and let the host application manage the ServiceProvider lifecycle.
+
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.md
new file mode 100644
index 00000000..cda960d0
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Cli.md
@@ -0,0 +1,8 @@
+# Namespace Flowthru.Cli
+
+### Classes
+
+ [FlowthruCli](Flowthru.Cli.FlowthruCli.md)
+
+Command-line interface wrapper for IFlowthruService.
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.CatalogOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.CatalogOptions.md
new file mode 100644
index 00000000..2070ff44
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.CatalogOptions.md
@@ -0,0 +1,88 @@
+# Class CatalogOptions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Configuration options for data catalog construction.
+
+```csharp
+public class CatalogOptions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[CatalogOptions](Flowthru.Configuration.CatalogOptions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Properties
+
+### BasePath
+
+Base path for dataset files (common constructor parameter).
+
+```csharp
+public string? BasePath { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+### ConnectionString
+
+Connection string for database catalogs (common constructor parameter).
+
+```csharp
+public string? ConnectionString { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+### ConstructorArgs
+
+Constructor arguments for the catalog (mapped to constructor parameters by name).
+
+```csharp
+public Dictionary ConstructorArgs { get; set; }
+```
+
+#### Property Value
+
+ [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [object](https://learn.microsoft.com/dotnet/api/system.object)\>
+
+### Environment
+
+Environment-specific catalog configuration (e.g., local vs. remote).
+
+```csharp
+public string? Environment { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+### Type
+
+The fully-qualified type name of the catalog class (e.g., "MyApp.Data.MyCatalog").
+
+```csharp
+public string? Type { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ConfigurationExtensions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ConfigurationExtensions.md
new file mode 100644
index 00000000..ed6be21f
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ConfigurationExtensions.md
@@ -0,0 +1,100 @@
+# Class ConfigurationExtensions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Extension methods for configuration-related operations.
+
+```csharp
+public static class ConfigurationExtensions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[ConfigurationExtensions](Flowthru.Configuration.ConfigurationExtensions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Methods
+
+### GetValidated\(IConfiguration, string\)
+
+Binds a configuration section to a strongly-typed object and validates it.
+
+```csharp
+public static T GetValidated(this IConfiguration configuration, string sectionPath) where T : new()
+```
+
+#### Parameters
+
+`configuration` [IConfiguration](https://learn.microsoft.com/dotnet/api/microsoft.extensions.configuration.iconfiguration)
+
+The configuration instance
+
+`sectionPath` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+The configuration section path (e.g., "DataScience:ModelParams")
+
+#### Returns
+
+ T
+
+The bound and validated object
+
+#### Type Parameters
+
+`T`
+
+The type to bind to
+
+#### Exceptions
+
+ [ValidationException](https://learn.microsoft.com/dotnet/api/system.componentmodel.dataannotations.validationexception)
+
+Thrown if DataAnnotations validation fails
+
+### GetValidatedOrDefault\(IConfiguration, string\)
+
+Attempts to bind and validate a configuration section, returning null if not found.
+
+```csharp
+public static T? GetValidatedOrDefault(this IConfiguration configuration, string sectionPath) where T : class, new()
+```
+
+#### Parameters
+
+`configuration` [IConfiguration](https://learn.microsoft.com/dotnet/api/microsoft.extensions.configuration.iconfiguration)
+
+The configuration instance
+
+`sectionPath` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+The configuration section path
+
+#### Returns
+
+ T?
+
+The bound and validated object, or null if section doesn't exist
+
+#### Type Parameters
+
+`T`
+
+The type to bind to
+
+#### Exceptions
+
+ [ValidationException](https://learn.microsoft.com/dotnet/api/system.componentmodel.dataannotations.validationexception)
+
+Thrown if DataAnnotations validation fails
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowOptions.md
new file mode 100644
index 00000000..20739fa7
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowOptions.md
@@ -0,0 +1,90 @@
+# Class FlowOptions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Configuration options for a single flow.
+
+```csharp
+public class FlowOptions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[FlowOptions](Flowthru.Configuration.FlowOptions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Properties
+
+### Description
+
+Human-readable description of the Flow.
+
+```csharp
+public string? Description { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+### FactoryMethod
+
+The name of the static factory method (default: "Create").
+
+```csharp
+public string FactoryMethod { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+### Parameters
+
+Flow-specific parameters (nested configuration section).
+The structure must match the Flow's parameter type.
+
+```csharp
+public Dictionary? Parameters { get; set; }
+```
+
+#### Property Value
+
+ [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [object](https://learn.microsoft.com/dotnet/api/system.object)\>?
+
+### Type
+
+The fully-qualified type name of the flow factory class.
+Must have a static Create method that accepts (catalog, parameters?).
+
+```csharp
+public string? Type { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+### Validation
+
+Validation configuration for this flow.
+
+```csharp
+public FlowValidationOptions? Validation { get; set; }
+```
+
+#### Property Value
+
+ [FlowValidationOptions](Flowthru.Configuration.FlowValidationOptions.md)?
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowValidationOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowValidationOptions.md
new file mode 100644
index 00000000..d69c5704
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowValidationOptions.md
@@ -0,0 +1,53 @@
+# Class FlowValidationOptions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Configuration options for flow validation behavior.
+
+```csharp
+public class FlowValidationOptions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[FlowValidationOptions](Flowthru.Configuration.FlowValidationOptions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Properties
+
+### DefaultInspectionLevel
+
+Default inspection level for all Layer 0 inputs.
+
+```csharp
+public string? DefaultInspectionLevel { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+### InspectionLevels
+
+Per-catalog-entry inspection level overrides.
+Key: catalog entry key, Value: inspection level (None, Shallow, Deep).
+
+```csharp
+public Dictionary InspectionLevels { get; set; }
+```
+
+#### Property Value
+
+ [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [string](https://learn.microsoft.com/dotnet/api/system.string)\>
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruConfigurationOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruConfigurationOptions.md
new file mode 100644
index 00000000..a42213ef
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruConfigurationOptions.md
@@ -0,0 +1,231 @@
+# Class FlowthruConfigurationOptions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Options for configuring how Flowthru loads configuration files.
+
+```csharp
+public class FlowthruConfigurationOptions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Remarks
+
+
+Flowthru uses Microsoft.Extensions.Configuration with layered configuration files.
+By default, configuration is loaded in the following order (later files override earlier):
+
+Both JSON and YAML formats are supported. YAML files follow the same pattern:
+appsettings.yml, appsettings.{Environment}.yml, appsettings.Local.yml
+
+
+## Properties
+
+### ConfigurationFileName
+
+The base filename for configuration files (without extension).
+
+```csharp
+public string ConfigurationFileName { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+#### Remarks
+
+Defaults to "appsettings". Change this to use a different naming convention
+(e.g., "parameters" to match Kedro's convention).
+
+### ConfigurationPath
+
+The base path where configuration files are located.
+
+```csharp
+public string ConfigurationPath { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+#### Remarks
+
+Defaults to the current directory. Can be set to "conf" for Kedro-style projects
+or any other directory containing configuration files.
+
+### EnableYamlSupport
+
+Whether to support YAML configuration files in addition to JSON.
+
+```csharp
+public bool EnableYamlSupport { get; set; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+#### Remarks
+
+When enabled, Flowthru will load both .json and .yml/.yaml files.
+Requires NetEscapades.Configuration.Yaml package.
+Defaults to true for Kedro compatibility.
+
+### Environment
+
+The environment name used to load environment-specific configuration files.
+
+```csharp
+public string? Environment { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+#### Remarks
+
+
+If not explicitly set, Flowthru will attempt to resolve the environment in this order:
+
+
Value passed to WithEnvironment()
Environment variable specified by
DOTNET_ENVIRONMENT environment variable
ASPNETCORE_ENVIRONMENT environment variable
"Production" (default)
+
+### EnvironmentVariable
+
+The name of the environment variable to check for environment name.
+
+```csharp
+public string? EnvironmentVariable { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+#### Remarks
+
+Defaults to "FLOWTHRU_ENV". Set to null to disable environment variable resolution.
+Standard .NET environment variables (DOTNET_ENVIRONMENT, ASPNETCORE_ENVIRONMENT)
+are always checked as fallbacks.
+
+## Methods
+
+### WithConfigurationFileName\(string\)
+
+Sets the base filename for configuration files (without extension).
+
+```csharp
+public FlowthruConfigurationOptions WithConfigurationFileName(string fileName)
+```
+
+#### Parameters
+
+`fileName` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+The base filename (e.g., "parameters", "config")
+
+#### Returns
+
+ [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md)
+
+This options instance for fluent chaining
+
+### WithConfigurationPath\(string\)
+
+Sets the base path where configuration files are located.
+
+```csharp
+public FlowthruConfigurationOptions WithConfigurationPath(string path)
+```
+
+#### Parameters
+
+`path` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+The configuration directory path
+
+#### Returns
+
+ [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md)
+
+This options instance for fluent chaining
+
+### WithEnvironment\(string\)
+
+Sets the environment name explicitly.
+
+```csharp
+public FlowthruConfigurationOptions WithEnvironment(string environment)
+```
+
+#### Parameters
+
+`environment` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+The environment name (e.g., "Development", "Production")
+
+#### Returns
+
+ [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md)
+
+This options instance for fluent chaining
+
+### WithEnvironmentVariable\(string?\)
+
+Sets the environment variable name to check for environment resolution.
+
+```csharp
+public FlowthruConfigurationOptions WithEnvironmentVariable(string? variableName)
+```
+
+#### Parameters
+
+`variableName` [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+The environment variable name
+
+#### Returns
+
+ [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md)
+
+This options instance for fluent chaining
+
+### WithYamlSupport\(bool\)
+
+Enables or disables YAML configuration file support.
+
+```csharp
+public FlowthruConfigurationOptions WithYamlSupport(bool enabled = true)
+```
+
+#### Parameters
+
+`enabled` [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+Whether to enable YAML support
+
+#### Returns
+
+ [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md)
+
+This options instance for fluent chaining
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruOptions.md
new file mode 100644
index 00000000..94e0a292
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.FlowthruOptions.md
@@ -0,0 +1,95 @@
+# Class FlowthruOptions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Root configuration options for Flowthru applications.
+
+```csharp
+public class FlowthruOptions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[FlowthruOptions](Flowthru.Configuration.FlowthruOptions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Remarks
+
+This class represents the top-level "Flowthru" section in configuration files.
+All Flowthru-specific configuration should be nested under this section.
+
+## Fields
+
+### SectionName
+
+Configuration section name in appsettings.json.
+
+```csharp
+public const string SectionName = "Flowthru"
+```
+
+#### Field Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+## Properties
+
+### Catalog
+
+Data catalog configuration.
+
+```csharp
+public CatalogOptions Catalog { get; set; }
+```
+
+#### Property Value
+
+ [CatalogOptions](Flowthru.Configuration.CatalogOptions.md)
+
+### Flows
+
+Flow registration and configuration.
+
+```csharp
+public Dictionary Flows { get; set; }
+```
+
+#### Property Value
+
+ [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [FlowOptions](Flowthru.Configuration.FlowOptions.md)\>
+
+### Logging
+
+Logging configuration (extends standard .NET logging configuration).
+
+```csharp
+public LoggingOptions? Logging { get; set; }
+```
+
+#### Property Value
+
+ [LoggingOptions](Flowthru.Configuration.LoggingOptions.md)?
+
+### Metadata
+
+Metadata collection and export configuration.
+
+```csharp
+public MetadataOptions Metadata { get; set; }
+```
+
+#### Property Value
+
+ [MetadataOptions](Flowthru.Configuration.MetadataOptions.md)
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ICatalogFactory.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ICatalogFactory.md
new file mode 100644
index 00000000..f8d7085b
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.ICatalogFactory.md
@@ -0,0 +1,43 @@
+# Interface ICatalogFactory
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Factory interface for creating data catalog instances from configuration.
+
+```csharp
+public interface ICatalogFactory
+```
+
+## Remarks
+
+Implement this interface to enable configuration-based catalog construction.
+The factory receives the full configuration and can use it to construct
+environment-specific catalogs (e.g., local files in dev, remote DB in prod).
+
+## Methods
+
+### CreateCatalog\(CatalogOptions, IServiceProvider\)
+
+Creates a catalog instance based on configuration.
+
+```csharp
+CatalogAbstract CreateCatalog(CatalogOptions options, IServiceProvider serviceProvider)
+```
+
+#### Parameters
+
+`options` [CatalogOptions](Flowthru.Configuration.CatalogOptions.md)
+
+Catalog configuration options
+
+`serviceProvider` [IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider)
+
+Service provider for dependency injection
+
+#### Returns
+
+ [CatalogAbstract](Flowthru.Data.CatalogAbstract.md)
+
+The configured catalog instance
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.JsonMetadataOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.JsonMetadataOptions.md
new file mode 100644
index 00000000..410b24c8
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.JsonMetadataOptions.md
@@ -0,0 +1,52 @@
+# Class JsonMetadataOptions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Configuration options for JSON metadata export.
+
+```csharp
+public class JsonMetadataOptions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[JsonMetadataOptions](Flowthru.Configuration.JsonMetadataOptions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Properties
+
+### IncludeTypeInfo
+
+Whether to include full type information in the export.
+
+```csharp
+public bool IncludeTypeInfo { get; set; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+### UseCompactFormat
+
+Whether to use compact (minified) JSON format.
+
+```csharp
+public bool UseCompactFormat { get; set; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.LoggingOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.LoggingOptions.md
new file mode 100644
index 00000000..8cc85225
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.LoggingOptions.md
@@ -0,0 +1,64 @@
+# Class LoggingOptions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Logging configuration options (extends standard .NET logging).
+
+```csharp
+public class LoggingOptions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[LoggingOptions](Flowthru.Configuration.LoggingOptions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Properties
+
+### EnableConsole
+
+Whether console logging is enabled.
+
+```csharp
+public bool EnableConsole { get; set; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+### LogLevel
+
+Per-category log level overrides.
+
+```csharp
+public Dictionary LogLevel { get; set; }
+```
+
+#### Property Value
+
+ [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [string](https://learn.microsoft.com/dotnet/api/system.string)\>
+
+### MinimumLevel
+
+Minimum log level (Trace, Debug, Information, Warning, Error, Critical).
+
+```csharp
+public string MinimumLevel { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MermaidMetadataOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MermaidMetadataOptions.md
new file mode 100644
index 00000000..9cd9b6e8
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MermaidMetadataOptions.md
@@ -0,0 +1,98 @@
+# Class MermaidMetadataOptions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Configuration options for Mermaid diagram export.
+
+```csharp
+public class MermaidMetadataOptions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[MermaidMetadataOptions](Flowthru.Configuration.MermaidMetadataOptions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Properties
+
+### ActiveDataColor
+
+Hex color code for active (sliced) catalog entries.
+
+```csharp
+public string ActiveDataColor { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+#### Remarks
+
+Color applied to data catalog entries produced by sliced nodes.
+Default: #2E7D32 (Material Design green-800).
+
+### ActiveStepColor
+
+Hex color code for active (sliced) nodes.
+
+```csharp
+public string ActiveStepColor { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+#### Remarks
+
+Color applied to nodes that are in the execution slice.
+Default: #2E7D32 (Material Design green-800).
+
+### Direction
+
+Flowchart direction (TopToBottom, LeftToRight, etc.).
+
+```csharp
+public string Direction { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+### ShowDatasetDetails
+
+Whether to include dataset details in nodes.
+
+```csharp
+public bool ShowDatasetDetails { get; set; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+### ShowParameters
+
+Whether to include parameter information in nodes.
+
+```csharp
+public bool ShowParameters { get; set; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MetadataOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MetadataOptions.md
new file mode 100644
index 00000000..1769439b
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.MetadataOptions.md
@@ -0,0 +1,130 @@
+# Class MetadataOptions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Configuration options for metadata collection and export.
+
+```csharp
+public class MetadataOptions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[MetadataOptions](Flowthru.Configuration.MetadataOptions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Properties
+
+### Enabled
+
+Whether metadata collection is enabled.
+
+```csharp
+public bool Enabled { get; set; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+### FilenameTemplate
+
+Filename template for metadata exports.
+
+```csharp
+public string FilenameTemplate { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+#### Remarks
+
+
+Supports dynamic tokens that are replaced during export:
+
+
{FlowName} - Sanitized flow name
{Timestamp} - Formatted timestamp (empty if disabled in Timestamp.IncludeTimestamp)
{SliceType} - "FromNodes", "Tags", "Mixed", or empty if unsliced
{FromNodes} - Comma-separated list of from-nodes
{ToNodes} - Comma-separated list of to-nodes
{FromInputs} - Comma-separated list of from-inputs
{OnlyNodes} - Comma-separated list of only-nodes
{Tags} - Comma-separated list of tags
+
+Empty tokens are automatically collapsed to prevent double-separators.
+File extensions are added by individual providers (.json, .md, etc.).
+
+
+### Json
+
+Configuration specific to the JSON metadata provider.
+
+```csharp
+public JsonMetadataOptions? Json { get; set; }
+```
+
+#### Property Value
+
+ [JsonMetadataOptions](Flowthru.Configuration.JsonMetadataOptions.md)?
+
+### Mermaid
+
+Configuration specific to the Mermaid metadata provider.
+
+```csharp
+public MermaidMetadataOptions? Mermaid { get; set; }
+```
+
+#### Property Value
+
+ [MermaidMetadataOptions](Flowthru.Configuration.MermaidMetadataOptions.md)?
+
+### OutputDirectory
+
+Directory where metadata files will be written.
+
+```csharp
+public string OutputDirectory { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+### Providers
+
+List of metadata providers to enable (e.g., "Json", "Mermaid", "Csv").
+
+```csharp
+public List Providers { get; set; }
+```
+
+#### Property Value
+
+ [List](https://learn.microsoft.com/dotnet/api/system.collections.generic.list\-1)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>
+
+### Timestamp
+
+Configuration for timestamp generation in metadata filenames.
+
+```csharp
+public TimestampConfiguration Timestamp { get; set; }
+```
+
+#### Property Value
+
+ [TimestampConfiguration](Flowthru.Configuration.TimestampConfiguration.md)
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineOptions.md
new file mode 100644
index 00000000..b04d7da1
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineOptions.md
@@ -0,0 +1,90 @@
+# Class PipelineOptions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Configuration options for a single pipeline.
+
+```csharp
+public class PipelineOptions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[PipelineOptions](Flowthru.Configuration.PipelineOptions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Properties
+
+### Description
+
+Human-readable description of the pipeline.
+
+```csharp
+public string? Description { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+### FactoryMethod
+
+The name of the static factory method (default: "Create").
+
+```csharp
+public string FactoryMethod { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+### Parameters
+
+Pipeline-specific parameters (nested configuration section).
+The structure must match the pipeline's parameter type.
+
+```csharp
+public Dictionary? Parameters { get; set; }
+```
+
+#### Property Value
+
+ [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [object](https://learn.microsoft.com/dotnet/api/system.object)\>?
+
+### Type
+
+The fully-qualified type name of the pipeline factory class.
+Must have a static Create method that accepts (catalog, parameters?).
+
+```csharp
+public string? Type { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+### Validation
+
+Validation configuration for this pipeline.
+
+```csharp
+public PipelineValidationOptions? Validation { get; set; }
+```
+
+#### Property Value
+
+ [PipelineValidationOptions](Flowthru.Configuration.PipelineValidationOptions.md)?
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineValidationOptions.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineValidationOptions.md
new file mode 100644
index 00000000..cd4c4980
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.PipelineValidationOptions.md
@@ -0,0 +1,53 @@
+# Class PipelineValidationOptions
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Configuration options for pipeline validation behavior.
+
+```csharp
+public class PipelineValidationOptions
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[PipelineValidationOptions](Flowthru.Configuration.PipelineValidationOptions.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Properties
+
+### DefaultInspectionLevel
+
+Default inspection level for all Layer 0 inputs.
+
+```csharp
+public string? DefaultInspectionLevel { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+### InspectionLevels
+
+Per-catalog-entry inspection level overrides.
+Key: catalog entry key, Value: inspection level (None, Shallow, Deep).
+
+```csharp
+public Dictionary InspectionLevels { get; set; }
+```
+
+#### Property Value
+
+ [Dictionary](https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary\-2)<[string](https://learn.microsoft.com/dotnet/api/system.string), [string](https://learn.microsoft.com/dotnet/api/system.string)\>
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.TimestampConfiguration.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.TimestampConfiguration.md
new file mode 100644
index 00000000..27d406e4
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.TimestampConfiguration.md
@@ -0,0 +1,64 @@
+# Class TimestampConfiguration
+
+Namespace: [Flowthru.Configuration](Flowthru.Configuration.md)
+Assembly: Flowthru.Core.dll
+
+Configuration for timestamp generation in metadata filenames.
+
+```csharp
+public class TimestampConfiguration
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[TimestampConfiguration](Flowthru.Configuration.TimestampConfiguration.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Properties
+
+### Format
+
+Timestamp format string (see .NET DateTime formatting).
+
+```csharp
+public string Format { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+### IncludeTimestamp
+
+Whether to include a timestamp in the filename.
+
+```csharp
+public bool IncludeTimestamp { get; set; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+### TimeZone
+
+Time zone for the timestamp (e.g., "UTC", "Local").
+
+```csharp
+public string TimeZone { get; set; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.md
new file mode 100644
index 00000000..b8cd4260
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Configuration.md
@@ -0,0 +1,54 @@
+# Namespace Flowthru.Configuration
+
+### Classes
+
+ [CatalogOptions](Flowthru.Configuration.CatalogOptions.md)
+
+Configuration options for data catalog construction.
+
+ [ConfigurationExtensions](Flowthru.Configuration.ConfigurationExtensions.md)
+
+Extension methods for configuration-related operations.
+
+ [FlowOptions](Flowthru.Configuration.FlowOptions.md)
+
+Configuration options for a single flow.
+
+ [FlowValidationOptions](Flowthru.Configuration.FlowValidationOptions.md)
+
+Configuration options for flow validation behavior.
+
+ [FlowthruConfigurationOptions](Flowthru.Configuration.FlowthruConfigurationOptions.md)
+
+Options for configuring how Flowthru loads configuration files.
+
+ [FlowthruOptions](Flowthru.Configuration.FlowthruOptions.md)
+
+Root configuration options for Flowthru applications.
+
+ [JsonMetadataOptions](Flowthru.Configuration.JsonMetadataOptions.md)
+
+Configuration options for JSON metadata export.
+
+ [LoggingOptions](Flowthru.Configuration.LoggingOptions.md)
+
+Logging configuration options (extends standard .NET logging).
+
+ [MermaidMetadataOptions](Flowthru.Configuration.MermaidMetadataOptions.md)
+
+Configuration options for Mermaid diagram export.
+
+ [MetadataOptions](Flowthru.Configuration.MetadataOptions.md)
+
+Configuration options for metadata collection and export.
+
+ [TimestampConfiguration](Flowthru.Configuration.TimestampConfiguration.md)
+
+Configuration for timestamp generation in metadata filenames.
+
+### Interfaces
+
+ [ICatalogFactory](Flowthru.Configuration.ICatalogFactory.md)
+
+Factory interface for creating data catalog instances from configuration.
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.StorageTraits.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.StorageTraits.md
new file mode 100644
index 00000000..39753a89
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.StorageTraits.md
@@ -0,0 +1,224 @@
+# Class StorageTraits
+
+Namespace: [Flowthru.Data.Capabilities](Flowthru.Data.Capabilities.md)
+Assembly: Flowthru.Core.dll
+
+Describes the structural constraints and capabilities of a storage implementation.
+Defaults represent filesystem-file baseline behavior.
+
+```csharp
+public record StorageTraits : IEquatable
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md)
+
+#### Implements
+
+[IEquatable](https://learn.microsoft.com/dotnet/api/system.iequatable\-1)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Remarks
+
+
+Design Philosophy:
+
+
+A filesystem file is the median storage mechanism — the "zero" from which we measure deviations.
+A constraint narrows from this baseline (e.g., read-only, non-persistent, non-inspectable).
+A capability widens beyond it (e.g., streamable, appendable, transactional).
+
+
+Constraint Examples:
+
+
Read-only sources: HTTP GET endpoints, Excel files, database views — set CanWrite = false
Transactional: Database writes, ACID-compliant stores — set IsTransactional = true
+
+Two-Level Constraint Model:
+
+
+Traits are declared at two levels:
+
+
Adapter level: The adapter author declares what the storage medium intrinsically supports.
+These are structural truths (e.g., an HTTP GET endpoint cannot write).
Catalog level: The pipeline author can further constrain an adapter using Item.Constrain().
+Constraints can only tighten, never loosen (one-way ratchet).
public IItem<IEnumerable<Company>> ReferenceData =>
+ GetOrCreateEntry(() => Items.Enumerable.Csv<Company>(
+ "ref_data", $"{_basePath}/reference.csv")
+ .Constrain(t => t with { CanWrite = false }));
+
+## Properties
+
+### CanAppend
+
+Can data be appended without replacing existing data?
+
+```csharp
+public bool CanAppend { get; init; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+#### Remarks
+
+Default: false (filesystem file writes typically overwrite).
+Set to true for append-only logs, Spark SaveMode.Append, incremental tables.
+
+### CanInspect
+
+Can the source be inspected for pre-flight validation?
+
+```csharp
+public bool CanInspect { get; init; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+#### Remarks
+
+Default: true (filesystem files can be sampled).
+Set to false for sources that are expensive to validate (remote HTTP, distributed Spark).
+
+### CanRead
+
+Can data be read from this source?
+
+```csharp
+public bool CanRead { get; init; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+#### Remarks
+
+Default: true (filesystem files are readable).
+Set to false for write-only sinks (logging endpoints, audit tables).
+
+### CanStream
+
+Can data be lazily streamed without full materialization?
+
+```csharp
+public bool CanStream { get; init; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+#### Remarks
+
+Default: false (filesystem file I/O typically buffers).
+Set to true for CSV streaming, database cursors, Parquet row groups.
+Enables memory-efficient processing of large datasets.
+
+### CanWrite
+
+Can data be written to this source?
+
+```csharp
+public bool CanWrite { get; init; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+#### Remarks
+
+Default: true (filesystem files are writable).
+Set to false for read-only sources (HTTP GET, Excel files, database views).
+
+### IsPersistent
+
+Does data survive across pipeline runs?
+
+```csharp
+public bool IsPersistent { get; init; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+#### Remarks
+
+Default: true (filesystem files persist).
+Set to false for in-memory caches, temporary buffers, or transient state.
+
+### IsTransactional
+
+Are writes atomic (all-or-nothing)?
+
+```csharp
+public bool IsTransactional { get; init; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+#### Remarks
+
+Default: false (filesystem file writes are not ACID).
+Set to true for database transactions, ACID-compliant stores.
+
+### RequiresNetwork
+
+Does this storage require network access?
+
+```csharp
+public bool RequiresNetwork { get; init; }
+```
+
+#### Property Value
+
+ [bool](https://learn.microsoft.com/dotnet/api/system.boolean)
+
+#### Remarks
+
+Default: false (filesystem files are local).
+Set to true for remote databases, S3, HTTP endpoints.
+Used for pre-flight validation in offline/CI environments.
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.md
new file mode 100644
index 00000000..be13d989
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.Capabilities.md
@@ -0,0 +1,9 @@
+# Namespace Flowthru.Data.Capabilities
+
+### Classes
+
+ [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md)
+
+Describes the structural constraints and capabilities of a storage implementation.
+Defaults represent filesystem-file baseline behavior.
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogAbstract.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogAbstract.md
new file mode 100644
index 00000000..94b36f3d
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogAbstract.md
@@ -0,0 +1,262 @@
+# Class CatalogAbstract
+
+Namespace: [Flowthru.Data](Flowthru.Data.md)
+Assembly: Flowthru.Core.dll
+
+Base class for strongly-typed catalog implementations with automatic property caching.
+
+```csharp
+public abstract class CatalogAbstract
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[CatalogAbstract](Flowthru.Data.CatalogAbstract.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Remarks
+
+
+Problem Solved:
+Expression-bodied properties (Property => new Item()) create new instances on each access,
+breaking DAG dependency resolution which relies on object identity.
+
+
+Solution:
+Uses reflection to:
+1. Discover all IItem properties on derived classes
+2. Create backing fields to cache instances
+3. Intercept property getters to return cached instances
+
+
+Usage Pattern:
+
public class MyCatalog : DataCatalogBase
+{
+ public MyCatalog(string basePath = "Data") : base()
+ {
+ BasePath = basePath;
+ InitializeCatalogProperties();
+ }
+
+ protected string BasePath { get; }
+
+ // Declare once - automatically cached!
+ public IItem<IEnumerable<MyData>> MyData =>
+ CreateItem(() => new CsvCatalogItem<MyData>("my_data", $"{BasePath}/data.csv"));
+}
+
+
+Key Benefits:
+- Declare catalog items ONCE (no redundant constructor code)
+- Automatic instance caching (object identity preserved)
+- Type-safe (compile-time checks)
+- Zero runtime overhead after first access (cached delegates)
+
+
+## Constructors
+
+### CatalogAbstract\(string?\)
+
+```csharp
+protected CatalogAbstract(string? catalogLabel = null)
+```
+
+#### Parameters
+
+`catalogLabel` [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+Optional display label for this catalog instance. When omitted, defaults to the
+concrete class name via GetType().Name.
+
+## Properties
+
+### CatalogLabel
+
+The display label used to identify this catalog instance in flow metadata.
+Defaults to the concrete class name when not specified.
+
+```csharp
+public string CatalogLabel { get; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+#### Remarks
+
+Pass an explicit label when constructing multiple instances of the same catalog type
+in a single flow (e.g., per-partition or per-shard catalogs) so their items
+receive distinct qualified identifiers in the DAG: CatalogLabel.ItemLabel.
+
+### Services
+
+Optional service provider for dependency injection into catalog items.
+
+```csharp
+public IServiceProvider? Services { get; set; }
+```
+
+#### Property Value
+
+ [IServiceProvider](https://learn.microsoft.com/dotnet/api/system.iserviceprovider)?
+
+#### Remarks
+
+Set by the service layer before flow execution to enable catalog
+items to resolve services (e.g., database connections, HTTP clients).
+
+## Methods
+
+### ClearCache\(\)
+
+Clears the property cache. Use with caution!
+
+```csharp
+protected void ClearCache()
+```
+
+#### Remarks
+
+
+Warning: Clearing the cache after flow construction will break
+DAG dependencies since new instances will be created on next access.
+
+
+Use Case: Primarily for testing scenarios where you need to reset
+catalog state between test runs.
+
+
+### CreateItem\(Func\>, string\)
+
+Gets or creates a unified catalog item, caching it for subsequent accesses.
+
+```csharp
+protected IItem CreateItem(Func> factory, string propertyName = "")
+```
+
+#### Parameters
+
+`factory` [Func](https://learn.microsoft.com/dotnet/api/system.func\-1)<[IItem](Flowthru.Data.IItem\-1.md)\>
+
+Factory function to create the item on first access
+
+`propertyName` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+Auto-populated by compiler with calling property name
+
+#### Returns
+
+ [IItem](Flowthru.Data.IItem\-1.md)
+
+Cached catalog item instance
+
+#### Type Parameters
+
+`T`
+
+The data type stored in this catalog item.
+For singletons: Use T directly (e.g., LinearRegressionModel)
+For collections: Use IEnumerable<T> (e.g., IEnumerable<FeatureRow>)
+
+#### Remarks
+
+
+Unified API (v0.5.0): This single method replaces GetOrCreateObject
+and GetOrCreateDataset. Cardinality is determined by the type parameter T.
+
+
+### Text\(string, string\)
+
+Creates a plain text file catalog entry.
+
+```csharp
+public static Item Text(string label, string filePath)
+```
+
+#### Parameters
+
+`label` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+Unique catalog label for DAG resolution
+
+`filePath` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+Path to text file (.txt, .md, etc.)
+
+#### Returns
+
+ [Item](Flowthru.Data.Item\-1.md)<[string](https://learn.microsoft.com/dotnet/api/system.string)\>
+
+Catalog entry for text file with string content
+
+#### Remarks
+
+
+Use Case: Markdown reports, plain text logs, configuration files
+
+
+Implementation: Reads entire file as single string.
+
+
+Storage Traits: All traits use filesystem baseline defaults
+
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntries.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntries.md
new file mode 100644
index 00000000..a8cd5d7c
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntries.md
@@ -0,0 +1,136 @@
+# Class Items
+
+Namespace: [Flowthru.Data](Flowthru.Data.md)
+Assembly: Flowthru.Core.dll
+
+Static factory methods for creating catalog entries with common configurations.
+
+```csharp
+public static class Items
+```
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[Items](Flowthru.Data.Items.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.MemberwiseClone\(\)](https://learn.microsoft.com/dotnet/api/system.object.memberwiseclone),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Examples
+
+
// Tier 1: No annotations - property names match external field names
+public record SimpleSchema(
+ int Id, // Looks for "Id" in CSV/Excel/JSON
+ string Name // Looks for "Name" in CSV/Excel/JSON
+) : IFlatSchema, ITextSerializable;
+
+var simple = Items.Enumerable.Csv<SimpleSchema>("data", "data.csv");
+
+// Tier 2: Explicit annotations - handle naming mismatches
+public record ShuttleSchema(
+ [SerializedLabel("id")]
+ string Id,
+
+ [SerializedLabel("shuttle_location")] // snake_case in file
+ string ShuttleLocation,
+
+ [SerializedLabel("d_check_complete")] // snake_case in file
+ bool DCheckComplete,
+
+ [SerializedLabel("Company ID")] // space-separated in file
+ int CompanyId
+) : IFlatSchema, ITextSerializable;
+
+var shuttles = Items.Enumerable.Excel<ShuttleSchema>(
+ "shuttles",
+ "data/shuttles.xlsx",
+ "Sheet1"
+);
+
+// Same schema works across all formats
+var csv = Items.Enumerable.Csv<ShuttleSchema>("shuttles", "data/shuttles.csv");
+var json = Items.Enumerable.Json<ShuttleSchema>("shuttles", "data/shuttles.json");
+
+## Remarks
+
+
+Design Pattern: Static factory methods that compose storage adapters
+from medium + format + container layers.
+
+
+Discoverability: All factory methods are in one place with IntelliSense support.
+
+
+Type Safety: Generic constraints enforce schema compatibility at compile-time.
+
+
+Field Name Mapping with SerializedLabel:
+
+
+Use the SerializedLabelAttribute to map C# property names to external field names
+when they differ. This works uniformly across CSV, Excel, JSON, and most formats.
+
+
+## Properties
+
+### Enumerable
+
+Factory methods for catalog entries.
+
+```csharp
+public static EnumerableItems Enumerable { get; }
+```
+
+#### Property Value
+
+ [EnumerableItems](Flowthru.Data.EnumerableItems.md)
+
+## Methods
+
+### Null\(string\)
+
+Creates a null catalog entry for side-effect-only nodes.
+
+```csharp
+public static Item Null(string label)
+```
+
+#### Parameters
+
+`label` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+Unique catalog label for DAG resolution
+
+#### Returns
+
+ [Item](Flowthru.Data.Item\-1.md)
+
+Catalog entry for void/no-data semantics
+
+#### Type Parameters
+
+`T`
+
+The data type (typically NoData)
+
+#### Remarks
+
+
+Use Case: Steps that perform side effects (logging, visualization) without producing meaningful data
+
+
+Implementation: Uses NullStorageAdapter which performs no I/O operations.
+
+
diff --git a/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntry-1.md b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntry-1.md
new file mode 100644
index 00000000..4c3309d5
--- /dev/null
+++ b/docs/reference/src/core/Flowthru.Core/Flowthru.Data.CatalogEntry-1.md
@@ -0,0 +1,347 @@
+# Class Item
+
+Namespace: [Flowthru.Data](Flowthru.Data.md)
+Assembly: Flowthru.Core.dll
+
+Standard catalog entry implementation that delegates to a storage adapter.
+
+```csharp
+public sealed class Item : IItem, IItem
+```
+
+#### Type Parameters
+
+`T`
+
+The data type (container with rows)
+
+#### Inheritance
+
+[object](https://learn.microsoft.com/dotnet/api/system.object) ←
+[Item](Flowthru.Data.Item\-1.md)
+
+#### Implements
+
+[IItem](Flowthru.Data.IItem\-1.md),
+[IItem](Flowthru.Data.IItem.md)
+
+#### Inherited Members
+
+[object.Equals\(object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\)),
+[object.Equals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.equals\#system\-object\-equals\(system\-object\-system\-object\)),
+[object.GetHashCode\(\)](https://learn.microsoft.com/dotnet/api/system.object.gethashcode),
+[object.GetType\(\)](https://learn.microsoft.com/dotnet/api/system.object.gettype),
+[object.ReferenceEquals\(object?, object?\)](https://learn.microsoft.com/dotnet/api/system.object.referenceequals),
+[object.ToString\(\)](https://learn.microsoft.com/dotnet/api/system.object.tostring)
+
+## Remarks
+
+
+Delegation Pattern:
+
+
+This class is a thin wrapper that delegates all operations to an .
+The storage adapter handles the actual I/O logic, while this class provides:
+- IItem interface implementation
+- Identity for DAG dependency resolution (via Key)
+- Type erasure for pipeline heterogeneous collections
+
+
+Construction:
+
+
+Typically created via static factory methods in :
+
+
var entry = Items.Csv<CompanySchema>("companies", "data.csv");
+// Returns: IItem<IEnumerable<CompanySchema>>
+
+Composition vs Inheritance:
+
+
+Previous design: Inheritance hierarchy (CsvItem, JsonItem, etc.)
+New design: Single class + composed storage adapter
+
+
+Benefits:
+- No class explosion for format × container combinations
+- Custom storage via IStorageAdapter implementation
+- Clear separation of concerns
+
+
+Capability Forwarding:
+
+
+The underlying storage adapter provides inspection methods, which this catalog
+entry automatically forwards. All storage adapters are required to implement inspection.
+
+
+## Constructors
+
+### Item\(string, IStorageAdapter\)
+
+Creates a new catalog entry with the specified key and storage adapter.
+
+```csharp
+public Item(string label, IStorageAdapter storage)
+```
+
+#### Parameters
+
+`label` [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+Unique identifier for this catalog entry
+
+`storage` [IStorageAdapter](Flowthru.Data.Storage.IStorageAdapter\-1.md)
+
+Storage adapter that handles I/O operations
+
+## Properties
+
+### DataType
+
+The runtime type of data stored in this catalog entry.
+For singletons: Returns typeof(T).
+For collections: Returns typeof(IEnumerable<T>).
+
+```csharp
+public Type DataType { get; }
+```
+
+#### Property Value
+
+ [Type](https://learn.microsoft.com/dotnet/api/system.type)
+
+### Label
+
+Unique label identifying this catalog entry within the data catalog.
+
+```csharp
+public string Label { get; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)
+
+### OwningCatalogLabel
+
+The label of the -derived class that created
+this entry. Set automatically by GetOrCreateEntry; null for entries created outside
+a catalog or by custom implementations.
+
+```csharp
+public string? OwningCatalogLabel { get; }
+```
+
+#### Property Value
+
+ [string](https://learn.microsoft.com/dotnet/api/system.string)?
+
+#### Remarks
+
+Used by the metadata layer to produce fully-qualified entry identifiers in the form
+CatalogLabel.EntryLabel. First-write-wins: cross-catalog shared entries retain
+the label of the catalog that originally created them.
+
+### PreferredInspectionLevel
+
+Gets the preferred inspection level for this catalog entry.
+
+```csharp
+public InspectionLevel? PreferredInspectionLevel { get; }
+```
+
+#### Property Value
+
+ [InspectionLevel](Flowthru.Data.Validation.InspectionLevel.md)?
+
+### Traits
+
+```csharp
+public StorageTraits Traits { get; }
+```
+
+#### Property Value
+
+ [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md)
+
+## Methods
+
+### Constrain\(Func\)
+
+```csharp
+public Item Constrain(Func constraintFn)
+```
+
+#### Parameters
+
+`constraintFn` [Func](https://learn.microsoft.com/dotnet/api/system.func\-2)<[StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md), [StorageTraits](Flowthru.Data.Capabilities.StorageTraits.md)\>
+
+#### Returns
+
+ [Item](Flowthru.Data.Item\-1.md)
+
+### Exists\(\)
+
+Checks if data exists at this catalog entry location.
+Returns an effect that can fail.
+
+```csharp
+public FlowIO Exists()
+```
+
+#### Returns
+
+ [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[bool](https://learn.microsoft.com/dotnet/api/system.boolean)\>
+
+### GetCountAsync\(\)
+
+Gets the count of items in this catalog entry.
+For collections (IEnumerable<T>), returns the enumerable count.
+For singletons, returns 1 if exists, 0 otherwise.
+
+```csharp
+public FlowIO GetCountAsync()
+```
+
+#### Returns
+
+ [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[int](https://learn.microsoft.com/dotnet/api/system.int32)\>
+
+### InspectDeep\(\)
+
+Performs deep validation of this catalog entry.
+
+```csharp
+public FlowIO InspectDeep()
+```
+
+#### Returns
+
+ [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\>
+
+Effect producing validation result
+
+#### Remarks
+
+Forwards the call directly to the underlying storage adapter.
+All storage adapters must implement inspection.
+
+### InspectShallow\(int\)
+
+Performs shallow validation of this catalog entry.
+
+```csharp
+public FlowIO InspectShallow(int sampleSize = 100)
+```
+
+#### Parameters
+
+`sampleSize` [int](https://learn.microsoft.com/dotnet/api/system.int32)
+
+Number of rows/records to sample for validation
+
+#### Returns
+
+ [FlowIO](Flowthru.Effects.FlowIO\-1.md)<[ValidationResult](Flowthru.Data.Validation.ValidationResult.md)\>
+
+Effect producing validation result
+
+#### Remarks
+
+Forwards the call directly to the underlying storage adapter.
+All storage adapters must implement inspection.
+
+### Load\(\)
+
+Load data as an effect (can fail, is async, can be cancelled).
+Returns T directly, which may itself be an IEnumerable or Seq.
+
+```csharp
+public FlowIO Load()
+```
+
+#### Returns
+
+ [FlowIO](Flowthru.Effects.FlowIO\-1.md)
+
+### LoadUntyped\(\)
+
+Loads data from the catalog entry as an untyped object.
+Returns an effect that can fail.
+The returned type matches the DataType property.
+
+```csharp
+public FlowIO