Skip to content

Commit eea4da3

Browse files
SpelkingtonSpelkington
committed
feat: GQL example
1 parent 10e2711 commit eea4da3

62 files changed

Lines changed: 29892 additions & 1544 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Flowthru.slnx‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
<Solution>
2+
<Folder Name="/examples/advanced/">
3+
<Project Path="examples/advanced/KedroSpaceflightsGQL/KedroSpaceflightsGQL.csproj" />
4+
</Folder>
25
<Folder Name="/src/">
36
<Project Path="src\core\Flowthru\Flowthru.csproj" />
47
<Project Path="src\core\Flowthru.Core\Flowthru.Core.csproj" />
@@ -17,6 +20,8 @@
1720
<Project Path="src\extensions\Flowthru.Extensions.Python.SourceGenerators\Flowthru.Extensions.Python.SourceGenerators.csproj" />
1821
<Project Path="src\misc\Flowthru.Misc.ML\Flowthru.Misc.ML.csproj" />
1922
</Folder>
23+
<Folder Name="/src/core/" />
24+
<Folder Name="/src/extensions/" />
2025
<Folder Name="/tests/">
2126
<Project Path="tests\Flowthru.Tests\Flowthru.Tests.csproj" />
2227
<Project Path="tests\Flowthru.Tests.Common\Flowthru.Tests.Common.csproj" />

‎examples/advanced/KedroSpaceflightsGQL/.gitignore‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Generated metadata
2+
Metadata/
3+
4+
# Ignore all generated data artifacts
5+
Data/**
6+
7+
# Un-ignore layer directories and their Schemas subdirectories so git can traverse them
8+
!Data/*/
9+
!Data/*/Schemas/
10+
!Data/_01_Raw/Datasets/
11+
12+
# Allow raw input dataset files
13+
!Data/_01_Raw/Datasets/*
14+
15+
# Allow all schema and catalog .cs files
16+
!Data/**/*.cs
17+
18+
# Un-ignore all .gitkeep files anywhere
19+
!**/.gitkeep

‎examples/advanced/KedroSpaceflightsGQL/.graphqlrc.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"schema": "Infra/GqlClient/schema.graphql",
3+
"documents": "Infra/GqlClient/**/*.graphql",
4+
"extensions": {
5+
"strawberryShake": {
6+
"name": "SpaceflightsClient",
7+
"namespace": "KedroSpaceflightsGQL.Infra.GqlClient",
8+
"dependencyInjection": true,
9+
"noStore": true,
10+
"records": {
11+
"inputs": true,
12+
"entities": false
13+
}
14+
}
15+
}
16+
}

‎examples/advanced/KedroSpaceflightsGQL/Data/Catalog.cs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
using Flowthru.Core.Data;
2+
using KedroSpaceflightsGQL.Infra.GqlClient;
3+
4+
namespace KedroSpaceflightsGQL.Data;
5+
6+
/// <summary>
7+
/// Data catalog for the Spaceflights GQL pipeline, providing access to datasets across all data layers.
8+
/// </summary>
9+
public partial class Catalog : CatalogAbstract
10+
{
11+
private readonly string _basePath;
12+
private readonly ISpaceflightsClient _client;
13+
14+
/// <summary>
15+
/// Initializes a new instance of the <see cref="Catalog"/> class.
16+
/// </summary>
17+
/// <param name="basePath">The base path for data storage.</param>
18+
/// <param name="client">
19+
/// StrawberryShake-generated GraphQL client. Swap this for a real endpoint by configuring
20+
/// the named <c>HttpClient</c> in <c>Program.ConfigureServices</c> to point at your GQL server.
21+
/// </param>
22+
public Catalog(string basePath, ISpaceflightsClient client)
23+
{
24+
_basePath = basePath;
25+
_client = client;
26+
InitializeCatalogProperties();
27+
}
28+
}

‎examples/advanced/KedroSpaceflightsGQL/Data/_01_Raw/Catalog.Raw.cs‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
using Flowthru.Core.Data;
2+
using Flowthru.Extensions.GQL.Data;
3+
using KedroSpaceflightsGQL.Data._01_Raw.Schemas;
4+
using KedroSpaceflightsGQL.Infra.GqlClient;
5+
6+
namespace KedroSpaceflightsGQL.Data;
7+
8+
public partial class Catalog
9+
{
10+
// ── Seed entries (CSV / Excel) — consumed by the Ingest flow ────────────
11+
12+
/// <summary>
13+
/// Raw company data read from CSV. Used by the Ingest flow to seed the GQL server.
14+
/// </summary>
15+
public IItem<IEnumerable<CompanySchema>> SeedCompanies =>
16+
CreateItem(
17+
() =>
18+
ItemFactory.Enumerable.Csv<CompanySchema>(
19+
label: "SeedCompanies",
20+
filePath: $"{_basePath}/_01_Raw/Datasets/companies.csv"
21+
)
22+
);
23+
24+
/// <summary>
25+
/// Raw review data read from CSV. Used by the Ingest flow to seed the GQL server.
26+
/// </summary>
27+
public IItem<IEnumerable<ReviewSchema>> SeedReviews =>
28+
CreateItem(
29+
() =>
30+
ItemFactory.Enumerable.Csv<ReviewSchema>(
31+
label: "SeedReviews",
32+
filePath: $"{_basePath}/_01_Raw/Datasets/reviews.csv"
33+
)
34+
);
35+
36+
/// <summary>
37+
/// Raw shuttle data read from Excel. Used by the Ingest flow to seed the GQL server.
38+
/// </summary>
39+
public IItem<IEnumerable<ShuttleSchema>> SeedShuttles =>
40+
CreateItem(
41+
() =>
42+
ItemFactory.Enumerable.Excel<ShuttleSchema>(
43+
label: "SeedShuttles",
44+
filePath: $"{_basePath}/_01_Raw/Datasets/shuttles.xlsx",
45+
sheetName: "Sheet1"
46+
)
47+
);
48+
49+
// /// <summary>
50+
// /// In-memory flag written by the Ingest flow after all mutations succeed.
51+
// /// Downstream flows depend on this via the DAG.
52+
// /// </summary>
53+
public IItem<bool> GqlDatabaseSeeded =>
54+
CreateItem(() => ItemFactory.Single.Memory<bool>("GqlDatabaseSeeded"));
55+
56+
// ── Raw GQL entries — consumed by the DataProcessing flow ───────────────
57+
58+
/// <summary>
59+
/// Raw company data queried from the GQL server.
60+
/// </summary>
61+
public IItem<IEnumerable<IGetCompanies_Companies>> Companies =>
62+
CreateItem(
63+
() =>
64+
GqlItemFactory.Enumerable.Query<IGetCompaniesResult, IGetCompanies_Companies>(
65+
label: "Companies",
66+
queryFunc: ct => _client.GetCompanies.ExecuteAsync(ct),
67+
selectData: r => r.Companies,
68+
allowEmptyData: true
69+
)
70+
);
71+
72+
/// <summary>
73+
/// Raw shuttle data queried from the GQL server.
74+
/// </summary>
75+
public IItem<IEnumerable<IGetShuttles_Shuttles>> Shuttles =>
76+
CreateItem(
77+
() =>
78+
GqlItemFactory.Enumerable.Query<IGetShuttlesResult, IGetShuttles_Shuttles>(
79+
label: "Shuttles",
80+
queryFunc: ct => _client.GetShuttles.ExecuteAsync(ct),
81+
selectData: r => r.Shuttles,
82+
allowEmptyData: true
83+
)
84+
);
85+
86+
/// <summary>
87+
/// Raw review data queried from the GQL server.
88+
/// </summary>
89+
public IItem<IEnumerable<IGetReviews_Reviews>> Reviews =>
90+
CreateItem(
91+
() =>
92+
GqlItemFactory.Enumerable.Query<IGetReviewsResult, IGetReviews_Reviews>(
93+
label: "Reviews",
94+
queryFunc: ct => _client.GetReviews.ExecuteAsync(ct),
95+
selectData: r => r.Reviews,
96+
allowEmptyData: true
97+
)
98+
);
99+
}

‎examples/advanced/KedroSpaceflightsGQL/Data/_01_Raw/Datasets/NOTICE‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
NOTICE
2+
3+
The dataset files in this directory (companies.csv, reviews.csv, shuttles.xlsx)
4+
are derived from the Kedro Spaceflights tutorial dataset, which is part of the
5+
kedro-starters project.
6+
7+
Source: https://github.com/kedro-org/kedro-starters
8+
Copyright: Kedro team (https://github.com/kedro-org)
9+
License: Apache License 2.0
10+
11+
These files are included here for demonstration and testing purposes as part of
12+
the Flowthru project's example implementations.
13+
14+
Licensed under the Apache License, Version 2.0 (the "License");
15+
you may not use this file except in compliance with the License.
16+
You may obtain a copy of the License at
17+
18+
http://www.apache.org/licenses/LICENSE-2.0
19+
20+
Unless required by applicable law or agreed to in writing, software
21+
distributed under the License is distributed on an "AS IS" BASIS,
22+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
23+
See the License for the specific language governing permissions and
24+
limitations under the License.

0 commit comments

Comments
 (0)