chore: partial diarization example

Author: Spelkington

examples/archived/DiarizationExample/.python-version

@@ -0,0 +1 @@
+3.10

examples/archived/DiarizationExample/Data/Catalog.cs

@@ -0,0 +1,19 @@
+using Flowthru.Core.Data;
+
+namespace DiarizationExample.Data;
+
+/// <summary>
+/// Data catalog for the Diarization pipeline. Inputs are batch audio files
+/// (one <see cref="Directory{T}"/> of <c>byte[]</c>); intermediate and output
+/// items are flat row schemas keyed by <c>clip_id</c> (= the source file path).
+/// </summary>
+public partial class Catalog : CatalogAbstract
+{
+  private readonly string _basePath;
+
+  public Catalog(string basePath)
+  {
+    _basePath = basePath;
+    InitializeCatalogProperties();
+  }
+}

examples/archived/DiarizationExample/Data/_01_Raw/Catalog.Raw.cs

@@ -0,0 +1,23 @@
+using Flowthru.Core.Data;
+
+namespace DiarizationExample.Data;
+
+public partial class Catalog
+{
+  /// <summary>
+  /// Batch of raw audio files dropped into <c>_01_Raw/Datasets/</c>. Each entry
+  /// in the directory is one independent recording; the key is the full file
+  /// path (used downstream as <c>clip_id</c> on every row). Glob covers the
+  /// common formats — anything ffmpeg can decode is fine since the
+  /// <c>NormalizeAudio</c> step transcodes to 16kHz mono PCM before either
+  /// Whisper or pyannote sees it.
+  /// </summary>
+  public IItem<Directory<byte[]>> AudioClips =>
+    CreateItem(() =>
+      ItemFactory.Enumerable.BinaryDirectory(
+        label: "AudioClips",
+        directoryPath: $"{_basePath}/_01_Raw/Datasets",
+        filePattern: "*.{wav,mp3,m4a,flac,ogg}"
+      )
+    );
+}

examples/archived/DiarizationExample/Data/_02_Intermediate/Catalog.Intermediate.cs

@@ -0,0 +1,20 @@
+using Flowthru.Core.Data;
+
+namespace DiarizationExample.Data;
+
+public partial class Catalog
+{
+  /// <summary>
+  /// Audio normalized to 16kHz mono PCM (WAV bytes). Both Whisper and pyannote
+  /// expect this format; doing the transcode once here means downstream steps
+  /// can run in parallel without each redoing the same work.
+  /// </summary>
+  public IItem<Directory<byte[]>> NormalizedAudio =>
+    CreateItem(() =>
+      ItemFactory.Enumerable.BinaryDirectory(
+        label: "NormalizedAudio",
+        directoryPath: $"{_basePath}/_02_Intermediate/normalized",
+        filePattern: "*.wav"
+      )
+    );
+}

examples/archived/DiarizationExample/Data/_02_Intermediate/normalized/.keep

@@ -0,0 +1,33 @@
+using DiarizationExample.Data._03_Primary.Schemas;
+using Flowthru.Core.Data;
+
+namespace DiarizationExample.Data;
+
+public partial class Catalog
+{
+  /// <summary>
+  /// Whisper transcript segments — one row per (clip_id, start, end) span.
+  /// Stored as Parquet so re-runs can skip transcription if the upstream
+  /// audio hasn't changed.
+  /// </summary>
+  public IItem<IEnumerable<TranscriptSegmentSchema>> Transcripts =>
+    CreateItem(() =>
+      ItemFactory.Enumerable.Parquet<TranscriptSegmentSchema>(
+        label: "Transcripts",
+        filePath: $"{_basePath}/_03_Primary/transcripts.parquet"
+      )
+    );
+
+  /// <summary>
+  /// pyannote diarization turns — one row per (clip_id, start, end, speaker).
+  /// Speaker indices are local to each clip (speaker_0, speaker_1, ...);
+  /// cross-clip speaker identity is out of scope for this example.
+  /// </summary>
+  public IItem<IEnumerable<DiarizationSegmentSchema>> DiarizationTurns =>
+    CreateItem(() =>
+      ItemFactory.Enumerable.Parquet<DiarizationSegmentSchema>(
+        label: "DiarizationTurns",
+        filePath: $"{_basePath}/_03_Primary/diarization.parquet"
+      )
+    );
+}

examples/archived/DiarizationExample/Data/_03_Primary/Catalog.Primary.cs

@@ -0,0 +1,24 @@
+using Flowthru.Core.Abstractions;
+
+namespace DiarizationExample.Data._03_Primary.Schemas;
+
+/// <summary>
+/// One pyannote speaker turn for one input clip. <c>SpeakerId</c> is local
+/// to the clip (e.g., <c>speaker_0</c>, <c>speaker_1</c>); cross-clip speaker
+/// identity is out of scope for this example.
+/// </summary>
+[FlowthruSchema]
+public partial record DiarizationSegmentSchema
+{
+  [SerializedLabel("clip_id")]
+  public string ClipId { get; init; } = null!;
+
+  [SerializedLabel("start")]
+  public double Start { get; init; }
+
+  [SerializedLabel("end")]
+  public double End { get; init; }
+
+  [SerializedLabel("speaker_id")]
+  public string SpeakerId { get; init; } = null!;
+}

examples/archived/DiarizationExample/Data/_03_Primary/Schemas/DiarizationSegmentSchema.cs

@@ -0,0 +1,24 @@
+using Flowthru.Core.Abstractions;
+
+namespace DiarizationExample.Data._03_Primary.Schemas;
+
+/// <summary>
+/// One Whisper transcript segment for one input clip. <c>ClipId</c> is the
+/// source audio's file path (the <c>Directory&lt;byte[]&gt;</c> key); a single
+/// clip produces many rows.
+/// </summary>
+[FlowthruSchema]
+public partial record TranscriptSegmentSchema
+{
+  [SerializedLabel("clip_id")]
+  public string ClipId { get; init; } = null!;
+
+  [SerializedLabel("start")]
+  public double Start { get; init; }
+
+  [SerializedLabel("end")]
+  public double End { get; init; }
+
+  [SerializedLabel("text")]
+  public string Text { get; init; } = null!;
+}

examples/archived/DiarizationExample/Data/_03_Primary/Schemas/TranscriptSegmentSchema.cs

@@ -0,0 +1,27 @@
+using Flowthru.Core.Abstractions;
+
+namespace DiarizationExample.Data._04_Feature.Schemas;
+
+/// <summary>
+/// A transcript segment with its dominant speaker attached. Produced by the
+/// alignment step — for each transcript span, the speaker is whichever
+/// diarization turn covers the largest fraction of the segment.
+/// </summary>
+[FlowthruSchema]
+public partial record AttributedSegmentSchema
+{
+  [SerializedLabel("clip_id")]
+  public string ClipId { get; init; } = null!;
+
+  [SerializedLabel("start")]
+  public double Start { get; init; }
+
+  [SerializedLabel("end")]
+  public double End { get; init; }
+
+  [SerializedLabel("speaker_id")]
+  public string SpeakerId { get; init; } = null!;
+
+  [SerializedLabel("text")]
+  public string Text { get; init; } = null!;
+}