fix: phase 1 of diagnostic setup

Author: Spelkington

scripts/_test/_lib.mjs

@@ -0,0 +1,105 @@
+/**
+ * Shared workspace-walking helpers for the `_test:*` private NX subtargets.
+ *
+ * Each `_test:*` subtarget under `tests:test`'s barrel has a focused script in this
+ * directory that enforces one structural workspace invariant. They share the helpers
+ * defined here — `findCsproj`, `findCs`, `findJson`, path resolution — so the cost of
+ * adding a new invariant is one new script, not one new toolchain.
+ *
+ * See [docs/scratch/data-extension-contract.md §6 "Workspace Target Architecture"](../../docs/scratch/data-extension-contract.md#6-workspace-target-architecture)
+ * for the full pattern; see [tests/README.md](../../tests/README.md) for kit infrastructure.
+ */
+
+import { readdirSync, existsSync } from 'node:fs';
+import { join, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { resolve, dirname } from 'node:path';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+/** Workspace root (two levels up from this file: scripts/_test/_lib.mjs → repo root). */
+export const ROOT = resolve(__dirname, '..', '..');
+
+/** Conventional source / tests / kit directories under ROOT. */
+export const SRC_DIR = join(ROOT, 'src');
+export const TESTS_DIR = join(ROOT, 'tests');
+export const KITS_DIR = join(TESTS_DIR, 'helpers', 'Flowthru.Tests.Kits');
+
+/** Recursively collect all .csproj paths under `dir`. */
+export function findCsproj(dir) {
+  const results = [];
+  if (!existsSync(dir)) return results;
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (entry.name.startsWith('.')) continue;
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      results.push(...findCsproj(fullPath));
+    } else if (entry.name.endsWith('.csproj')) {
+      results.push(fullPath);
+    }
+  }
+  return results;
+}
+
+/**
+ * Recursively collect all .cs files under `dir`, skipping `obj/`, `bin/`, and
+ * `TestResults/` directories.
+ */
+export function findCs(dir) {
+  const results = [];
+  if (!existsSync(dir)) return results;
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (entry.name.startsWith('.')) continue;
+    if (entry.name === 'obj' || entry.name === 'bin' || entry.name === 'TestResults') {
+      continue;
+    }
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      results.push(...findCs(fullPath));
+    } else if (entry.name.endsWith('.cs')) {
+      results.push(fullPath);
+    }
+  }
+  return results;
+}
+
+/** Recursively collect all .json files under `dir`, skipping standard build dirs. */
+export function findJson(dir) {
+  const results = [];
+  if (!existsSync(dir)) return results;
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (entry.name.startsWith('.')) continue;
+    if (entry.name === 'obj' || entry.name === 'bin' || entry.name === 'TestResults') {
+      continue;
+    }
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      results.push(...findJson(fullPath));
+    } else if (entry.name.endsWith('.json')) {
+      results.push(fullPath);
+    }
+  }
+  return results;
+}
+
+/** Recursively collect all .md files under `dir`, skipping standard build dirs. */
+export function findMd(dir) {
+  const results = [];
+  if (!existsSync(dir)) return results;
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (entry.name.startsWith('.') && entry.name !== '.claude') continue;
+    if (entry.name === 'obj' || entry.name === 'bin' || entry.name === 'node_modules') continue;
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      results.push(...findMd(fullPath));
+    } else if (entry.name.endsWith('.md')) {
+      results.push(fullPath);
+    }
+  }
+  return results;
+}
+
+/** Relative path from ROOT, using forward slashes for legible output. */
+export function rel(absPath) {
+  return absPath.slice(ROOT.length + 1).replaceAll(sep, '/');
+}

scripts/_test/conformance-presence.mjs

@@ -0,0 +1,172 @@
+#!/usr/bin/env node
+/**
+ * `_test:conformance-presence` — every first-party Flowthru extension that implements a
+ * Core extension surface (`IStorageAdapter<T>`, `IFormatSerializer<TRow>`, `IStorageMedium`,
+ * `IStorageMediumProvider`, `IMetadataProvider`) has at least one corresponding `*Conformance`
+ * subclass in its sibling `tests/extensions/<Ext>.Tests/` project.
+ *
+ * The check is at the *extension level*, not per implementor: an extension passes if any
+ * conformance subclass for the relevant surface kind exists, regardless of which specific
+ * implementor it covers. This admits patterns like Gql, where `GqlQueryStorageAdapter`
+ * doesn't fit the kit's contract (read-only deferred handle) but its sibling
+ * `GqlStorageAdapter` does and has a conformance subclass. The extension-level
+ * aggregation also lets multi-shape conformance (e.g., EFCore's flat + nested entities)
+ * cover multiple implementors with one parameterized base.
+ *
+ * Container-adapter conformance (`IContainerAdapter<TContainer, TRow>`) is intentionally
+ * excluded: the kit does not yet codify a `ContainerAdapterConformance` base, and the only
+ * first-party container adapter outside Core (MLNet's `DataViewContainerAdapter`) was
+ * descoped from the conformance initiative because ONNX is structurally different from
+ * the row-oriented surfaces the kit targets. Restore it to the SURFACES list when
+ * container conformance is added.
+ *
+ * Exits with non-zero on any failure. Extracted from the previous monolithic
+ * `verify-test-coverage.mjs` Pass 2.
+ *
+ * Usage:
+ *   node scripts/_test/conformance-presence.mjs
+ */
+
+import { readFileSync, existsSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { findCs, rel, SRC_DIR, TESTS_DIR } from './_lib.mjs';
+
+// ── Surfaces in scope for conformance enforcement ────────────────────────────
+//
+// Each surface maps from the Core interface name (matched on the impl side) to the
+// kit base class name (matched on the test side).
+//
+const SURFACES = [
+  { iface: 'IStorageAdapter', base: 'StorageAdapterConformance' },
+  { iface: 'IFormatSerializer', base: 'FormatSerializerConformance' },
+  { iface: 'IStorageMedium', base: 'StorageMediumConformance' },
+  { iface: 'IStorageMediumProvider', base: 'StorageMediumProviderConformance' },
+  { iface: 'IMetadataProvider', base: 'MetadataProviderConformance' },
+];
+
+/**
+ * Returns the set of surface keys (e.g., 'IStorageAdapter') the file declares as base
+ * types or implemented interfaces. Matches `: IStorageAdapter<` and similar across multi-
+ * line declarations by collapsing whitespace before regex matching.
+ */
+function detectSurfaceImpls(filePath) {
+  const text = readFileSync(filePath, 'utf8');
+  const collapsed = text.replace(/\s+/g, ' ');
+  const found = new Set();
+  for (const { iface } of SURFACES) {
+    // Match `: IStorageAdapter<` (with or without surrounding whitespace) anywhere a base
+    // list could appear. Also match `, IStorageAdapter<` (additional interfaces).
+    const re = new RegExp(`[:,]\\s*${iface}(?:<|\\s)`, 'g');
+    if (re.test(collapsed)) {
+      found.add(iface);
+    }
+  }
+  return found;
+}
+
+/**
+ * Returns the set of conformance-base keys (e.g., 'StorageAdapterConformance') the file
+ * declares as a base class.
+ */
+function detectConformanceBases(filePath) {
+  const text = readFileSync(filePath, 'utf8');
+  const collapsed = text.replace(/\s+/g, ' ');
+  const found = new Set();
+  for (const { base } of SURFACES) {
+    // Match `: StorageAdapterConformance<` or `: StorageAdapterConformance ` (no generics).
+    const re = new RegExp(`[:,]\\s*${base}(?:<|\\s|$)`, 'g');
+    if (re.test(collapsed)) {
+      found.add(base);
+    }
+  }
+  return found;
+}
+
+const EXTENSIONS_SRC = join(SRC_DIR, 'extensions');
+const EXTENSIONS_TESTS = join(TESTS_DIR, 'extensions');
+
+const conformanceFailures = [];
+
+if (existsSync(EXTENSIONS_SRC)) {
+  for (const entry of readdirSync(EXTENSIONS_SRC, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    if (!entry.name.startsWith('Flowthru.Extensions.')) continue;
+
+    const extName = entry.name;
+    const extSrcDir = join(EXTENSIONS_SRC, extName);
+    const extTestsDir = join(EXTENSIONS_TESTS, `${extName}.Tests`);
+
+    // Collect all surface kinds this extension implements.
+    const implementedSurfaces = new Set();
+    for (const csFile of findCs(extSrcDir)) {
+      const impls = detectSurfaceImpls(csFile);
+      for (const surface of impls) {
+        implementedSurfaces.add(surface);
+      }
+    }
+
+    if (implementedSurfaces.size === 0) {
+      // Extension implements no kit-tracked surface (e.g., EFCore.Bulk supplies saveFunc
+      // delegates only). Nothing to enforce.
+      continue;
+    }
+
+    // Collect all conformance bases the test project covers.
+    const coveredBases = new Set();
+    if (existsSync(extTestsDir)) {
+      for (const csFile of findCs(extTestsDir)) {
+        const bases = detectConformanceBases(csFile);
+        for (const base of bases) {
+          coveredBases.add(base);
+        }
+      }
+    }
+
+    // Map each implemented surface to its expected conformance base, then check coverage.
+    const missingBases = [];
+    for (const surface of implementedSurfaces) {
+      const entry = SURFACES.find((s) => s.iface === surface);
+      if (!entry) continue;
+      if (!coveredBases.has(entry.base)) {
+        missingBases.push({ surface, base: entry.base });
+      }
+    }
+
+    if (missingBases.length > 0) {
+      conformanceFailures.push({
+        extension: extName,
+        testsDir: existsSync(extTestsDir) ? rel(extTestsDir) : '<missing>',
+        missing: missingBases,
+      });
+    }
+  }
+}
+
+let exitCode = 0;
+
+if (conformanceFailures.length > 0) {
+  exitCode = 1;
+  console.error(
+    `\n${conformanceFailures.length} extension(s) implement Core surfaces but lack conformance coverage:\n`
+  );
+  for (const { extension, testsDir, missing } of conformanceFailures) {
+    console.error(`  extension: ${extension}`);
+    console.error(`  tests:     ${testsDir}`);
+    for (const { surface, base } of missing) {
+      console.error(`    - implements ${surface} but no ${base} subclass found`);
+    }
+    console.error('');
+  }
+  console.error(
+    'See `tests/README.md` (Extension Conformance Kits) and ' +
+      '`docs/scratch/extension-conformance-kits.md` for the kit pattern.\n'
+  );
+}
+
+if (exitCode === 0) {
+  console.log(
+    '_test:conformance-presence — every first-party extension surface implementor has a conformance subclass.'
+  );
+}
+
+process.exit(exitCode);

scripts/_test/dead-fixtures.mjs

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+/**
+ * `_test:dead-fixtures` — every fixture file under
+ * `tests/helpers/Flowthru.Tests.Kits/Fixtures/` must be referenced by at least one
+ * conformance subclass (typically as an entry in a `static IEnumerable<string> Fixtures`
+ * member, surfaced via `[TestFixtureSource(nameof(Fixtures))]`).
+ *
+ * A "dead fixture" is one that exists in the kit's data catalog but isn't loaded by any
+ * conformance suite. Like dead schemas, it's a false-confidence trap.
+ *
+ * Detection strategy:
+ *   1. Enumerate `.json` fixture files under `Fixtures/`.
+ *   2. For each fixture, compute its "kit-relative" path (e.g., `Flat/Simple/rows.json`).
+ *   3. Search `tests/extensions/**\/*.cs` and `tests/core/**\/*.cs` for that string. If
+ *      it appears literally in any source file, the fixture is alive.
+ *
+ * Caveats:
+ *   - The literal-string match is intentional: `[TestFixtureSource]` resolves at test-
+ *     run time via NUnit reflection, so a static analyzer can't follow `nameof(Fixtures)`
+ *     back to the array. The kit's authoring convention is to inline fixture paths as
+ *     string literals, so the match is straightforward.
+ *   - A fixture referenced via dynamic concatenation (e.g., `$"{Prefix}/rows.json"`)
+ *     would evade detection. None of the existing kit subclasses use that pattern, but
+ *     callers of this kit who do should be aware.
+ *
+ * Usage:
+ *   node scripts/_test/dead-fixtures.mjs
+ */
+
+import { readFileSync } from 'node:fs';
+import { join, sep } from 'node:path';
+import { findJson, findCs, rel, KITS_DIR, TESTS_DIR } from './_lib.mjs';
+
+const FIXTURES_DIR = join(KITS_DIR, 'Fixtures');
+const REFERENCE_SEARCH_DIRS = [
+  join(TESTS_DIR, 'extensions'),
+  join(TESTS_DIR, 'core'),
+];
+
+// ── Enumerate fixtures and compute their kit-relative paths ──────────────────
+
+const fixtures = []; // { absPath, kitRelPath }
+for (const fixturePath of findJson(FIXTURES_DIR)) {
+  const kitRelPath = fixturePath.slice(FIXTURES_DIR.length + 1).replaceAll(sep, '/');
+  fixtures.push({ absPath: fixturePath, kitRelPath });
+}
+
+// ── Build reference index: every .cs source under conformance dirs ───────────
+
+const sourceTexts = [];
+for (const dir of REFERENCE_SEARCH_DIRS) {
+  for (const csFile of findCs(dir)) {
+    sourceTexts.push(readFileSync(csFile, 'utf8'));
+  }
+}
+
+// ── Match each fixture path against the reference index ─────────────────────
+
+const orphans = [];
+for (const { absPath, kitRelPath } of fixtures) {
+  const found = sourceTexts.some((text) => text.includes(kitRelPath));
+  if (!found) {
+    orphans.push({ path: kitRelPath, file: rel(absPath) });
+  }
+}
+
+let exitCode = 0;
+if (orphans.length > 0) {
+  exitCode = 1;
+  console.error(
+    `\n${orphans.length} kit fixture(s) exist but are not referenced by any conformance subclass:\n`
+  );
+  for (const { path, file } of orphans) {
+    console.error(`  ${path}  (${file})`);
+  }
+  console.error(
+    '\nA fixture is "alive" iff at least one conformance subclass references its kit-relative path'
+  );
+  console.error(
+    '(typically in a `static IEnumerable<string> Fixtures` array on a `[TestFixtureSource]`-decorated subclass).'
+  );
+  console.error(
+    'Either wire up a conformance subclass that exercises this fixture, or remove the fixture.\n'
+  );
+}
+
+if (exitCode === 0) {
+  console.log(
+    `_test:dead-fixtures — all ${fixtures.length} kit fixtures are referenced by at least one conformance subclass.`
+  );
+}
+
+process.exit(exitCode);