M2: eval harness — golden tests gated on edit distance

- SkillEvaluator (internal to Skills): runs each golden test through an ISkillExecutor and passes only if normalized edit distance <= threshold (the north-star metric). The executor is a stub in M2 (no model runtime); M4's assembler supplies the real one and publishing is gated on the report. The indexer's structural gate (roles + >=1 golden test) stands until then. - InternalsVisibleTo the integration tests so the harness is exercised directly. Verified: build green; ArchitectureTests 8/8; IntegrationTests 25/25 (+3 eval-harness unit tests: pass on match, fail on divergence, fail with no golden tests).
2026-06-09 18:42:19 +03:30
parent bfcd223374
commit e987e33c0a
3 changed files with 99 additions and 0 deletions
@@ -0,0 +1,49 @@
+using TeamUp.Modules.Skills.Domain;
+using TeamUp.Modules.Skills.Eval;
+using Xunit;
+
+namespace TeamUp.IntegrationTests;
+
+/// <summary>Unit coverage for the eval harness (no database). Uses a stub executor for the model.</summary>
+public sealed class SkillEvaluatorTests
+{
+    private sealed class StubExecutor(Func<string, string> respond) : ISkillExecutor
+    {
+        public Task<string> ExecuteAsync(string skillBody, string input, CancellationToken cancellationToken = default) =>
+            Task.FromResult(respond(input));
+    }
+
+    private static List<GoldenExample> Golden(string input, string expected) =>
+        [new GoldenExample { Input = input, Expected = expected }];
+
+    [Fact]
+    public async Task Passes_when_output_matches_expected()
+    {
+        var report = await new SkillEvaluator().EvaluateAsync(
+            Golden("anything", "a clear logout button in the header"),
+            "body",
+            new StubExecutor(_ => "a clear logout button in the header"));
+
+        Assert.True(report.Passed);
+        Assert.Equal(0d, report.WorstDistance, precision: 3);
+    }
+
+    [Fact]
+    public async Task Fails_when_output_diverges()
+    {
+        var report = await new SkillEvaluator().EvaluateAsync(
+            Golden("anything", "a clear logout button in the header"),
+            "body",
+            new StubExecutor(_ => "something completely unrelated and very different indeed"));
+
+        Assert.False(report.Passed);
+        Assert.True(report.WorstDistance > 0.34);
+    }
+
+    [Fact]
+    public async Task Fails_when_there_are_no_golden_tests()
+    {
+        var report = await new SkillEvaluator().EvaluateAsync([], "body", new StubExecutor(_ => "x"));
+        Assert.False(report.Passed);
+    }
+}