From 2b905d3fb2ff5585f52cea97f189cd9f027a81e6 Mon Sep 17 00:00:00 2001 From: Pat Garrity Date: Wed, 30 Jul 2025 22:21:02 -0500 Subject: [PATCH] Working on updating models and defining the reporting module. --- build.sbt | 17 +++++- .../scala/gs/test/v0/api}/GroupResult.scala | 13 +++-- .../scala/gs/test/v0/api/SuiteExecution.scala | 22 ++++++-- .../scala/gs/test/v0/reporting/Reporter.scala | 45 ++++++++++++++++ .../test/v0/runtime/engine/EngineStats.scala | 37 +++++++------ .../test/v0/runtime/engine/TestEngine.scala | 52 ++++++++++++++++--- 6 files changed, 157 insertions(+), 29 deletions(-) rename modules/{runtime/src/main/scala/gs/test/v0/runtime/engine => api/src/main/scala/gs/test/v0/api}/GroupResult.scala (68%) create mode 100644 modules/reporting/src/main/scala/gs/test/v0/reporting/Reporter.scala diff --git a/build.sbt b/build.sbt index 80f8dec..e67043e 100644 --- a/build.sbt +++ b/build.sbt @@ -96,10 +96,25 @@ lazy val api = project ) ) +lazy val reporting = project + .in(file("modules/reporting")) + .dependsOn(`test-support` % "test->test") + .dependsOn(api) + .settings(sharedSettings) + .settings(testSettings) + .settings( + name := s"${gsProjectName.value}-reporting-v${semVerMajor.value}" + ) + .settings( + libraryDependencies ++= Seq( + Deps.Fs2.Core + ) + ) + lazy val runtime = project .in(file("modules/runtime")) .dependsOn(`test-support` % "test->test") - .dependsOn(api) + .dependsOn(api, reporting) .settings(sharedSettings) .settings(testSettings) .settings( diff --git a/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/GroupResult.scala b/modules/api/src/main/scala/gs/test/v0/api/GroupResult.scala similarity index 68% rename from modules/runtime/src/main/scala/gs/test/v0/runtime/engine/GroupResult.scala rename to modules/api/src/main/scala/gs/test/v0/api/GroupResult.scala index e306f77..6731c5b 100644 --- a/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/GroupResult.scala +++ b/modules/api/src/main/scala/gs/test/v0/api/GroupResult.scala @@ -1,7 +1,5 @@ -package gs.test.v0.runtime.engine +package gs.test.v0.api -import gs.test.v0.api.TestExecution -import gs.test.v0.api.TestGroupDefinition import scala.concurrent.duration.FiniteDuration /** Represents the results of executing an entire group of tests. @@ -12,6 +10,12 @@ import scala.concurrent.duration.FiniteDuration * The documentation for the group. * @param duration * The overall duration of execution. + * @param seen + * The number of tests seen. + * @param passed + * The number of tests which passed. + * @param failed + * The number of tests which failed. * @param testExecutions * List of test results. */ @@ -19,5 +23,8 @@ final class GroupResult( val name: TestGroupDefinition.Name, val documentation: Option[String], val duration: FiniteDuration, + val seen: Long, + val passed: Long, + val failed: Long, val testExecutions: List[TestExecution] ) diff --git a/modules/api/src/main/scala/gs/test/v0/api/SuiteExecution.scala b/modules/api/src/main/scala/gs/test/v0/api/SuiteExecution.scala index 64a75a6..c5f9ed2 100644 --- a/modules/api/src/main/scala/gs/test/v0/api/SuiteExecution.scala +++ b/modules/api/src/main/scala/gs/test/v0/api/SuiteExecution.scala @@ -4,13 +4,29 @@ import gs.uuid.v0.UUID import java.time.Instant import scala.concurrent.duration.FiniteDuration +/** Describes the overall result of execution a suite of tests. + * + * @param id + * Unique identifier for this execution. + * @param suite + * Suite-level identifiers and metadata. + * @param duration + * Overall amount of time it took to execute the suite. + * @param countSeen + * Overall number of tests seen. + * @param countPassed + * Overall number of passed tests. + * @param countFailed + * Overall number of failed tests. + * @param executedAt + * Timestamp at which this suite was executed. + */ case class SuiteExecution( id: UUID, - name: String, - documentation: Option[String], + testSuite: TestSuite, duration: FiniteDuration, countSeen: Long, - countSucceeded: Long, + countPassed: Long, countFailed: Long, executedAt: Instant ) diff --git a/modules/reporting/src/main/scala/gs/test/v0/reporting/Reporter.scala b/modules/reporting/src/main/scala/gs/test/v0/reporting/Reporter.scala new file mode 100644 index 0000000..3ff2451 --- /dev/null +++ b/modules/reporting/src/main/scala/gs/test/v0/reporting/Reporter.scala @@ -0,0 +1,45 @@ +package gs.test.v0.reporting + +import gs.test.v0.api.GroupResult +import gs.test.v0.api.SuiteExecution + +/** Interface for reporters - implementations that report on test results. + * + * Example implementations include writing to standard output or writing to + * some JSON-formatted file. + * + * ## Order of Operations + * + * 1. `beginReporting()` 2. `reportGroup` for each group executed 3. + * `reportSuite` 4. `endReporting()` + */ +trait Reporter[F[_]]: + /** Hook for the beginning of the reporting lifecycle. This allows + * implementations to perform "setup" actions, such as opening a JSON object + * or writing a header. + */ + def beginReporting(): F[Unit] + + /** Report the results of a single group. + * + * @param groupResult + * The [[GroupResult]] that describes results. + * @return + * Side-effect that describes the reporting operation. + */ + def reportGroup(groupResult: GroupResult): F[Unit] + + /** Report the results of an entire suite. + * + * @param suiteExecution + * The [[SuiteExecution]] that describes results. + * @return + * Side-effect that describes the reporting operation. + */ + def reportSuite(suiteExecution: SuiteExecution): F[Unit] + + /** Hook for the end of the reporting lifecycle. This allows implementations + * to perform "finish" actions, such as closing a JSON object or writing a + * footer. + */ + def endReporting(): F[Unit] diff --git a/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/EngineStats.scala b/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/EngineStats.scala index e7a5b5b..9e730df 100644 --- a/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/EngineStats.scala +++ b/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/EngineStats.scala @@ -13,15 +13,15 @@ import scala.concurrent.duration.FiniteDuration * Duration of all recorded tests. * @param countSeen * Number of tests encountered. - * @param countSucceeded - * Number of tests that succeeded. + * @param countPassed + * Number of tests that passed. * @param countFailed * Number of tests that failed. */ final class EngineStats[F[_]: Async] private ( overallDuration: Ref[F, FiniteDuration], countSeen: Ref[F, Long], - countSucceeded: Ref[F, Long], + countPassed: Ref[F, Long], countFailed: Ref[F, Long] ): /** @return @@ -35,9 +35,9 @@ final class EngineStats[F[_]: Async] private ( def seen: F[Long] = countSeen.get /** @return - * Number of tests that succeeded. + * Number of tests that passed. */ - def succeeded: F[Long] = countSucceeded.get + def passed: F[Long] = countPassed.get /** @return * Number of tests that failed. @@ -46,15 +46,20 @@ final class EngineStats[F[_]: Async] private ( /** Update the stats based on the results of an entire group. * - * @param groupResult - * The [[GroupResult]] representing the group. + * @param duration + * The length of time it took to execute the group. + * @param testExecutions + * The list of all [[TestExecution]] produced by the group. * @return * Side-effect which updates statistic values. */ - def updateForGroup(groupResult: GroupResult): F[Unit] = + def updateForGroup( + duration: FiniteDuration, + testExecutions: List[TestExecution] + ): F[Unit] = for - _ <- overallDuration.update(base => base + groupResult.duration) - _ <- groupResult.testExecutions.map(updateForTest).sequence + _ <- overallDuration.update(base => base + duration) + _ <- testExecutions.map(updateForTest).sequence yield () /** Update the stats based on the results of a single test. @@ -69,7 +74,7 @@ final class EngineStats[F[_]: Async] private ( _ <- countSeen.update(_ + 1L) _ <- testExecution.result match case Left(_) => countFailed.update(_ + 1L) - case Right(_) => countSucceeded.update(_ + 1L) + case Right(_) => countPassed.update(_ + 1L) yield () object EngineStats: @@ -81,14 +86,14 @@ object EngineStats: */ def initialize[F[_]: Async]: F[EngineStats[F]] = for - duration <- Ref.of(FiniteDuration(0L, TimeUnit.NANOSECONDS)) - seen <- Ref.of(0L) - succeeded <- Ref.of(0L) - failed <- Ref.of(0L) + duration <- Ref.of(FiniteDuration(0L, TimeUnit.NANOSECONDS)) + seen <- Ref.of(0L) + passed <- Ref.of(0L) + failed <- Ref.of(0L) yield new EngineStats[F]( overallDuration = duration, countSeen = seen, - countSucceeded = succeeded, + countPassed = passed, countFailed = failed ) diff --git a/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/TestEngine.scala b/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/TestEngine.scala index 7239cfd..c657c8e 100644 --- a/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/TestEngine.scala +++ b/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/TestEngine.scala @@ -2,12 +2,14 @@ package gs.test.v0.runtime.engine import cats.effect.Async import cats.syntax.all.* +import gs.test.v0.api.GroupResult import gs.test.v0.api.SuiteExecution import gs.test.v0.api.TestDefinition import gs.test.v0.api.TestExecution import gs.test.v0.api.TestFailure import gs.test.v0.api.TestGroupDefinition import gs.test.v0.api.TestSuite +import gs.test.v0.reporting.Reporter import gs.timing.v0.Timing import gs.uuid.v0.UUID import java.time.Clock @@ -51,6 +53,7 @@ import natchez.Span */ final class TestEngine[F[_]: Async]( val configuration: EngineConfiguration, + reporter: Reporter[F], timing: Timing[F], suiteExecutionIdGenerator: UUID.Generator, testExecutionIdGenerator: UUID.Generator, @@ -66,13 +69,36 @@ final class TestEngine[F[_]: Async]( for executedAt <- Async[F].delay(Instant.now(clock)) stats <- EngineStats.initialize[F] + + // Start reporting + _ <- reporter.beginReporting() + // TODO: Just do telemetry for the whole damn thing. _ <- tests .mapAsync(configuration.groupConcurrency.toInt())(runGroup) - .evalTap(stats.updateForGroup) + .evalTap(groupResult => + for + // Update the overall statistics based on this group. + _ <- stats.updateForGroup( + duration = groupResult.duration, + testExecutions = groupResult.testExecutions + ) + + // Report group level results for this group. + _ <- reporter.reportGroup(groupResult) + yield () + ) .compile .drain + + // Calculate the final summary of execution at the suite level. suiteExecution <- makeSuiteExecution(suite, stats, executedAt) + + // Report suite level results. + _ <- reporter.reportSuite(suiteExecution) + + // Finish reporting. + _ <- reporter.endReporting() yield suiteExecution def runGroup( @@ -80,6 +106,7 @@ final class TestEngine[F[_]: Async]( ): F[GroupResult] = entryPoint.root(EngineConstants.Tracing.RootSpan).use { rootSpan => for + groupStats <- EngineStats.initialize[F] // Augment the span with all group-level metadata. _ <- rootSpan .put(EngineConstants.MetaData.TestGroupName -> group.name.show) @@ -106,10 +133,24 @@ final class TestEngine[F[_]: Async]( // Calculate the overall elapsed time for this group. elapsed <- timer.checkpoint() + + // Calculate group-level statistics. + _ <- groupStats.updateForGroup( + duration = elapsed.duration, + testExecutions = testExecutions + ) + + // Extract the group statistic values for inclusion in the result.. + seen <- groupStats.seen + passed <- groupStats.passed + failed <- groupStats.failed yield new GroupResult( name = group.name, documentation = group.documentation, duration = elapsed.duration, + seen = seen, + passed = passed, + failed = failed, testExecutions = testExecutions ) } @@ -171,7 +212,7 @@ final class TestEngine[F[_]: Async]( tags = test.tags, markers = test.markers, result = result, - // TODO TraceID isn't that useful here, need SpanID + // TODO: TraceID isn't that useful here, need SpanID traceId = traceId, sourcePosition = test.sourcePosition, duration = elapsed.duration @@ -208,15 +249,14 @@ final class TestEngine[F[_]: Async]( for overallDuration <- stats.duration countSeen <- stats.seen - countSucceeded <- stats.succeeded + countPassed <- stats.passed countFailed <- stats.failed yield SuiteExecution( id = suiteExecutionIdGenerator.next(), - name = suite.name, - documentation = suite.documentation, + testSuite = suite, duration = overallDuration, countSeen = countSeen, - countSucceeded = countSucceeded, + countPassed = countPassed, countFailed = countFailed, executedAt = executedAt )