Working on updating models and defining the reporting module.

build.sbt

@@ -96,10 +96,25 @@ lazy val api = project
     )
   )
 
+lazy val reporting = project
+  .in(file("modules/reporting"))
+  .dependsOn(`test-support` % "test->test")
+  .dependsOn(api)
+  .settings(sharedSettings)
+  .settings(testSettings)
+  .settings(
+    name := s"${gsProjectName.value}-reporting-v${semVerMajor.value}"
+  )
+  .settings(
+    libraryDependencies ++= Seq(
+      Deps.Fs2.Core
+    )
+  )
+
 lazy val runtime = project
   .in(file("modules/runtime"))
   .dependsOn(`test-support` % "test->test")
-  .dependsOn(api)
+  .dependsOn(api, reporting)
   .settings(sharedSettings)
   .settings(testSettings)
   .settings(

modules/api/src/main/scala/gs/test/v0/api/GroupResult.scala

@@ -1,7 +1,5 @@
-package gs.test.v0.runtime.engine
+package gs.test.v0.api
 
-import gs.test.v0.api.TestExecution
-import gs.test.v0.api.TestGroupDefinition
 import scala.concurrent.duration.FiniteDuration
 
 /** Represents the results of executing an entire group of tests.
@@ -12,6 +10,12 @@ import scala.concurrent.duration.FiniteDuration
   *   The documentation for the group.
   * @param duration
   *   The overall duration of execution.
+  * @param seen
+  *   The number of tests seen.
+  * @param passed
+  *   The number of tests which passed.
+  * @param failed
+  *   The number of tests which failed.
   * @param testExecutions
   *   List of test results.
   */
@@ -19,5 +23,8 @@ final class GroupResult(
   val name: TestGroupDefinition.Name,
   val documentation: Option[String],
   val duration: FiniteDuration,
+  val seen: Long,
+  val passed: Long,
+  val failed: Long,
   val testExecutions: List[TestExecution]
 )

modules/api/src/main/scala/gs/test/v0/api/SuiteExecution.scala

@@ -4,13 +4,29 @@ import gs.uuid.v0.UUID
 import java.time.Instant
 import scala.concurrent.duration.FiniteDuration
 
+/** Describes the overall result of execution a suite of tests.
+  *
+  * @param id
+  *   Unique identifier for this execution.
+  * @param suite
+  *   Suite-level identifiers and metadata.
+  * @param duration
+  *   Overall amount of time it took to execute the suite.
+  * @param countSeen
+  *   Overall number of tests seen.
+  * @param countPassed
+  *   Overall number of passed tests.
+  * @param countFailed
+  *   Overall number of failed tests.
+  * @param executedAt
+  *   Timestamp at which this suite was executed.
+  */
 case class SuiteExecution(
   id: UUID,
-  name: String,
+  testSuite: TestSuite,
-  documentation: Option[String],
   duration: FiniteDuration,
   countSeen: Long,
-  countSucceeded: Long,
+  countPassed: Long,
   countFailed: Long,
   executedAt: Instant
 )

modules/reporting/src/main/scala/gs/test/v0/reporting/Reporter.scala

@@ -0,0 +1,45 @@
+package gs.test.v0.reporting
+
+import gs.test.v0.api.GroupResult
+import gs.test.v0.api.SuiteExecution
+
+/** Interface for reporters - implementations that report on test results.
+  *
+  * Example implementations include writing to standard output or writing to
+  * some JSON-formatted file.
+  *
+  * ## Order of Operations
+  *
+  *   1. `beginReporting()` 2. `reportGroup` for each group executed 3.
+  *      `reportSuite` 4. `endReporting()`
+  */
+trait Reporter[F[_]]:
+  /** Hook for the beginning of the reporting lifecycle. This allows
+    * implementations to perform "setup" actions, such as opening a JSON object
+    * or writing a header.
+    */
+  def beginReporting(): F[Unit]
+
+  /** Report the results of a single group.
+    *
+    * @param groupResult
+    *   The [[GroupResult]] that describes results.
+    * @return
+    *   Side-effect that describes the reporting operation.
+    */
+  def reportGroup(groupResult: GroupResult): F[Unit]
+
+  /** Report the results of an entire suite.
+    *
+    * @param suiteExecution
+    *   The [[SuiteExecution]] that describes results.
+    * @return
+    *   Side-effect that describes the reporting operation.
+    */
+  def reportSuite(suiteExecution: SuiteExecution): F[Unit]
+
+  /** Hook for the end of the reporting lifecycle. This allows implementations
+    * to perform "finish" actions, such as closing a JSON object or writing a
+    * footer.
+    */
+  def endReporting(): F[Unit]

modules/runtime/src/main/scala/gs/test/v0/runtime/engine/EngineStats.scala

@@ -13,15 +13,15 @@ import scala.concurrent.duration.FiniteDuration
   *   Duration of all recorded tests.
   * @param countSeen
   *   Number of tests encountered.
-  * @param countSucceeded
+  * @param countPassed
-  *   Number of tests that succeeded.
+  *   Number of tests that passed.
   * @param countFailed
   *   Number of tests that failed.
   */
 final class EngineStats[F[_]: Async] private (
   overallDuration: Ref[F, FiniteDuration],
   countSeen: Ref[F, Long],
-  countSucceeded: Ref[F, Long],
+  countPassed: Ref[F, Long],
   countFailed: Ref[F, Long]
 ):
   /** @return
@@ -35,9 +35,9 @@ final class EngineStats[F[_]: Async] private (
   def seen: F[Long] = countSeen.get
 
   /** @return
-    *   Number of tests that succeeded.
+    *   Number of tests that passed.
     */
-  def succeeded: F[Long] = countSucceeded.get
+  def passed: F[Long] = countPassed.get
 
   /** @return
     *   Number of tests that failed.
@@ -46,15 +46,20 @@ final class EngineStats[F[_]: Async] private (
 
   /** Update the stats based on the results of an entire group.
     *
-    * @param groupResult
+    * @param duration
-    *   The [[GroupResult]] representing the group.
+    *   The length of time it took to execute the group.
+    * @param testExecutions
+    *   The list of all [[TestExecution]] produced by the group.
     * @return
     *   Side-effect which updates statistic values.
     */
-  def updateForGroup(groupResult: GroupResult): F[Unit] =
+  def updateForGroup(
+    duration: FiniteDuration,
+    testExecutions: List[TestExecution]
+  ): F[Unit] =
     for
-      _ <- overallDuration.update(base => base + groupResult.duration)
+      _ <- overallDuration.update(base => base + duration)
-      _ <- groupResult.testExecutions.map(updateForTest).sequence
+      _ <- testExecutions.map(updateForTest).sequence
     yield ()
 
   /** Update the stats based on the results of a single test.
@@ -69,7 +74,7 @@ final class EngineStats[F[_]: Async] private (
       _ <- countSeen.update(_ + 1L)
       _ <- testExecution.result match
         case Left(_)  => countFailed.update(_ + 1L)
-        case Right(_) => countSucceeded.update(_ + 1L)
+        case Right(_) => countPassed.update(_ + 1L)
     yield ()
 
 object EngineStats:
@@ -83,12 +88,12 @@ object EngineStats:
     for
       duration <- Ref.of(FiniteDuration(0L, TimeUnit.NANOSECONDS))
       seen     <- Ref.of(0L)
-      succeeded <- Ref.of(0L)
+      passed   <- Ref.of(0L)
       failed   <- Ref.of(0L)
     yield new EngineStats[F](
       overallDuration = duration,
       countSeen = seen,
-      countSucceeded = succeeded,
+      countPassed = passed,
       countFailed = failed
     )
 

modules/runtime/src/main/scala/gs/test/v0/runtime/engine/TestEngine.scala

@@ -2,12 +2,14 @@ package gs.test.v0.runtime.engine
 
 import cats.effect.Async
 import cats.syntax.all.*
+import gs.test.v0.api.GroupResult
 import gs.test.v0.api.SuiteExecution
 import gs.test.v0.api.TestDefinition
 import gs.test.v0.api.TestExecution
 import gs.test.v0.api.TestFailure
 import gs.test.v0.api.TestGroupDefinition
 import gs.test.v0.api.TestSuite
+import gs.test.v0.reporting.Reporter
 import gs.timing.v0.Timing
 import gs.uuid.v0.UUID
 import java.time.Clock
@@ -51,6 +53,7 @@ import natchez.Span
   */
 final class TestEngine[F[_]: Async](
   val configuration: EngineConfiguration,
+  reporter: Reporter[F],
   timing: Timing[F],
   suiteExecutionIdGenerator: UUID.Generator,
   testExecutionIdGenerator: UUID.Generator,
@@ -66,13 +69,36 @@ final class TestEngine[F[_]: Async](
     for
       executedAt <- Async[F].delay(Instant.now(clock))
       stats      <- EngineStats.initialize[F]
+
+      // Start reporting
+      _ <- reporter.beginReporting()
+
       // TODO: Just do telemetry for the whole damn thing.
       _ <- tests
         .mapAsync(configuration.groupConcurrency.toInt())(runGroup)
-        .evalTap(stats.updateForGroup)
+        .evalTap(groupResult =>
+          for
+            // Update the overall statistics based on this group.
+            _ <- stats.updateForGroup(
+              duration = groupResult.duration,
+              testExecutions = groupResult.testExecutions
+            )
+
+            // Report group level results for this group.
+            _ <- reporter.reportGroup(groupResult)
+          yield ()
+        )
         .compile
         .drain
+
+      // Calculate the final summary of execution at the suite level.
       suiteExecution <- makeSuiteExecution(suite, stats, executedAt)
+
+      // Report suite level results.
+      _ <- reporter.reportSuite(suiteExecution)
+
+      // Finish reporting.
+      _ <- reporter.endReporting()
     yield suiteExecution
 
   def runGroup(
@@ -80,6 +106,7 @@ final class TestEngine[F[_]: Async](
   ): F[GroupResult] =
     entryPoint.root(EngineConstants.Tracing.RootSpan).use { rootSpan =>
       for
+        groupStats <- EngineStats.initialize[F]
         // Augment the span with all group-level metadata.
         _ <- rootSpan
           .put(EngineConstants.MetaData.TestGroupName -> group.name.show)
@@ -106,10 +133,24 @@ final class TestEngine[F[_]: Async](
 
         // Calculate the overall elapsed time for this group.
         elapsed <- timer.checkpoint()
+
+        // Calculate group-level statistics.
+        _ <- groupStats.updateForGroup(
+          duration = elapsed.duration,
+          testExecutions = testExecutions
+        )
+
+        // Extract the group statistic values for inclusion in the result..
+        seen   <- groupStats.seen
+        passed <- groupStats.passed
+        failed <- groupStats.failed
       yield new GroupResult(
         name = group.name,
         documentation = group.documentation,
         duration = elapsed.duration,
+        seen = seen,
+        passed = passed,
+        failed = failed,
         testExecutions = testExecutions
       )
     }
@@ -171,7 +212,7 @@ final class TestEngine[F[_]: Async](
           tags = test.tags,
           markers = test.markers,
           result = result,
-          // TODO TraceID isn't that useful here, need SpanID
+          // TODO: TraceID isn't that useful here, need SpanID
           traceId = traceId,
           sourcePosition = test.sourcePosition,
           duration = elapsed.duration
@@ -208,15 +249,14 @@ final class TestEngine[F[_]: Async](
     for
       overallDuration <- stats.duration
       countSeen       <- stats.seen
-      countSucceeded  <- stats.succeeded
+      countPassed     <- stats.passed
       countFailed     <- stats.failed
     yield SuiteExecution(
       id = suiteExecutionIdGenerator.next(),
-      name = suite.name,
+      testSuite = suite,
-      documentation = suite.documentation,
       duration = overallDuration,
       countSeen = countSeen,
-      countSucceeded = countSucceeded,
+      countPassed = countPassed,
       countFailed = countFailed,
       executedAt = executedAt
     )