WIP: writing tests for the runtime and improving test data

build.sbt

@@ -1,4 +1,4 @@
-val scala3: String = "3.7.1"
+val scala3: String = "3.7.2"
 
 ThisBuild / scalaVersion  := scala3
 ThisBuild / versionScheme := Some("semver-spec")
@@ -101,6 +101,24 @@ lazy val api = project
     )
   )
 
+/** Internal project used for generating test data.
+  */
+lazy val `test-data` = project
+  .in(file("modules/test-data"))
+  .dependsOn(api)
+  .settings(sharedSettings)
+  .settings(testSettings)
+  .settings(noPublishSettings)
+  .settings(
+    name := s"${gsProjectName.value}-test-data"
+  )
+  .settings(
+    libraryDependencies ++= Seq(
+      Deps.Cats.Core,
+      Deps.Cats.Effect
+    )
+  )
+
 /** Reporting API and implementations.
   */
 lazy val reporting = project
@@ -123,6 +141,7 @@ lazy val reporting = project
 lazy val runtime = project
   .in(file("modules/runtime"))
   .dependsOn(`test-support` % "test->test")
+  .dependsOn(`test-data` % "test->test")
   .dependsOn(api, reporting)
   .settings(sharedSettings)
   .settings(testSettings)

modules/reporting/src/main/scala/gs/test/v0/reporting/InMemoryReporter.scala

@@ -0,0 +1,50 @@
+package gs.test.v0.reporting
+
+import cats.effect.Async
+import cats.effect.Ref
+import cats.effect.std.Queue
+import cats.syntax.all.*
+import gs.test.v0.api.GroupResult
+import gs.test.v0.api.SuiteExecution
+import gs.test.v0.api.TestExecution
+
+final class InMemoryReporter[F[_]: Async] private (
+  suiteExecution: Ref[F, Option[SuiteExecution]],
+  groupResults: Queue[F, (GroupResult, List[TestExecution])]
+) extends Reporter[F]:
+
+  /** @inheritDocs
+    */
+  override def startReport(): F[Unit] = Async[F].unit
+
+  /** @inheritDocs
+    */
+  override def reportGroup(
+    groupResult: GroupResult,
+    testExecutions: List[TestExecution]
+  ): F[Unit] = groupResults.offer(groupResult -> testExecutions)
+
+  /** @inheritDocs
+    */
+  override def reportSuite(suiteExecution: SuiteExecution): F[Unit] =
+    this.suiteExecution.set(Some(suiteExecution))
+
+  /** @inheritDocs
+    */
+  override def endReport(): F[Unit] = Async[F].unit
+
+  def getSuiteExecution(): F[Option[SuiteExecution]] = suiteExecution.get
+
+  // TODO: make stream to consume and reify to list
+  def getGroupResults(): F[List[(GroupResult, List[TestExecution])]] =
+    ???
+
+object InMemoryReporter:
+
+  def initialize[F[_]: Async]: F[InMemoryReporter[F]] =
+    for
+      se <- Ref.of[F, Option[SuiteExecution]](None)
+      gr <- Queue.unbounded[F, (GroupResult, List[TestExecution])]
+    yield new InMemoryReporter(se, gr)
+
+end InMemoryReporter

modules/runtime/src/main/scala/gs/test/v0/runtime/engine/TestEngine.scala

@@ -60,6 +60,18 @@ final class TestEngine[F[_]: Async](
   private def testIdGen  = configuration.testIdGenerator
   private def suiteIdGen = configuration.suiteIdGenerator
 
+  /** Execute a suite of tests.
+    *
+    * This function only provides a summary output. Results are streamed using a
+    * [[Reporter]] instance.
+    *
+    * @param suite
+    *   The metadata that describes the suite.
+    * @param tests
+    *   The stream of groups that define the tests.
+    * @return
+    *   Summary of the execution.
+    */
   def runSuite(
     suite: TestSuite,
     tests: fs2.Stream[F, TestGroupDefinition[F]]

modules/runtime/src/test/scala/gs/test/v0/runtime/engine/EngineStatsTests.scala

@@ -0,0 +1,77 @@
+package gs.test.v0.runtime.engine
+
+import cats.effect.IO
+import gs.datagen.v0.Gen
+import gs.datagen.v0.generators.Size
+import java.util.concurrent.TimeUnit
+import munit.*
+import scala.concurrent.duration.FiniteDuration
+import support.*
+
+class EngineStatsTests extends IOSuite:
+
+  iotest("should initialize empty stats") {
+    for
+      stats    <- EngineStats.initialize[IO]
+      duration <- stats.duration
+      seen     <- stats.seen
+      passed   <- stats.passed
+      failed   <- stats.failed
+    yield
+      assertEquals(duration, Durations.Zero)
+      assertEquals(seen, 0L)
+      assertEquals(passed, 0L)
+      assertEquals(failed, 0L)
+  }
+
+  iotest("should update based on a group with no test executions") {
+    val expected = FiniteDuration(2L, TimeUnit.MILLISECONDS)
+    for
+      stats    <- EngineStats.initialize[IO]
+      _        <- stats.updateForGroup(Durations.OneMilli, Nil)
+      _        <- stats.updateForGroup(Durations.OneMilli, Nil)
+      duration <- stats.duration
+      seen     <- stats.seen
+      passed   <- stats.passed
+      failed   <- stats.failed
+    yield
+      assertEquals(duration, expected)
+      assertEquals(seen, 0L)
+      assertEquals(passed, 0L)
+      assertEquals(failed, 0L)
+  }
+
+  iotest("should update based on a single test execution") {
+    for
+      stats    <- EngineStats.initialize[IO]
+      _        <- stats.updateForTest(Generators.testExecutionPassed())
+      _        <- stats.updateForTest(Generators.testExecutionFailed())
+      duration <- stats.duration
+      seen     <- stats.seen
+      passed   <- stats.passed
+      failed   <- stats.failed
+    yield
+      assertEquals(duration, Durations.Zero)
+      assertEquals(seen, 2L)
+      assertEquals(passed, 1L)
+      assertEquals(failed, 1L)
+  }
+
+  iotest("should update based on a test group") {
+    val duration   = Generators.testDuration()
+    val size       = 4
+    val executions =
+      Gen.list(Size.fixed(size), Generators.GenTestExecutionPassed).gen()
+    for
+      stats    <- EngineStats.initialize[IO]
+      _        <- stats.updateForGroup(duration, executions)
+      duration <- stats.duration
+      seen     <- stats.seen
+      passed   <- stats.passed
+      failed   <- stats.failed
+    yield
+      assertEquals(duration, duration)
+      assertEquals(seen, size.toLong)
+      assertEquals(passed, size.toLong)
+      assertEquals(failed, 0L)
+  }

modules/test-data/src/test/scala/support/Generators.scala

@@ -0,0 +1,99 @@
+package support
+
+import gs.datagen.v0.*
+import gs.datagen.v0.generators.Size
+import gs.test.v0.api.*
+import scala.concurrent.duration.FiniteDuration
+
+object Generators:
+
+  val NoSourcePosition: SourcePosition = SourcePosition("TEST", 0)
+
+  val GenTestExecutionId: Gen[TestExecution.Id] =
+    Gen.uuid.random().map(TestExecution.Id(_))
+
+  given Generated[TestExecution.Id] = Generated.of(GenTestExecutionId)
+
+  val GenPermanentId: Gen[PermanentId] =
+    Gen.string.alphaNumeric(Size.Fixed(12)).map(x => PermanentId(s"pid-$x"))
+
+  given Generated[PermanentId] = Generated.of(GenPermanentId)
+
+  val GenTag: Gen[Tag] =
+    Gen.string.alphaNumeric(Size.Fixed(6)).map(x => Tag(s"tag-$x"))
+
+  val GenTagList: Gen[List[Tag]] = Gen.list(Size.between(0, 8), GenTag)
+
+  given Generated[Tag] = Generated.of(GenTag)
+
+  val GenTraceId: Gen[String] = Gen.uuid.string().map(_.filterNot(_ == '-'))
+
+  val GenSpanId: Gen[String] = GenTraceId.map(_.take(16))
+
+  val GenTestDuration: Gen[FiniteDuration] =
+    Gen.duration.finiteMilliseconds(1L, 100L)
+
+  val GenTestResult: Gen[Either[TestFailure, Any]] =
+    Gen.boolean().map(makeResult)
+
+  private def makeResult(passed: Boolean): TestResult =
+    passed match {
+      case true  => Right(())
+      case false =>
+        Left(TestFailure.TestRequestedFailure("Failed", NoSourcePosition))
+    }
+
+  val InputGenTestExecution: Datagen[TestExecution, Boolean] =
+    for
+      id          <- GenTestExecutionId
+      permanentId <- GenPermanentId
+      tags        <- GenTagList
+      spanId      <- GenSpanId
+      duration    <- GenTestDuration
+    yield (passed: Boolean) =>
+      TestExecution(
+        id = id,
+        permanentId = permanentId,
+        documentation = None,
+        tags = tags,
+        markers = Nil,
+        result = makeResult(passed),
+        spanId = spanId,
+        sourcePosition = NoSourcePosition,
+        duration = duration
+      )
+
+  val GenTestExecution: Gen[TestExecution] =
+    Gen.boolean().map(passed => InputGenTestExecution.generate(passed))
+
+  val GenTestExecutionPassed: Gen[TestExecution] =
+    Gen.single(true).map(InputGenTestExecution.generate)
+
+  val GenTestExecutionFailed: Gen[TestExecution] =
+    Gen.single(false).map(InputGenTestExecution.generate)
+
+  given Generated[TestExecution] = Generated.of(GenTestExecution)
+
+  def testExecutionId(): TestExecution.Id = GenTestExecutionId.gen()
+
+  def permanentId(): PermanentId = GenPermanentId.gen()
+
+  def tag(): Tag = GenTag.gen()
+
+  def tags(): List[Tag] = GenTagList.gen()
+
+  def traceId(): String = GenTraceId.gen()
+
+  def spanId(): String = GenSpanId.gen()
+
+  def testDuration(): FiniteDuration = GenTestDuration.gen()
+
+  def testExecution(): TestExecution = GenTestExecution.gen()
+
+  def testExecutionPassed(): TestExecution =
+    InputGenTestExecution.generate(true)
+
+  def testExecutionFailed(): TestExecution =
+    InputGenTestExecution.generate(false)
+
+end Generators

modules/test-support/src/test/scala/support/Durations.scala

@@ -0,0 +1,11 @@
+package support
+
+import java.util.concurrent.TimeUnit
+import scala.concurrent.duration.FiniteDuration
+
+object Durations:
+
+  val Zero: FiniteDuration     = FiniteDuration(0L, TimeUnit.NANOSECONDS)
+  val OneMilli: FiniteDuration = FiniteDuration(1L, TimeUnit.MILLISECONDS)
+
+end Durations

project/build.properties

@@ -1 +1 @@
-sbt.version=1.11.2
+sbt.version=1.11.6