Working on the engine and API.

build.sbt

@@ -34,8 +34,13 @@ val Deps = new {
     val Core: ModuleID = "co.fs2" %% "fs2-core" % "3.10.2"
   }
 
+  val Natchez = new {
+    val Core: ModuleID = "org.tpolecat" %% "natchez-core" % "0.3.6"
+  }
+
   val Gs = new {
     val Uuid: ModuleID    = "gs" %% "gs-uuid-v0"         % "0.3.0"
+    val Timing: ModuleID  = "gs" %% "gs-timing-v0"       % "0.1.1"
     val Datagen: ModuleID = "gs" %% "gs-datagen-core-v0" % "0.2.0"
   }
 
@@ -69,12 +74,13 @@ lazy val `api-definition` = project
     libraryDependencies ++= Seq(
       Deps.Cats.Core,
       Deps.Cats.Effect,
-      Deps.Fs2.Core
+      Deps.Natchez.Core
     )
   )
 
 lazy val `api-execution` = project
   .in(file("modules/api-execution"))
+  .dependsOn(`api-definition`)
   .settings(sharedSettings)
   .settings(testSettings)
   .settings(
@@ -82,8 +88,11 @@ lazy val `api-execution` = project
   )
   .settings(
     libraryDependencies ++= Seq(
+      Deps.Gs.Uuid,
+      Deps.Gs.Timing,
       Deps.Cats.Core,
       Deps.Cats.Effect,
-      Deps.Fs2.Core
+      Deps.Fs2.Core,
+      Deps.Natchez.Core
     )
   )

modules/api-definition/src/main/scala/gs/test/v0/definition/Assertion.scala

@@ -9,7 +9,9 @@ object Assertion:
 
   private def success(): Either[TestFailure, Unit] = Right(())
 
-  def renderInput[A](value: A): String = ""
+  // TODO: Use a smart rendering solution, consider diffs.
+  // For now, this is fine -- add a diff library later as data.
+  def renderInput[A](value: A): String = value.toString()
 
   case object IsEqualTo extends Assertion("isEqualTo"):
 

modules/api-definition/src/main/scala/gs/test/v0/definition/Check.scala

@@ -4,10 +4,19 @@ import cats.effect.Sync
 import gs.test.v0.definition.pos.SourcePosition
 import scala.reflect.ClassTag
 
+/** Opaque type used to check candidate values against expected values.
+  */
 opaque type Check[A] = A
 
 object Check:
 
+  /** Instantiate a new Check.
+    *
+    * @param candidate
+    *   The value to check.
+    * @return
+    *   The new [[Check]] instance.
+    */
   def apply[A](candidate: A): Check[A] = candidate
 
   extension [A: ClassTag](candidate: Check[A])

modules/api-definition/src/main/scala/gs/test/v0/definition/TestDefinition.scala

@@ -1,7 +1,8 @@
 package gs.test.v0.definition
 
 import cats.Show
-import cats.data.EitherT
+import gs.test.v0.definition.pos.SourcePosition
+import natchez.Trace
 
 /** Each instance of this class indicates the _definition_ of some test.
   *
@@ -19,15 +20,18 @@ import cats.data.EitherT
   *   The number of iterations of this test to run.
   * @param unitOfWork
   *   The function that the test evaluates.
+  * @param sourcePosition
+  *   The location of this test in source code.
   */
-final class TestDefinition[F[_]](
+final class TestDefinition[F[_]: Trace](
   val name: TestDefinition.Name,
   val permanentId: PermanentId,
   val documentation: Option[String],
   val tags: List[Tag],
   val markers: List[Marker],
   val iterations: TestIterations,
-  val unitOfWork: EitherT[F, TestFailure, Unit]
+  val unitOfWork: F[Either[TestFailure, Unit]],
+  val sourcePosition: SourcePosition
 )
 
 object TestDefinition:

modules/api-definition/src/main/scala/gs/test/v0/definition/TestGroup.scala

@@ -3,7 +3,9 @@ package gs.test.v0.definition
 import cats.data.EitherT
 import cats.effect.Async
 import cats.syntax.all.*
+import gs.test.v0.definition.pos.SourcePosition
 import java.util.concurrent.ConcurrentHashMap
+import natchez.Trace
 import scala.collection.mutable.ListBuffer
 import scala.jdk.CollectionConverters.*
 
@@ -25,7 +27,7 @@ import scala.jdk.CollectionConverters.*
   *   }
   * }}}
   */
-abstract class TestGroup[F[_]: Async]:
+abstract class TestGroup[F[_]: Async: Trace]:
   /** @return
     *   The display name for this group.
     */
@@ -137,22 +139,21 @@ abstract class TestGroup[F[_]: Async]:
   protected def test(
     permanentId: PermanentId,
     name: String
+  )(
+    using
+    pos: SourcePosition
   ): TestGroup.TestBuilder[F] =
     new TestGroup.TestBuilder[F](
       registry = registry,
       name = TestDefinition.Name(name),
       permanentId = permanentId,
       tags = ListBuffer(tags*),
-      markers = ListBuffer(markers*)
+      markers = ListBuffer(markers*),
+      pos = pos
     )
 
 object TestGroup:
 
-  /** Specialization of [[TestGroup]] for `cats.effect.IO`, the typical use
-    * case.
-    */
-  abstract class IO extends TestGroup[cats.effect.IO]
-
   /** Builder to assist with defining tests.
     *
     * @param registry
@@ -162,6 +163,8 @@ object TestGroup:
     *   The name of the test.
     * @param permanentId
     *   The [[PermanentId]] of the test.
+    * @param pos
+    *   The [[SourcePosition]] of the test.
     * @param tags
     *   List of [[TestDefinition.Tag]] applicable to this test.
     * @param markers
@@ -171,10 +174,11 @@ object TestGroup:
     * @param iterations
     *   Number of iterations to run this test.
     */
-  final protected class TestBuilder[F[_]: Async](
+  final protected class TestBuilder[F[_]: Async: Trace](
     val registry: Registry[F],
     val name: TestDefinition.Name,
     val permanentId: PermanentId,
+    val pos: SourcePosition,
     private val tags: ListBuffer[Tag],
     private val markers: ListBuffer[Marker],
     private var documentation: Option[String] = None,
@@ -248,6 +252,7 @@ object TestGroup:
         registry = registry,
         name = name,
         permanentId = permanentId,
+        pos = pos,
         inputFunction = f,
         tags = tags,
         markers = markers,
@@ -284,7 +289,8 @@ object TestGroup:
           tags = tags.distinct.toList,
           markers = markers.distinct.toList,
           iterations = iterations,
-          unitOfWork = unitOfWork
+          unitOfWork = unitOfWork.value,
+          sourcePosition = pos
         )
       )
 
@@ -298,6 +304,8 @@ object TestGroup:
     *   The name of the test.
     * @param permanentId
     *   The [[PermanentId]] of the test.
+    * @param pos
+    *   The [[SourcePosition]] of the test.
     * @param inputFunction
     *   The function that provides input to this test.
     * @param tags
@@ -309,10 +317,11 @@ object TestGroup:
     * @param iterations
     *   Number of iterations to run this test.
     */
-  final protected class InputTestBuilder[F[_]: Async, Input](
+  final protected class InputTestBuilder[F[_]: Async: Trace, Input](
     val registry: Registry[F],
     val name: TestDefinition.Name,
     val permanentId: PermanentId,
+    val pos: SourcePosition,
     val inputFunction: F[Input],
     private val tags: ListBuffer[Tag],
     private val markers: ListBuffer[Marker],
@@ -404,7 +413,8 @@ object TestGroup:
           tags = tags.distinct.toList,
           markers = markers.distinct.toList,
           iterations = iterations,
-          unitOfWork = EitherT.right(inputFunction).flatMap(unitOfWork)
+          unitOfWork = EitherT.right(inputFunction).flatMap(unitOfWork).value,
+          sourcePosition = pos
         )
       )
 

modules/api-definition/src/main/scala/gs/test/v0/definition/TestGroupDefinition.scala

@@ -1,6 +1,8 @@
 package gs.test.v0.definition
 
 import cats.Show
+import cats.effect.Async
+import natchez.Trace
 
 /** Each group is comprised of a list of [[Test]]. This list may be empty.
   *
@@ -17,7 +19,7 @@ import cats.Show
   * @param tests
   *   The list of tests in this group.
   */
-final class TestGroupDefinition[F[_]](
+final class TestGroupDefinition[F[_]: Async: Trace](
   val name: TestGroupDefinition.Name,
   val documentation: Option[String],
   val testTags: List[Tag],

modules/api-definition/src/main/scala/gs/test/v0/definition/TestSuite.scala

@@ -4,18 +4,15 @@ package gs.test.v0.definition
   * execution _typically_ runs a single test suite. For example, the unit tests
   * for some project would likely comprise of a single suite.
   *
-  * Within each suite is a list of [[TestGroup]], arbitrary ways to organize
-  * individual [[Test]] definitions.
-  *
+  * @param permanentId
+  *   A permanent identifier used to reference this suite over time.
   * @param name
   *   The name of this test suite.
   * @param documentation
   *   Arbitrary documentation for this suite of tests.
-  * @param groups
-  *   List of [[TestGroup]] owned by this suite.
   */
-case class TestSuite[F[_]](
+case class TestSuite(
+  permanentId: PermanentId,
   name: String,
-  documentation: Option[String],
-  groups: List[TestGroupDefinition[F]]
+  documentation: Option[String]
 )

modules/api-definition/src/test/scala/gs/test/v0/definition/GroupImplementationTests.scala

@@ -1,14 +1,18 @@
 package gs.test.v0.definition
 
+import cats.data.Kleisli
+import cats.effect.Async
 import cats.effect.IO
 import gs.test.v0.definition.{Tag => GsTag}
 import munit.*
+import natchez.Span
+import natchez.Trace
 
 class GroupImplementationTests extends FunSuite:
   import GroupImplementationTests.*
 
   test("should support a group with a simple, pure, test") {
-    val g1    = new G1
+    val g1    = new G1[[A] =>> Kleisli[IO, Span[IO], A]]
     val group = g1.compile()
     assertEquals(group.name, TestGroupDefinition.Name("G1"))
     assertEquals(group.documentation, None)
@@ -32,7 +36,7 @@ class GroupImplementationTests extends FunSuite:
   }
 
   test("should support a group with all values set") {
-    val g2    = new G2
+    val g2    = new G2[[A] =>> Kleisli[IO, Span[IO], A]]
     val group = g2.compile()
     assertEquals(group.name, TestGroupDefinition.Name("G2"))
     assertEquals(group.documentation, Some("docs"))
@@ -56,7 +60,7 @@ class GroupImplementationTests extends FunSuite:
   }
 
   test("should support a simple group with a configured test") {
-    val g3    = new G3
+    val g3    = new G3[[A] =>> Kleisli[IO, Span[IO], A]]
     val group = g3.compile()
     assertEquals(group.name, TestGroupDefinition.Name("G3"))
     assertEquals(group.documentation, None)
@@ -89,12 +93,12 @@ object GroupImplementationTests:
 
   end Ids
 
-  class G1 extends TestGroup.IO:
+  class G1[F[_]: Async: Trace] extends TestGroup[F]:
     override def name: String = "G1"
     test(Ids.T1, "simple").pure(Right(()))
   end G1
 
-  class G2 extends TestGroup.IO:
+  class G2[F[_]: Async: Trace] extends TestGroup[F]:
 
     override def name: String =
       "G2"
@@ -108,15 +112,15 @@ object GroupImplementationTests:
     override def markers: List[Marker] =
       List(Marker.Ignored)
 
-    beforeGroup(IO.unit)
-    afterGroup(IO.unit)
-    beforeEachTest(IO.unit)
-    afterEachTest(IO.unit)
+    beforeGroup(Async[F].unit)
+    afterGroup(Async[F].unit)
+    beforeEachTest(Async[F].unit)
+    afterEachTest(Async[F].unit)
 
     test(Ids.T2, "inherit from group").pure(Right(()))
   end G2
 
-  class G3 extends TestGroup.IO:
+  class G3[F[_]: Async: Trace] extends TestGroup[F]:
     override def name: String = "G3"
 
     test(Ids.T3, "configure test")

modules/api-definition/src/test/scala/gs/test/v0/definition/pos/SourcePositionTests.scala

@@ -1,8 +1,17 @@
 package gs.test.v0.definition.pos
 
+import cats.data.Kleisli
+import cats.effect.IO
+import cats.effect.kernel.Resource
 import gs.test.v0.IOSuite
 import gs.test.v0.definition.*
 import munit.*
+import natchez.EntryPoint
+import natchez.Kernel
+import natchez.Span
+import natchez.Span.Options
+import natchez.Trace
+import natchez.Trace.Implicits.noop
 
 /** These tests are sensitive to changes, even in formatting! They are looking
   * for specific line numbers in this source code, so any sort of newline that
@@ -13,35 +22,64 @@ class SourcePositionTests extends IOSuite:
 
   import SourcePositionTests.*
 
+  val ep: EntryPoint[IO] = new EntryPoint[IO] {
+    override def root(
+      name: String,
+      options: Options
+    ): Resource[IO, Span[IO]] =
+      Resource.make(IO(Span.noop[IO]))(_ => IO.unit)
+
+    override def continue(
+      name: String,
+      kernel: Kernel,
+      options: Options
+    ): Resource[IO, Span[IO]] =
+      Resource.make(IO(Span.noop[IO]))(_ => IO.unit)
+
+    override def continueOrElseRoot(
+      name: String,
+      kernel: Kernel,
+      options: Options
+    ): Resource[IO, Span[IO]] =
+      Resource.make(IO(Span.noop[IO]))(_ => IO.unit)
+  }
+
   test("should provide the source position of a failed check") {
-    lookForSourcePosition(new G1, 46)
+    lookForSourcePosition(new G1, 73)
   }
 
   test("should provide the source position of an explicit failure") {
-    lookForSourcePosition(new G2, 55)
+    lookForSourcePosition(new G2, 82)
   }
 
   private def lookForSourcePosition(
-    groupDef: TestGroup.IO,
+    groupDef: TestGroup[TracedIO],
     line: Int
   ): Unit =
     val group = groupDef.compile()
     group.tests match
       case t1 :: Nil =>
-        t1.unitOfWork.value.unsafeRunSync() match
-          case Left(TestFailure.AssertionFailed(_, _, _, pos)) =>
-            assertEquals(pos.file.endsWith(SourceFileName), true)
-            assertEquals(pos.line, line)
-          case Left(TestFailure.TestRequestedFailure(_, pos)) =>
-            assertEquals(pos.file.endsWith(SourceFileName), true)
-            assertEquals(pos.line, line)
-          case _ => fail("Sub-test did not fail as expected.")
+        ep.root("unit-test")
+          .use { span =>
+            t1.unitOfWork.run(span).map {
+              case Left(TestFailure.AssertionFailed(_, _, _, pos)) =>
+                assertEquals(pos.file.endsWith(SourceFileName), true)
+                assertEquals(pos.line, line)
+              case Left(TestFailure.TestRequestedFailure(_, pos)) =>
+                assertEquals(pos.file.endsWith(SourceFileName), true)
+                assertEquals(pos.line, line)
+              case _ => fail("Sub-test did not fail as expected.")
+            }
+          }
+          .unsafeRunSync()
       case _ =>
         fail("Wrong number of tests - position tests need one test per group.")
 
 object SourcePositionTests:
 
-  class G1 extends TestGroup.IO:
+  type TracedIO[A] = Kleisli[IO, Span[IO], A]
+
+  class G1 extends TestGroup[TracedIO]:
     override def name: String = "G1"
 
     test(pid"t1", "pos").pure {
@@ -50,7 +88,7 @@ object SourcePositionTests:
 
   end G1
 
-  class G2 extends TestGroup.IO:
+  class G2 extends TestGroup[TracedIO]:
     override def name: String = "G2"
 
     test(pid"t2", "pos").pure {

modules/api-execution/src/main/scala/gs/test/v0/execution/SuiteExecution.scala

@@ -0,0 +1,17 @@
+package gs.test.v0.execution
+
+import gs.uuid.v0.UUID
+import java.time.Instant
+import scala.concurrent.duration.FiniteDuration
+
+case class SuiteExecution(
+  id: UUID,
+  name: String,
+  documentation: Option[String],
+  duration: FiniteDuration,
+  countSeen: Int,
+  countSucceeded: Int,
+  countFailed: Int,
+  countIgnored: Int,
+  executedAt: Instant
+)

modules/api-execution/src/main/scala/gs/test/v0/execution/TestExecution.scala

@@ -0,0 +1,83 @@
+package gs.test.v0.execution
+
+import gs.test.v0.definition.Marker
+import gs.test.v0.definition.PermanentId
+import gs.test.v0.definition.Tag
+import gs.test.v0.definition.TestFailure
+import gs.test.v0.definition.pos.SourcePosition
+import gs.uuid.v0.UUID
+import scala.concurrent.duration.FiniteDuration
+
+/** Represents a single _Test Execution_. Each _Test Execution_ represents
+  * evaluating the unit of work for some
+  * [[gs.test.v0.definition.TestDefinition]] exactly once. It describes the
+  * result of the test.
+  *
+  * @param id
+  *   The unique identifier for this execution.
+  * @param permanentId
+  *   The [[gs.test.v0.definition.PermanentId]] for the
+  *   [[gs.test.v0.definition.TestDefinition]] that was executed.
+  * @param documentation
+  *   Documentation for the test that was executed.
+  * @param tags
+  *   Tags for the test that was executed.
+  * @param markers
+  *   Markers for the test that was executed.
+  * @param result
+  *   The result of the test.
+  * @param traceId
+  *   The 128-bit trace identifier used for this test.
+  * @param sourcePosition
+  *   The position, in source code, of the test that was executed.
+  * @param duration
+  *   The amount of time it took to execute the test. This amount includes setup
+  *   and cleanup.
+  */
+case class TestExecution(
+  id: TestExecution.Id,
+  permanentId: PermanentId,
+  documentation: Option[String],
+  tags: List[Tag],
+  markers: List[Marker],
+  result: Either[TestFailure, Unit],
+  traceId: UUID,
+  sourcePosition: SourcePosition,
+  duration: FiniteDuration
+)
+
+object TestExecution:
+
+  /** Opaque type for UUID representing a unique test execution.
+    */
+  opaque type Id = UUID
+
+  object Id:
+
+    /** Instantiate a new [[TestExecution.Id]].
+      *
+      * @param value
+      *   The underlying UUID.
+      * @return
+      *   The new [[TestExecution.Id]] instance.
+      */
+    def apply(value: UUID): Id = value
+
+    given UUID.Generator = UUID.Generator.version7()
+
+    given CanEqual[Id, Id] = CanEqual.derived
+
+    /** @return
+      *   New ID based on a UUIDv7.
+      */
+    def generate(): Id = UUID.generate()
+
+    extension (id: Id)
+      /** @return
+        *   The underlying UUID.
+        */
+      def toUUID(): UUID = id
+
+  end Id
+
+end TestExecution

modules/api-execution/src/main/scala/gs/test/v0/execution/engine/ConcurrencySetting.scala

@@ -0,0 +1,16 @@
+package gs.test.v0.execution.engine
+
+sealed abstract class ConcurrencySetting(val name: String):
+  def toInt(): Int
+
+object ConcurrencySetting:
+
+  case object Serial extends ConcurrencySetting("serial"):
+    override def toInt(): Int = 1
+
+  case class Concurrent(
+    maximum: MaximumConcurrency
+  ) extends ConcurrencySetting("concurrent"):
+    override def toInt(): Int = maximum.toInt()
+
+end ConcurrencySetting

modules/api-execution/src/main/scala/gs/test/v0/execution/engine/EngineConfiguration.scala

@@ -0,0 +1,5 @@
+package gs.test.v0.execution.engine
+
+case class EngineConfiguration(
+  testConcurrency: ConcurrencySetting
+)

modules/api-execution/src/main/scala/gs/test/v0/execution/engine/EngineResult.scala

@@ -0,0 +1,9 @@
+package gs.test.v0.execution.engine
+
+import gs.test.v0.execution.SuiteExecution
+import gs.test.v0.execution.TestExecution
+
+final class EngineResult[F[_]](
+  val suiteExecution: SuiteExecution,
+  val testExecutions: fs2.Stream[F, TestExecution]
+)

modules/api-execution/src/main/scala/gs/test/v0/execution/engine/GroupResult.scala

@@ -0,0 +1,12 @@
+package gs.test.v0.execution.engine
+
+import gs.test.v0.definition.TestGroupDefinition
+import gs.test.v0.execution.TestExecution
+import scala.concurrent.duration.FiniteDuration
+
+final class GroupResult(
+  val name: TestGroupDefinition.Name,
+  val documentation: Option[String],
+  val duration: FiniteDuration,
+  val testExecutions: List[TestExecution]
+)

modules/api-execution/src/main/scala/gs/test/v0/execution/engine/MaximumConcurrency.scala

@@ -0,0 +1,18 @@
+package gs.test.v0.execution.engine
+
+opaque type MaximumConcurrency = Int
+
+object MaximumConcurrency:
+
+  def apply(candidate: Int): MaximumConcurrency =
+    if candidate <= 0 then
+      throw new IllegalArgumentException(
+        "Maximum concurrency must be 1 or greater."
+      )
+    else candidate
+
+  given CanEqual[MaximumConcurrency, MaximumConcurrency] = CanEqual.derived
+
+  extension (mc: MaximumConcurrency) def toInt(): Int = mc
+
+end MaximumConcurrency

modules/api-execution/src/main/scala/gs/test/v0/execution/engine/TestEngine.scala

@@ -0,0 +1,72 @@
+package gs.test.v0.execution.engine
+
+import cats.effect.Async
+import cats.syntax.all.*
+import gs.test.v0.definition.TestGroupDefinition
+import gs.test.v0.definition.TestSuite
+import gs.test.v0.execution.TestExecution
+import gs.timing.v0.Timing
+import natchez.EntryPoint
+
+final class TestEngine[F[_]: Async](
+  val configuration: EngineConfiguration,
+  timing: Timing[F],
+  val entryPoint: EntryPoint[F]
+):
+
+  def runSuite(
+    suite: TestSuite,
+    tests: fs2.Stream[F, TestGroupDefinition[F]]
+  ): EngineResult[F] =
+    EngineResult[F](
+      suiteExecution = ???,
+      testExecutions = tests.mapAsync(4)(group => runGroup(group)).map(_ => ???)
+    )
+
+  def runGroup(
+    group: TestGroupDefinition[F]
+  ): F[GroupResult] =
+    for
+      _      <- group.beforeGroup.getOrElse(Async[F].unit)
+      stream <- executeGroupTests(group)
+      _      <- group.afterGroup.getOrElse(Async[F].unit)
+    yield stream
+
+  private def executeGroupTests(group: TestGroupDefinition[F]): F[GroupResult] =
+    for
+      timer      <- timing.start()
+      _          <- group.beforeGroup.getOrElse(Async[F].unit)
+      executions <- streamGroupTests(group).compile.toList
+      _          <- group.afterGroup.getOrElse(Async[F].unit)
+      elapsed    <- timer.checkpoint()
+    yield new GroupResult(
+      name = group.name,
+      documentation = group.documentation,
+      duration = elapsed.duration,
+      testExecutions = executions
+    )
+
+  private def streamGroupTests(group: TestGroupDefinition[F])
+    : fs2.Stream[F, TestExecution] =
+    fs2.Stream
+      .emits(group.tests)
+      .mapAsync(configuration.testConcurrency.toInt()) { test =>
+        for
+          testExecutionId <- Async[F].delay(TestExecution.Id.generate())
+          timer           <- timing.start()
+          _               <- group.beforeEachTest.getOrElse(Async[F].unit)
+          result          <- test.unitOfWork
+          _               <- group.afterEachTest.getOrElse(Async[F].unit)
+          elapsed         <- timer.checkpoint()
+        yield TestExecution(
+          id = testExecutionId,
+          permanentId = test.permanentId,
+          documentation = test.documentation,
+          tags = test.tags,
+          markers = test.markers,
+          result = result,
+          traceId = ???,
+          sourcePosition = test.sourcePosition,
+          duration = elapsed.duration
+        )
+      }