Compare commits

...
Sign in to create a new pull request.

12 commits

34 changed files with 1608 additions and 272 deletions

View file

@ -1,5 +1,5 @@
// See: https://github.com/scalameta/scalafmt/tags for the latest tags.
version = 3.8.1
version = 3.9.9
runner.dialect = scala3
maxColumn = 80

View file

@ -1,4 +1,4 @@
val scala3: String = "3.5.1"
val scala3: String = "3.7.3"
ThisBuild / scalaVersion := scala3
ThisBuild / versionScheme := Some("semver-spec")
@ -10,7 +10,7 @@ ThisBuild / externalResolvers := Seq(
)
ThisBuild / licenses := Seq(
"MIT" -> url("https://garrity.co/MIT.html")
"MIT" -> url("https://git.garrity.co/garrity-software/gs-test/LICENSE")
)
val noPublishSettings = Seq(
@ -26,25 +26,25 @@ val sharedSettings = Seq(
val Deps = new {
val Cats = new {
val Core: ModuleID = "org.typelevel" %% "cats-core" % "2.12.0"
val Effect: ModuleID = "org.typelevel" %% "cats-effect" % "3.5.4"
val Core: ModuleID = "org.typelevel" %% "cats-core" % "2.13.0"
val Effect: ModuleID = "org.typelevel" %% "cats-effect" % "3.6.3"
}
val Fs2 = new {
val Core: ModuleID = "co.fs2" %% "fs2-core" % "3.10.2"
val Core: ModuleID = "co.fs2" %% "fs2-core" % "3.12.0"
}
val Natchez = new {
val Core: ModuleID = "org.tpolecat" %% "natchez-core" % "0.3.6"
val Core: ModuleID = "org.tpolecat" %% "natchez-core" % "0.3.8"
}
val Gs = new {
val Uuid: ModuleID = "gs" %% "gs-uuid-v0" % "0.3.0"
val Timing: ModuleID = "gs" %% "gs-timing-v0" % "0.1.1"
val Datagen: ModuleID = "gs" %% "gs-datagen-core-v0" % "0.2.0"
val Uuid: ModuleID = "gs" %% "gs-uuid-v0" % "0.4.1"
val Timing: ModuleID = "gs" %% "gs-timing-v0" % "0.1.2"
val Datagen: ModuleID = "gs" %% "gs-datagen-core-v0" % "0.3.3"
}
val MUnit: ModuleID = "org.scalameta" %% "munit" % "1.0.1"
val MUnit: ModuleID = "org.scalameta" %% "munit" % "1.1.1"
}
lazy val testSettings = Seq(
@ -59,11 +59,14 @@ lazy val `gs-test` = project
.aggregate(
`test-support`,
api,
reporting,
runtime
)
.settings(noPublishSettings)
.settings(name := s"${gsProjectName.value}-v${semVerMajor.value}")
/** Internal project used for unit tests.
*/
lazy val `test-support` = project
.in(file("modules/test-support"))
.settings(sharedSettings)
@ -75,10 +78,13 @@ lazy val `test-support` = project
.settings(
libraryDependencies ++= Seq(
Deps.Cats.Core,
Deps.Cats.Effect
Deps.Cats.Effect,
Deps.Natchez.Core
)
)
/** Core API - the only dependency needed to write tests.
*/
lazy val api = project
.in(file("modules/api"))
.dependsOn(`test-support` % "test->test")
@ -89,16 +95,55 @@ lazy val api = project
)
.settings(
libraryDependencies ++= Seq(
Deps.Gs.Uuid,
Deps.Cats.Core,
Deps.Cats.Effect,
Deps.Natchez.Core
)
)
/** Internal project used for generating test data.
*/
lazy val `test-data` = project
.in(file("modules/test-data"))
.dependsOn(api)
.settings(sharedSettings)
.settings(testSettings)
.settings(noPublishSettings)
.settings(
name := s"${gsProjectName.value}-test-data"
)
.settings(
libraryDependencies ++= Seq(
Deps.Cats.Core,
Deps.Cats.Effect
)
)
/** Reporting API and implementations.
*/
lazy val reporting = project
.in(file("modules/reporting"))
.dependsOn(`test-support` % "test->test")
.dependsOn(api)
.settings(sharedSettings)
.settings(testSettings)
.settings(
name := s"${gsProjectName.value}-reporting-v${semVerMajor.value}"
)
.settings(
libraryDependencies ++= Seq(
Deps.Fs2.Core
)
)
/** Runtime - the dependency needed to _run_ tests.
*/
lazy val runtime = project
.in(file("modules/runtime"))
.dependsOn(`test-support` % "test->test")
.dependsOn(api)
.dependsOn(`test-data` % "test->test")
.dependsOn(api, reporting)
.settings(sharedSettings)
.settings(testSettings)
.settings(
@ -106,7 +151,6 @@ lazy val runtime = project
)
.settings(
libraryDependencies ++= Seq(
Deps.Gs.Uuid,
Deps.Gs.Timing,
Deps.Cats.Core,
Deps.Cats.Effect,

View file

@ -1,62 +0,0 @@
package gs.test.v0.api
import scala.reflect.*
sealed abstract class Assertion(val name: String)
object Assertion:
private def success(): Either[TestFailure, Unit] = Right(())
// TODO: Use a smart rendering solution, consider diffs.
// For now, this is fine -- add a diff library later as data.
def renderInput[A](value: A): String = value.toString()
case object IsEqualTo extends Assertion("isEqualTo"):
def evaluate[A: ClassTag](
candidate: A,
expected: A
)(
using
CanEqual[A, A]
)(
using
pos: SourcePosition
): Either[TestFailure, Unit] =
if candidate == expected then success()
else
val runtimeType = classTag[A].runtimeClass.getName()
Left(
TestFailure.AssertionFailed(
assertionName = name,
inputs = Map(
"candidate" -> runtimeType,
"expected" -> runtimeType
),
message =
s"'${renderInput(candidate)}' was not equal to '${renderInput(candidate)}'",
pos = pos
)
)
case object IsTrue extends Assertion("isTrue"):
def evaluate(
candidate: Boolean
)(
using
pos: SourcePosition
): Either[TestFailure, Unit] =
if candidate then success()
else
Left(
TestFailure.AssertionFailed(
assertionName = name,
inputs = Map("candidate" -> "Boolean"),
message = s"Expected '$candidate' to be 'true'.",
pos = pos
)
)
end Assertion

View file

@ -1,9 +1,11 @@
package gs.test.v0.api
import cats.data.EitherT
import cats.effect.Sync
import scala.reflect.*
import scala.reflect.ClassTag
/** Opaque type used to check candidate values against expected values.
/** Opaque type used to check obtained values against expected values.
*/
opaque type Check[A] = A
@ -11,18 +13,38 @@ object Check:
/** Instantiate a new Check.
*
* @param candidate
* @param obtained
* The value to check.
* @return
* The new [[Check]] instance.
*/
def apply[A](candidate: A): Check[A] = candidate
def apply[A](obtained: A): Check[A] = obtained
extension [A: ClassTag](candidate: Check[A])
private def testPassed(): TestResult = Right(())
private def render[A](value: A): String = value.toString()
private def assertionFailed(
assertionName: String,
inputs: Map[String, String],
message: String
)(
using
pos: SourcePosition
): TestResult = Left(
TestFailure.AssertionFailed(
assertionName = assertionName,
inputs = inputs,
message = message,
pos = pos
)
)
extension [A](obtained: Check[A])
/** @return
* The unwrapped value of this [[Check]].
*/
def unwrap(): A = candidate
def unwrap(): A = obtained
/** ## Usage
*
@ -34,7 +56,7 @@ object Check:
* @param expected
* The expected value.
* @return
* Successful test result if the candidate value is equal to the expected
* Successful test result if the obtained value is equal to the expected
* value, an error describing the test failure otherwise.
*/
def isEqualTo(
@ -42,9 +64,21 @@ object Check:
)(
using
CanEqual[A, A],
ClassTag[A],
SourcePosition
): TestResult =
Assertion.IsEqualTo.evaluate(candidate, expected)
if obtained == expected then testPassed()
else
val runtimeType = classTag[A].runtimeClass.getName()
assertionFailed(
assertionName = "isEqualTo",
inputs = Map(
"obtained" -> runtimeType,
"expected" -> runtimeType
),
message =
s"'${render(obtained)}' was not equal to '${render(expected)}'"
)
/** ## Usage
*
@ -59,19 +93,30 @@ object Check:
* The expected value.
* @return
* Effect that when evaluated will produce: successful test result if the
* candidate value is equal to the expected value, an error describing
* the test failure otherwise.
* obtained value is equal to the expected value, an error describing the
* test failure otherwise.
*/
def isEqualToF[F[_]: Sync](
expected: A
)(
using
CanEqual[A, A],
ClassTag[A],
SourcePosition
): F[TestResult] =
Sync[F].delay(isEqualTo(expected))
extension (check: Check[Boolean])
def isEqualToT[F[_]: Sync](
expected: A
)(
using
CanEqual[A, A],
ClassTag[A],
SourcePosition
): EitherT[F, TestFailure, Any] =
EitherT(Sync[F].delay(isEqualTo(expected)))
extension (obtained: Check[Boolean])
/** ## Usage
*
@ -81,7 +126,7 @@ object Check:
* }}}
*
* @return
* Successful test result if the candidate value is `true`, an error
* Successful test result if the obtained value is `true`, an error
* describing the test failure otherwise.
*/
def isTrue(
@ -89,7 +134,13 @@ object Check:
using
SourcePosition
): TestResult =
Assertion.IsTrue.evaluate(check)
if obtained then testPassed()
else
assertionFailed(
assertionName = "isTrue",
inputs = Map("obtained" -> "Boolean"),
message = s"Expected '$obtained' to be 'true'."
)
/** ## Usage
*
@ -102,7 +153,7 @@ object Check:
*
* @return
* Effect that when evaluated will produce: successful test result if the
* candidate value is `true`, an error describing the test failure
* obtained value is `true`, an error describing the test failure
* otherwise.
*/
def isTrueF[F[_]: Sync](
@ -112,4 +163,11 @@ object Check:
): F[TestResult] =
Sync[F].delay(isTrue())
def isTrueT[F[_]: Sync](
)(
using
SourcePosition
): EitherT[F, TestFailure, Any] =
EitherT(Sync[F].delay(isTrue()))
end Check

View file

@ -0,0 +1,28 @@
package gs.test.v0.api
import scala.concurrent.duration.FiniteDuration
/** Represents the results of executing an entire group of tests.
*
* @param name
* The name of the executed group.
* @param documentation
* The documentation for the group.
* @param duration
* The overall duration of execution.
* @param seen
* The number of tests seen.
* @param passed
* The number of tests which passed.
* @param failed
* The number of tests which failed.
*/
final class GroupResult(
val name: TestGroupDefinition.Name,
val documentation: Option[String],
val duration: FiniteDuration,
val seen: Long,
val passed: Long,
val failed: Long
):
def millis: Long = duration.toMillis

View file

@ -0,0 +1,36 @@
package gs.test.v0.api
import gs.uuid.v0.UUID
import java.time.Instant
import scala.concurrent.duration.FiniteDuration
/** Describes the overall result of execution a suite of tests.
*
* @param id
* Unique identifier for this execution.
* @param testSuite
* Suite-level identifiers and metadata.
* @param traceId
* The 128-bit trace identifier used for this suite.
* @param duration
* Overall amount of time it took to execute the suite.
* @param seen
* Overall number of tests seen.
* @param passed
* Overall number of passed tests.
* @param failed
* Overall number of failed tests.
* @param executedAt
* Timestamp at which this suite was executed.
*/
case class SuiteExecution(
id: UUID,
testSuite: TestSuite,
traceId: String,
duration: FiniteDuration,
seen: Long,
passed: Long,
failed: Long,
executedAt: Instant
):
def millis: Long = duration.toMillis

View file

@ -1,11 +1,6 @@
package gs.test.v0.runtime
package gs.test.v0.api
import cats.Show
import gs.test.v0.api.Marker
import gs.test.v0.api.PermanentId
import gs.test.v0.api.SourcePosition
import gs.test.v0.api.Tag
import gs.test.v0.api.TestFailure
import gs.uuid.v0.UUID
import scala.concurrent.duration.FiniteDuration
@ -26,8 +21,8 @@ import scala.concurrent.duration.FiniteDuration
* Markers for the test that was executed.
* @param result
* The result of the test.
* @param traceId
* The 128-bit trace identifier used for this test.
* @param spanId
* The 64-bit span identifier used for this test (including before/after).
* @param sourcePosition
* The position, in source code, of the test that was executed.
* @param duration
@ -41,10 +36,23 @@ case class TestExecution(
tags: List[Tag],
markers: List[Marker],
result: Either[TestFailure, Any],
traceId: UUID,
spanId: String,
sourcePosition: SourcePosition,
duration: FiniteDuration
)
):
/** @return
* The string "passed" if the test passed, and "failed" otherwise.
*/
def textResult: String = result match {
case Left(_) => "failed"
case Right(_) => "passed"
}
/** @return
* The duration, in milliseconds, it took this test to execute.
*/
def millis: Long = duration.toMillis
object TestExecution:

View file

@ -2,7 +2,8 @@ package gs.test.v0.api
/** Base trait for all failures recognized by gs-test.
*/
sealed trait TestFailure
sealed trait TestFailure:
def message: String
object TestFailure:
@ -44,6 +45,7 @@ object TestFailure:
*/
case class ExceptionThrown(
cause: Throwable
) extends TestFailure
) extends TestFailure:
override def message: String = cause.getMessage()
end TestFailure

View file

@ -1,7 +1,6 @@
package gs.test.v0.api
import cats.Show
import cats.effect.Async
/** Each group is comprised of a list of [[Test]]. This list may be empty.
*
@ -18,7 +17,7 @@ import cats.effect.Async
* @param tests
* The list of tests in this group.
*/
final class TestGroupDefinition[F[_]: Async](
final class TestGroupDefinition[F[_]](
val name: TestGroupDefinition.Name,
val documentation: Option[String],
val testTags: List[Tag],

View file

@ -180,4 +180,4 @@ def checkAllF[F[_]: Sync](
}
}
def check[A](candidate: A): Check[A] = Check(candidate)
def check[A](obtained: A): Check[A] = Check(obtained)

View file

@ -132,12 +132,4 @@ object GroupImplementationTests:
end G3
class G4[F[_]: Async] extends TestGroup[F]:
override def name: String = "G4"
// TODO: Make test entrypoint and test Trace[F]
test(Ids.T4, "Effectful test").effectful {
???
}
end GroupImplementationTests

View file

@ -0,0 +1,80 @@
package gs.test.v0.reporting
import cats.effect.Async
import cats.effect.Ref
import cats.effect.Resource
import cats.effect.std.Queue
import cats.syntax.all.*
import gs.test.v0.api.GroupResult
import gs.test.v0.api.SuiteExecution
import gs.test.v0.api.TestExecution
/** Reporter that collects all results into memory.
*
* ### Lifecycle
*
* This reporter is intended to be used to get results _exactly once_, after
* the engine has completed its work. The function `terminateAndGetResults`
* will remove all results from the internal collection.
*
* @param suiteExecution
* Collector for the [[SuiteExecution]].
* @param groupResults
* Collector for grouped test results.
*/
final class InMemoryReporter[F[_]: Async] private (
suiteExecution: Ref[F, Option[SuiteExecution]],
groupResults: Queue[F, Option[(GroupResult, List[TestExecution])]]
) extends Reporter[F]:
/** @inheritDocs
*/
override def startReport(): F[Unit] = Async[F].unit
/** @inheritDocs
*/
override def reportGroup(
groupResult: GroupResult,
testExecutions: List[TestExecution]
): F[Unit] = groupResults.offer(Some(groupResult -> testExecutions))
/** @inheritDocs
*/
override def reportSuite(suiteExecution: SuiteExecution): F[Unit] =
this.suiteExecution.set(Some(suiteExecution))
/** @inheritDocs
*/
override def endReport(): F[Unit] = groupResults.offer(None)
/** @return
* The recorded [[SuiteExecution]]. This will be empty unless `reportSuite`
* was called.
*/
def getSuiteExecution(): F[Option[SuiteExecution]] = suiteExecution.get
/** Immediatelly call `endReport` (to ensure terminal state) and collect all
* recorded results. This is a destructive call that will remove the results
* from this reporter.
*
* @return
* The list of extracted results.
*/
def terminateAndGetResults(): F[List[(GroupResult, List[TestExecution])]] =
endReport() *>
fs2.Stream.fromQueueNoneTerminated(groupResults).compile.toList
object InMemoryReporter:
/** @return
* Resource that provides a new [[InMemoryReporter]].
*/
def provision[F[_]: Async]: Resource[F, InMemoryReporter[F]] =
Resource.make(
for
se <- Ref.of[F, Option[SuiteExecution]](None)
gr <- Queue.unbounded[F, Option[(GroupResult, List[TestExecution])]]
yield new InMemoryReporter(se, gr)
)(_ => Async[F].unit)
end InMemoryReporter

View file

@ -0,0 +1,30 @@
package gs.test.v0.reporting
import cats.Applicative
import gs.test.v0.api.GroupResult
import gs.test.v0.api.SuiteExecution
import gs.test.v0.api.TestExecution
/** No-op implementation of [[Reporter]].
*/
final class NoopReporter[F[_]: Applicative] extends Reporter[F]:
/** @inheritDocs
*/
override def startReport(): F[Unit] = Applicative[F].unit
/** @inheritDocs
*/
override def reportGroup(
groupResult: GroupResult,
testExecutions: List[TestExecution]
): F[Unit] = Applicative[F].unit
/** @inheritDocs
*/
override def reportSuite(suiteExecution: SuiteExecution): F[Unit] =
Applicative[F].unit
/** @inheritDocs
*/
override def endReport(): F[Unit] = Applicative[F].unit

View file

@ -0,0 +1,27 @@
package gs.test.v0.reporting
import gs.test.v0.api.GroupResult
import gs.test.v0.api.SuiteExecution
import gs.test.v0.api.TestExecution
final class NoopResultFormatter extends ResultFormatter:
/** @inheritDocs
*/
override def prefix: String = ""
/** @inheritDocs
*/
override def suffix: String = ""
/** @inheritDocs
*/
override def formatGroupResult(groupResult: GroupResult): String = ""
/** @inheritDocs
*/
override def formatTestExecution(testExecution: TestExecution): String = ""
/** @inheritDocs
*/
override def formatSuiteExecution(suiteExecution: SuiteExecution): String = ""

View file

@ -0,0 +1,118 @@
package gs.test.v0.reporting
import cats.effect.Async
import cats.effect.Concurrent
import cats.effect.Resource
import cats.effect.kernel.Fiber
import cats.effect.std.Queue
import cats.effect.syntax.all.*
import cats.syntax.all.*
import fs2.text
import gs.test.v0.api.GroupResult
import gs.test.v0.api.SuiteExecution
import gs.test.v0.api.TestExecution
import java.io.OutputStream
/** Implementation of [[Reporter]] that writes bytes to an `OutputStream`.
*
* @param formatter
* The [[ResultFormatter]] used to render test results.
* @param state
* The internal state of the reporter.
*/
final class OutputStreamReporter[F[_]: Async] private (
formatter: ResultFormatter,
state: OutputStreamReporter.State[F]
) extends Reporter[F]:
/** @inheritDocs
*/
override def startReport(): F[Unit] =
write(formatter.prefix)
/** @inheritDocs
*/
override def reportGroup(
groupResult: GroupResult,
testExecutions: List[TestExecution]
): F[Unit] =
write(formatter.formatGroupResult(groupResult)) *>
testExecutions
.map(formatter.formatTestExecution)
.map(write)
.sequence
.as(())
/** @inheritDocs
*/
override def reportSuite(suiteExecution: SuiteExecution): F[Unit] =
write(formatter.formatSuiteExecution(suiteExecution))
/** @inheritDocs
*/
override def endReport(): F[Unit] =
write(formatter.suffix)
private def write(output: String): F[Unit] =
state.queue.offer(Some(output))
/** Produce an effect that, when executed, will cause the underlying stream to
* terminate. After executing this effect, the `OutputStreamReporter` will no
* longer be capable of writing more output.
*
* @return
* The effect that describes the stop operation.
*/
def stop(): F[Unit] = state.queue.offer(None)
object OutputStreamReporter:
/** Provision a new [[OutputStreamReporter]].
*
* @param formatter
* The [[ResultFormatter]] this reporter should use to render test results.
* @param output
* Resource which manages the `OutputStream` where bytes will be written.
* @return
* Resource which manages the [[OutputStreamReporter]].
*/
def provision[F[_]: Concurrent: Async](
formatter: ResultFormatter,
output: Resource[F, OutputStream]
): Resource[F, OutputStreamReporter[F]] =
output.flatMap { os =>
Resource.make(acquireReporter(formatter, os))(_.stop())
}
private def acquireReporter[F[_]: Concurrent: Async](
formatter: ResultFormatter,
output: OutputStream
): F[OutputStreamReporter[F]] =
for
queue <- Queue.unbounded[F, Option[String]]
process <- startProcess[F](queue, output)
yield new OutputStreamReporter[F](
formatter = formatter,
state = new State[F](queue, process)
)
private def startProcess[F[_]: Concurrent: Async](
queue: Queue[F, Option[String]],
output: OutputStream
): F[Fiber[F, Throwable, Unit]] =
fs2.Stream
.fromQueueNoneTerminated(queue)
.through(text.utf8.encode)
.through(
fs2.io.writeOutputStream(Async[F].delay(output), closeAfterUse = false)
)
.compile
.drain
.start
private class State[F[_]](
val queue: Queue[F, Option[String]],
val process: Fiber[F, Throwable, Unit]
)
end OutputStreamReporter

View file

@ -0,0 +1,60 @@
package gs.test.v0.reporting
import cats.syntax.all.*
import gs.test.v0.api.GroupResult
import gs.test.v0.api.SuiteExecution
import gs.test.v0.api.TestExecution
import gs.test.v0.api.TestFailure
/** Implmentation of [[ResultFormatter]] that uses an unstructured text format.
*/
final class PlainResultFormatter extends ResultFormatter:
/** @inheritDocs
*/
override def prefix: String = ""
/** @inheritDocs
*/
override def suffix: String = ""
/** @inheritDocs
*/
override def formatGroupResult(groupResult: GroupResult): String =
def gr = groupResult
s"""
Group: '${gr.name.show}'
Stats: Seen=${gr.seen} Passed=${gr.passed} Failed=${gr.failed}
Duration: ${gr.millis}ms
Docs: ${gr.documentation.getOrElse("None")}
""".stripMargin
/** @inheritDocs
*/
override def formatTestExecution(testExecution: TestExecution): String =
def te = testExecution
s"""
Test: ${te.permanentId.show} (id=${te.id.show}) (span=${te.spanId})
Result: *${te.textResult}* in ${te.millis}ms
Tags: ${te.tags.mkString(", ")}
Docs: ${te.documentation.getOrElse("None")}${makeFailure(te.result)}
""".stripMargin
/** @inheritDocs
*/
override def formatSuiteExecution(suiteExecution: SuiteExecution): String =
def se = suiteExecution
s"""
Suite: '${se.testSuite.permanentId.show}' (id=${se.id.str}) (trace=${se.traceId})
Name: ${se.testSuite.name}
Stats: Seen=${se.seen} Passed=${se.passed} Failed=${se.failed}
Duration: ${se.millis}ms
""".stripMargin
private def makeFailure(result: Either[TestFailure, Any]): String =
result match
case Right(_) => ""
case Left(f) =>
s"""\n------
${f.message}
""".stripMargin

View file

@ -0,0 +1,63 @@
package gs.test.v0.reporting
import cats.Applicative
import gs.test.v0.api.GroupResult
import gs.test.v0.api.SuiteExecution
import gs.test.v0.api.TestExecution
/** Interface for reporters - implementations that report on test results.
*
* Example implementations include writing to standard output or writing to
* some JSON-formatted file.
*
* ## Order of Operations
*
* - `beginReporting()`
* - `reportGroup` for each group executed
* - `reportSuite`
* - `endReporting()`
*/
trait Reporter[F[_]]:
/** Hook for the beginning of the reporting lifecycle. This allows
* implementations to perform "setup" actions, such as opening a JSON object
* or writing a header.
*/
def startReport(): F[Unit]
/** Report the results of a single group.
*
* @param groupResult
* The [[GroupResult]] that describes the group level summary.
* @param testExecutions
* The list of [[TestExecution]] describing the result of each test.
* @return
* Side-effect that describes the reporting operation.
*/
def reportGroup(
groupResult: GroupResult,
testExecutions: List[TestExecution]
): F[Unit]
/** Report the results of an entire suite.
*
* @param suiteExecution
* The [[SuiteExecution]] that describes results.
* @return
* Side-effect that describes the reporting operation.
*/
def reportSuite(suiteExecution: SuiteExecution): F[Unit]
/** Hook for the end of the reporting lifecycle. This allows implementations
* to perform "finish" actions, such as closing a JSON object or writing a
* footer.
*/
def endReport(): F[Unit]
object Reporter:
/** @return
* New instance of the no-op Reporter implementation.
*/
def noop[F[_]: Applicative]: Reporter[F] = new NoopReporter[F]
end Reporter

View file

@ -0,0 +1,49 @@
package gs.test.v0.reporting
import gs.test.v0.api.GroupResult
import gs.test.v0.api.SuiteExecution
import gs.test.v0.api.TestExecution
/** Interface for formatters - implementations that transform test results into
* string representations.
*
* Example implementations include producing plain text or JSON
* representations.
*/
trait ResultFormatter:
/** @return
* The prefix for the format (if any).
*/
def prefix: String
/** @return
* The suffix for the format (if any).
*/
def suffix: String
/** Format a single [[GroupResult]] as a string.
*
* @param groupResult
* The result to format.
* @return
* The string rendition.
*/
def formatGroupResult(groupResult: GroupResult): String
/** Format a single [[TestExecution]] as a string.
*
* @param testExecution
* The result to format.
* @return
* The string rendition.
*/
def formatTestExecution(testExecution: TestExecution): String
/** Format a single [[SuiteExecution]] as a string.
*
* @param suiteExecution
* The result to format.
* @return
* The string rendition.
*/
def formatSuiteExecution(suiteExecution: SuiteExecution): String

View file

@ -1,17 +0,0 @@
package gs.test.v0.runtime
import gs.uuid.v0.UUID
import java.time.Instant
import scala.concurrent.duration.FiniteDuration
case class SuiteExecution(
id: UUID,
name: String,
documentation: Option[String],
duration: FiniteDuration,
countSeen: Long,
countSucceeded: Long,
countFailed: Long,
countIgnored: Long,
executedAt: Instant
)

View file

@ -1,5 +1,8 @@
package gs.test.v0.runtime.engine
import gs.uuid.v0.UUID
import java.time.Clock
/** Used to control the behavior of some [[TestEngine]]
*
* @param groupConcurrency
@ -8,8 +11,17 @@ package gs.test.v0.runtime.engine
* @param testConcurrency
* [[ConcurrencySetting]] for tests; the number of tests allowed to execute
* at the same time within some group.
* @param clock
* The `Clock` instance used to inform all date/time operations.
* @param suiteIdGenerator
* UUID provider that is used at the suite level.
* @param testIdGenerator
* UUID provider that is used at the test level.
*/
case class EngineConfiguration(
groupConcurrency: ConcurrencySetting,
testConcurrency: ConcurrencySetting
testConcurrency: ConcurrencySetting,
clock: Clock,
suiteIdGenerator: UUID.Generator,
testIdGenerator: UUID.Generator
)

View file

@ -0,0 +1,27 @@
package gs.test.v0.runtime.engine
object EngineConstants:
object Tracing:
val RootSpan: String = "suite"
val FullGroup: String = "full-group"
val BeforeGroup: String = "before-group"
val AfterGroup: String = "after-group"
val FullTest: String = "full-test"
val BeforeTest: String = "before-test"
val AfterTest: String = "after-test"
val InGroup: String = "in-group"
val TestSpan: String = "test"
end Tracing
object MetaData:
val TestGroupName: String = "test_group_name"
val TestExecutionId: String = "test_execution_id"
val TestName: String = "test_name"
end MetaData
end EngineConstants

View file

@ -1,7 +0,0 @@
package gs.test.v0.runtime.engine
import gs.test.v0.runtime.SuiteExecution
final class EngineResult(
val suiteExecution: SuiteExecution
)

View file

@ -3,32 +3,98 @@ package gs.test.v0.runtime.engine
import cats.effect.Async
import cats.effect.Ref
import cats.syntax.all.*
import gs.test.v0.api.TestExecution
import java.util.concurrent.TimeUnit
import scala.concurrent.duration.FiniteDuration
final class EngineStats[F[_]: Async](
val overallDuration: Ref[F, FiniteDuration],
val countSeen: Ref[F, Long],
val countSucceeded: Ref[F, Long],
val countFailed: Ref[F, Long],
val countIgnored: Ref[F, Long]
)
/** Statistics for executed tests. Used by the [[TestEngine]].
*
* @param overallDuration
* Duration of all recorded tests.
* @param countSeen
* Number of tests encountered.
* @param countPassed
* Number of tests that passed.
* @param countFailed
* Number of tests that failed.
*/
final class EngineStats[F[_]: Async] private (
overallDuration: Ref[F, FiniteDuration],
countSeen: Ref[F, Long],
countPassed: Ref[F, Long],
countFailed: Ref[F, Long]
):
/** @return
* The accumulated duration of test executions.
*/
def duration: F[FiniteDuration] = overallDuration.get
/** @return
* Number of tests encountered.
*/
def seen: F[Long] = countSeen.get
/** @return
* Number of tests that passed.
*/
def passed: F[Long] = countPassed.get
/** @return
* Number of tests that failed.
*/
def failed: F[Long] = countFailed.get
/** Update the stats based on the results of an entire group.
*
* @param duration
* The length of time it took to execute the group.
* @param testExecutions
* The list of all [[TestExecution]] produced by the group.
* @return
* Side-effect which updates statistic values.
*/
def updateForGroup(
duration: FiniteDuration,
testExecutions: List[TestExecution]
): F[Unit] =
for
_ <- overallDuration.update(base => base + duration)
_ <- testExecutions.map(updateForTest).sequence
yield ()
/** Update the stats based on the results of a single test.
*
* @param testExecution
* The [[TestExecution]] representing the test.
* @return
* Side-effect which updates statistic values.
*/
def updateForTest(testExecution: TestExecution): F[Unit] =
for
_ <- countSeen.update(_ + 1L)
_ <- testExecution.result match
case Left(_) => countFailed.update(_ + 1L)
case Right(_) => countPassed.update(_ + 1L)
yield ()
object EngineStats:
/** Initialize a new [[EngineStats]] instance with all values set to 0.
*
* @return
* The new [[EngineStats]] instance.
*/
def initialize[F[_]: Async]: F[EngineStats[F]] =
for
duration <- Ref.of(FiniteDuration(0L, TimeUnit.NANOSECONDS))
seen <- Ref.of(0L)
succeeded <- Ref.of(0L)
failed <- Ref.of(0L)
ignored <- Ref.of(0L)
duration <- Ref.of(FiniteDuration(0L, TimeUnit.NANOSECONDS))
seen <- Ref.of(0L)
passed <- Ref.of(0L)
failed <- Ref.of(0L)
yield new EngineStats[F](
overallDuration = duration,
countSeen = seen,
countSucceeded = succeeded,
countFailed = failed,
countIgnored = ignored
countPassed = passed,
countFailed = failed
)
end EngineStats

View file

@ -1,12 +0,0 @@
package gs.test.v0.runtime.engine
import gs.test.v0.api.TestGroupDefinition
import gs.test.v0.runtime.TestExecution
import scala.concurrent.duration.FiniteDuration
final class GroupResult(
val name: TestGroupDefinition.Name,
val documentation: Option[String],
val duration: FiniteDuration,
val testExecutions: List[TestExecution]
)

View file

@ -2,112 +2,189 @@ package gs.test.v0.runtime.engine
import cats.effect.Async
import cats.syntax.all.*
import gs.test.v0.api.GroupResult
import gs.test.v0.api.SuiteExecution
import gs.test.v0.api.TestDefinition
import gs.test.v0.api.TestExecution
import gs.test.v0.api.TestFailure
import gs.test.v0.api.TestGroupDefinition
import gs.test.v0.api.TestSuite
import gs.test.v0.runtime.SuiteExecution
import gs.test.v0.runtime.TestExecution
import gs.test.v0.reporting.Reporter
import gs.timing.v0.Timing
import gs.uuid.v0.UUID
import java.time.Clock
import java.time.Instant
import natchez.EntryPoint
import natchez.Span
/** This class is responsible for executing suites of tests.
*
* ## How Execution Works
*
* Test execution starts at the group level, via a stream of
* [[TestGroupDefinition]]. Each group of tests is executed concurrently based
* on the [[EngineConfiguration]].
*
* ### Executing a Single Group
*
* Each [[TestGroupDefinition]] is executed by executing, in order:
*
* - The `beforeGroup` effect.
* - Each test (configurable concurrency).
* - The `afterGroup` effect.
*
* The before/after effects are described at the group level.
*
* ### Executing a Single Test
*
* Each [[TestDefinition]] is executed by executing, in order:
*
* - The `beforeEachTest` effect.
* - The test code.
* - The `afterEachTest` effect.
*
* The before/after effects are described at the group level.
*
* ## OpenTelemetry Support
*
* Each [[SuiteExecution]] produces a single trace that encompasses all tests.
* Spans are used to designate different related portions of work.
*/
final class TestEngine[F[_]: Async](
val configuration: EngineConfiguration,
timing: Timing[F],
suiteExecutionIdGenerator: UUID.Generator,
testExecutionIdGenerator: UUID.Generator,
clock: Clock,
val entryPoint: EntryPoint[F]
val reporter: Reporter[F],
val entryPoint: EntryPoint[F],
timing: Timing[F]
):
private def clock = configuration.clock
private def testIdGen = configuration.testIdGenerator
private def suiteIdGen = configuration.suiteIdGenerator
/** Execute a suite of tests.
*
* This function only provides a summary output. Results are streamed using a
* [[Reporter]] instance.
*
* @param suite
* The metadata that describes the suite.
* @param tests
* The stream of groups that define the tests.
* @return
* Summary of the execution.
*/
def runSuite(
suite: TestSuite,
tests: fs2.Stream[F, TestGroupDefinition[F]]
): F[SuiteExecution] =
for
executedAt <- Async[F].delay(Instant.now(clock))
stats <- EngineStats.initialize[F]
_ <- tests
.mapAsync(configuration.groupConcurrency.toInt())(runGroup)
.evalTap(updateGroupStats)
.evalTap(reportGroup)
.flatMap(groupResult => fs2.Stream.emits(groupResult.testExecutions))
.evalTap(updateTestStats)
.evalMap(reportTestExecution)
.compile
.drain
overallDuration <- stats.overallDuration.get
countSeen <- stats.countSeen.get
countSucceeded <- stats.countSucceeded.get
countFailed <- stats.countFailed.get
countIgnored <- stats.countIgnored.get
yield SuiteExecution(
id = suiteExecutionIdGenerator.next(),
name = suite.name,
documentation = suite.documentation,
duration = overallDuration,
countSeen = countSeen,
countSucceeded = countSucceeded,
countFailed = countFailed,
countIgnored = countIgnored,
executedAt = executedAt
)
private def updateGroupStats(groupResult: GroupResult): F[Unit] = ???
private def updateTestStats(testExecution: TestExecution): F[Unit] = ???
private def reportGroup(groupResult: GroupResult): F[Unit] = ???
private def reportTestExecution(testExecution: TestExecution): F[Unit] = ???
private def runSpan[A](
name: String,
root: Span[F],
f: F[A]
): F[A] =
root.span(name).use(_ => f)
def runGroup(
group: TestGroupDefinition[F]
): F[GroupResult] =
entryPoint.root("test-group").use { rootSpan =>
entryPoint.root(EngineConstants.Tracing.RootSpan).use { rootSpan =>
for
_ <- rootSpan.put("test_group_name" -> group.name.show)
_ <- runSpan(
"before-group",
rootSpan,
group.beforeGroup.getOrElse(Async[F].unit)
)
stream <- executeGroupTests(group, rootSpan)
_ <- runSpan(
"after-group",
rootSpan,
group.afterGroup.getOrElse(Async[F].unit)
)
yield stream
executedAt <- Async[F].delay(Instant.now(clock))
stats <- EngineStats.initialize[F]
// Start reporting
_ <- reporter.startReport()
// Run all tests, group by group.
_ <- tests
.mapAsync(configuration.groupConcurrency.toInt())(
runGroup(rootSpan, _)
)
.evalTap(
(
groupResult,
testExecutions
) =>
for
// Update the overall statistics based on this group.
_ <- stats.updateForGroup(
duration = groupResult.duration,
testExecutions = testExecutions
)
// Report group level results for this group.
_ <- reporter.reportGroup(
groupResult = groupResult,
testExecutions = testExecutions
)
yield ()
)
.compile
.drain
// Calculate the final summary of execution at the suite level.
suiteExecution <- makeSuiteExecution(rootSpan, suite, stats, executedAt)
// Report suite level results.
_ <- reporter.reportSuite(suiteExecution)
// Finish reporting.
_ <- reporter.endReport()
yield suiteExecution
}
private def executeGroupTests(
group: TestGroupDefinition[F],
rootSpan: Span[F]
): F[GroupResult] =
rootSpan.span("group").use { groupSpan =>
def runGroup(
suiteSpan: Span[F],
group: TestGroupDefinition[F]
): F[(GroupResult, List[TestExecution])] =
suiteSpan.span(EngineConstants.Tracing.FullGroup).use { fullGroupSpan =>
for
traceId <- rootSpan.traceId.map(parseTraceId)
timer <- timing.start()
executions <- streamGroupTests(group, groupSpan).compile.toList
elapsed <- timer.checkpoint()
groupStats <- EngineStats.initialize[F]
// Augment the span with all group-level metadata.
_ <- fullGroupSpan
.put(EngineConstants.MetaData.TestGroupName -> group.name.show)
// Start the timer for the entire group.
timer <- timing.start()
// Run the before-group logic (in its own span).
_ <- runSpan(
EngineConstants.Tracing.BeforeGroup,
fullGroupSpan,
group.beforeGroup.getOrElse(Async[F].unit)
)
// Execute all tests within this group.
testExecutions <- executeGroupTests(group, fullGroupSpan)
// Run the after-group logic (in its own span).
_ <- runSpan(
EngineConstants.Tracing.AfterGroup,
fullGroupSpan,
group.afterGroup.getOrElse(Async[F].unit)
)
// Calculate the overall elapsed time for this group.
elapsed <- timer.checkpoint()
// Calculate group-level statistics.
_ <- groupStats.updateForGroup(
duration = elapsed.duration,
testExecutions = testExecutions
)
// Extract the group statistic values for inclusion in the result..
seen <- groupStats.seen
passed <- groupStats.passed
failed <- groupStats.failed
yield new GroupResult(
name = group.name,
documentation = group.documentation,
duration = elapsed.duration,
testExecutions = executions
)
seen = seen,
passed = passed,
failed = failed
) -> testExecutions
}
private def executeGroupTests(
group: TestGroupDefinition[F],
fullGroupSpan: Span[F]
): F[List[TestExecution]] =
fullGroupSpan.span(EngineConstants.Tracing.InGroup).use { groupSpan =>
for
// If, for some reason, the generated span has no Trace ID, this will
// throw an exception.
executions <- streamGroupTests(group, groupSpan).compile.toList
yield executions
}
private def streamGroupTests(
@ -117,26 +194,50 @@ final class TestEngine[F[_]: Async](
fs2.Stream
.emits(group.tests)
.mapAsync(configuration.testConcurrency.toInt()) { test =>
for
testExecutionId <- Async[F].delay(
TestExecution.Id(testExecutionIdGenerator.next())
groupSpan.span(EngineConstants.Tracing.FullTest).use { fullSpan =>
for
// Generate a unique TestExecutionId for this execution.
testExecutionId <- Async[F].delay(
TestExecution.Id(testIdGen.next())
)
testSpanId <- fullSpan.spanId.map(parseSpanId)
// Start the timer for the test, including the before/after
// components.
timer <- timing.start()
// Run the before-test logic (in its own span).
_ <- runSpan(
EngineConstants.Tracing.BeforeTest,
groupSpan,
group.beforeEachTest.getOrElse(Async[F].unit)
)
// Run the test (in its own span).
result <- runSingleTest(testExecutionId, test, groupSpan)
// Run the after-test logic (in its own span).
_ <- runSpan(
EngineConstants.Tracing.AfterTest,
groupSpan,
group.afterEachTest.getOrElse(Async[F].unit)
)
// Calculate the overall elapsed time for this single test.
elapsed <- timer.checkpoint()
yield TestExecution(
id = testExecutionId,
permanentId = test.permanentId,
documentation = test.documentation,
tags = test.tags,
markers = test.markers,
result = result,
spanId = testSpanId,
sourcePosition = test.sourcePosition,
duration = elapsed.duration
)
timer <- timing.start()
_ <- group.beforeEachTest.getOrElse(Async[F].unit)
result <- runSingleTest(testExecutionId, test, groupSpan)
_ <- group.afterEachTest.getOrElse(Async[F].unit)
elapsed <- timer.checkpoint()
yield TestExecution(
id = testExecutionId,
permanentId = test.permanentId,
documentation = test.documentation,
tags = test.tags,
markers = test.markers,
result = result,
traceId = ???,
sourcePosition = test.sourcePosition,
duration = elapsed.duration
)
}
}
private def runSingleTest(
@ -144,13 +245,94 @@ final class TestEngine[F[_]: Async](
test: TestDefinition[F],
groupSpan: Span[F]
): F[Either[TestFailure, Any]] =
groupSpan.span("test").use { span =>
groupSpan.span(EngineConstants.Tracing.TestSpan).use { span =>
for
// TODO: Constants
_ <- span.put("test_execution_id" -> testExecutionId.show)
_ <- span.put("test_name" -> test.name.show)
_ <- span
.put(EngineConstants.MetaData.TestExecutionId -> testExecutionId.show)
_ <- span.put(EngineConstants.MetaData.TestName -> test.name.show)
result <- test.unitOfWork.doWork(span)
yield result
}
private def parseTraceId(candidate: Option[String]): UUID = ???
private def parseTraceId(candidate: Option[String]): String =
candidate match
case Some(traceId) => traceId
case None =>
throw new IllegalArgumentException("Created a span without a Trace ID!")
private def parseSpanId(candidate: Option[String]): String =
candidate match
case Some(spanId) => spanId
case None =>
throw new IllegalArgumentException("Created a span without a Span ID!")
private def makeSuiteExecution(
rootSpan: Span[F],
suite: TestSuite,
stats: EngineStats[F],
executedAt: Instant
): F[SuiteExecution] =
for
traceId <- rootSpan.traceId.map(parseTraceId)
overallDuration <- stats.duration
seen <- stats.seen
passed <- stats.passed
failed <- stats.failed
yield SuiteExecution(
id = suiteIdGen.next(),
testSuite = suite,
traceId = traceId,
duration = overallDuration,
seen = seen,
passed = passed,
failed = failed,
executedAt = executedAt
)
/** Run some effect as a child span for some root span.
*
* @param name
* The name of the span.
* @param root
* The root span.
* @param f
* The effect to execute in a child span.
* @return
* The contextualized effect.
*/
private def runSpan[A](
name: String,
root: Span[F],
f: F[A]
): F[A] =
root.span(name).use(_ => f)
object TestEngine:
/** Initialize a new [[TestEngine]].
*
* @param configuration
* The [[EngineConfiguration]] used for this instance.
* @param reporter
* Reports test results.
* @param entryPoint
* Entry point for OpenTelemetry support.
* @param timing
* Timing controller.
* @return
* Resource which manages the [[TestEngine]].
*/
def initialize[F[_]: Async](
configuration: EngineConfiguration,
reporter: Reporter[F],
entryPoint: EntryPoint[F],
timing: Timing[F]
): TestEngine[F] =
new TestEngine(
configuration = configuration,
reporter = reporter,
entryPoint = entryPoint,
timing = timing
)
end TestEngine

View file

@ -0,0 +1,77 @@
package gs.test.v0.runtime.engine
import cats.effect.IO
import gs.datagen.v0.Gen
import gs.datagen.v0.generators.Size
import java.util.concurrent.TimeUnit
import munit.*
import scala.concurrent.duration.FiniteDuration
import support.*
class EngineStatsTests extends IOSuite:
iotest("should initialize empty stats") {
for
stats <- EngineStats.initialize[IO]
duration <- stats.duration
seen <- stats.seen
passed <- stats.passed
failed <- stats.failed
yield
assertEquals(duration, Durations.Zero)
assertEquals(seen, 0L)
assertEquals(passed, 0L)
assertEquals(failed, 0L)
}
iotest("should update based on a group with no test executions") {
val expected = FiniteDuration(2L, TimeUnit.MILLISECONDS)
for
stats <- EngineStats.initialize[IO]
_ <- stats.updateForGroup(Durations.OneMilli, Nil)
_ <- stats.updateForGroup(Durations.OneMilli, Nil)
duration <- stats.duration
seen <- stats.seen
passed <- stats.passed
failed <- stats.failed
yield
assertEquals(duration, expected)
assertEquals(seen, 0L)
assertEquals(passed, 0L)
assertEquals(failed, 0L)
}
iotest("should update based on a single test execution") {
for
stats <- EngineStats.initialize[IO]
_ <- stats.updateForTest(Generators.testExecutionPassed())
_ <- stats.updateForTest(Generators.testExecutionFailed())
duration <- stats.duration
seen <- stats.seen
passed <- stats.passed
failed <- stats.failed
yield
assertEquals(duration, Durations.Zero)
assertEquals(seen, 2L)
assertEquals(passed, 1L)
assertEquals(failed, 1L)
}
iotest("should update based on a test group") {
val duration = Generators.testDuration()
val size = 4
val executions =
Gen.list(Size.fixed(size), Generators.GenTestExecutionPassed).gen()
for
stats <- EngineStats.initialize[IO]
_ <- stats.updateForGroup(duration, executions)
duration <- stats.duration
seen <- stats.seen
passed <- stats.passed
failed <- stats.failed
yield
assertEquals(duration, duration)
assertEquals(seen, size.toLong)
assertEquals(passed, size.toLong)
assertEquals(failed, 0L)
}

View file

@ -0,0 +1,192 @@
package gs.test.v0.runtime.engine
import cats.effect.IO
import cats.effect.Resource
import gs.test.v0.api.*
import gs.test.v0.reporting.InMemoryReporter
import gs.test.v0.runtime.engine.TestEngineTests.*
import gs.timing.v0.MonotonicProvider.ManualTickProvider
import gs.timing.v0.Timing
import gs.uuid.v0.UUID
import java.time.Clock
import munit.*
import support.*
class TestEngineTests extends IOSuite:
import TestEngineTests.TestData
iotest("should run an engine with no tests") {
newEngine().use { obs =>
val spanDb = obs.entryPoint.spanDb
for
suiteExecution <- obs.engine.runSuite(
suite = Generators.testSuite(),
tests = emptyStream[TestGroupDefinition[IO]]
)
rootSpan <- spanDb.get(EngineConstants.Tracing.RootSpan)
results <- obs.reporter.terminateAndGetResults()
yield
assertEquals(rootSpan.size, 1)
assertEquals(results.isEmpty, true)
assertEquals(suiteExecution.seen, 0L)
assertEquals(suiteExecution.passed, 0L)
assertEquals(suiteExecution.failed, 0L)
}
}
iotest("should run an engine with a single passing test") {
newEngine().use { obs =>
val spanDb = obs.entryPoint.spanDb
val g1 = new G1
val group = g1.compile()
for
suiteExecution <- obs.engine.runSuite(
suite = Generators.testSuite(),
tests = fs2.Stream.apply(group)
)
rootSpan <- spanDb.get(EngineConstants.Tracing.RootSpan)
groupSpan <- spanDb.get(EngineConstants.Tracing.FullGroup)
beforeGroupSpan <- spanDb.get(
EngineConstants.Tracing.BeforeGroup
)
afterGroupSpan <- spanDb.get(
EngineConstants.Tracing.AfterGroup
)
inGroupSpan <- spanDb.get(EngineConstants.Tracing.InGroup)
fullTestSpan <- spanDb.get(EngineConstants.Tracing.FullTest)
beforeTestSpan <- spanDb.get(
EngineConstants.Tracing.BeforeTest
)
afterTestSpan <- spanDb.get(
EngineConstants.Tracing.AfterTest
)
testSpan <- spanDb.get(EngineConstants.Tracing.TestSpan)
results <- obs.reporter.terminateAndGetResults()
yield
assertEquals(rootSpan.size, 1)
assertEquals(groupSpan.size, 1)
assertEquals(beforeGroupSpan.size, 1)
assertEquals(afterGroupSpan.size, 1)
assertEquals(inGroupSpan.size, 1)
assertEquals(fullTestSpan.size, 1)
assertEquals(beforeTestSpan.size, 1)
assertEquals(afterTestSpan.size, 1)
assertEquals(testSpan.size, 1)
assertEquals(results.size, 1)
assertEquals(suiteExecution.seen, 1L)
assertEquals(suiteExecution.passed, 1L)
assertEquals(suiteExecution.failed, 0L)
}
}
iotest("should run an engine with a single failing test") {
newEngine().use { obs =>
val spanDb = obs.entryPoint.spanDb
val g2 = new G2
val group = g2.compile()
for
suiteExecution <- obs.engine.runSuite(
suite = Generators.testSuite(),
tests = fs2.Stream.apply(group)
)
rootSpan <- spanDb.get(EngineConstants.Tracing.RootSpan)
groupSpan <- spanDb.get(EngineConstants.Tracing.FullGroup)
beforeGroupSpan <- spanDb.get(
EngineConstants.Tracing.BeforeGroup
)
afterGroupSpan <- spanDb.get(
EngineConstants.Tracing.AfterGroup
)
inGroupSpan <- spanDb.get(EngineConstants.Tracing.InGroup)
fullTestSpan <- spanDb.get(EngineConstants.Tracing.FullTest)
beforeTestSpan <- spanDb.get(
EngineConstants.Tracing.BeforeTest
)
afterTestSpan <- spanDb.get(
EngineConstants.Tracing.AfterTest
)
testSpan <- spanDb.get(EngineConstants.Tracing.TestSpan)
results <- obs.reporter.terminateAndGetResults()
yield
// TODO rip out a validation function for a full set of stuff.
assertEquals(rootSpan.size, 1)
assertEquals(groupSpan.size, 1)
assertEquals(beforeGroupSpan.size, 1)
assertEquals(afterGroupSpan.size, 1)
assertEquals(inGroupSpan.size, 1)
assertEquals(fullTestSpan.size, 1)
assertEquals(beforeTestSpan.size, 1)
assertEquals(afterTestSpan.size, 1)
assertEquals(testSpan.size, 1)
assertEquals(results.size, 1)
assertEquals(suiteExecution.seen, 1L)
assertEquals(suiteExecution.passed, 0L)
assertEquals(suiteExecution.failed, 1L)
}
}
private def emptyStream[A]: fs2.Stream[IO, A] =
fs2.Stream.empty
private def liftToResource[A](io: IO[A]): Resource[IO, A] =
Resource.make(io)(_ => IO.unit)
def newEngine(): Resource[IO, EngineObservation] =
for
(tickProvider, timing) <- liftToResource(Timing.manual[IO])
reporter <- InMemoryReporter.provision[IO]
entryPoint <- TestEntryPoint.provision()
yield EngineObservation(
tickProvider = tickProvider,
reporter = reporter,
entryPoint = entryPoint,
engine = TestEngine.initialize[IO](
configuration = TestData.Config,
reporter = reporter,
entryPoint = entryPoint,
timing = timing
)
)
object TestEngineTests:
private object TestData:
val Config: EngineConfiguration = EngineConfiguration(
groupConcurrency = ConcurrencySetting.Serial,
testConcurrency = ConcurrencySetting.Serial,
clock = Clock.systemUTC(),
suiteIdGenerator = UUID.Generator.version7,
testIdGenerator = UUID.Generator.version7
)
end TestData
case class EngineObservation(
tickProvider: ManualTickProvider[IO],
reporter: InMemoryReporter[IO],
entryPoint: TestEntryPoint,
engine: TestEngine[IO]
)
class G1 extends TestGroup[IO]:
override def name: String = "single-passing-test"
test(pid"engine:g1", "show that true is true") {
check(true).isTrueT()
}
end G1
class G2 extends TestGroup[IO]:
override def name: String = "single-failing-test"
test(pid"engine:g2", "this will fail") {
check(1).isEqualToT(2)
}
end G2
end TestEngineTests

View file

@ -0,0 +1,109 @@
package support
import gs.datagen.v0.*
import gs.datagen.v0.generators.Size
import gs.test.v0.api.*
import scala.concurrent.duration.FiniteDuration
object Generators:
val NoSourcePosition: SourcePosition = SourcePosition("TEST", 0)
val GenTestExecutionId: Gen[TestExecution.Id] =
Gen.uuid.random().map(TestExecution.Id(_))
given Generated[TestExecution.Id] = Generated.of(GenTestExecutionId)
val GenPermanentId: Gen[PermanentId] =
Gen.string.alphaNumeric(Size.Fixed(12)).map(x => PermanentId(s"pid-$x"))
given Generated[PermanentId] = Generated.of(GenPermanentId)
val GenTag: Gen[Tag] =
Gen.string.alphaNumeric(Size.Fixed(6)).map(x => Tag(s"tag-$x"))
val GenTagList: Gen[List[Tag]] = Gen.list(Size.between(0, 8), GenTag)
given Generated[Tag] = Generated.of(GenTag)
val GenTraceId: Gen[String] = Gen.uuid.string().map(_.filterNot(_ == '-'))
val GenSpanId: Gen[String] = GenTraceId.map(_.take(16))
val GenTestDuration: Gen[FiniteDuration] =
Gen.duration.finiteMilliseconds(1L, 100L)
val GenTestResult: Gen[Either[TestFailure, Any]] =
Gen.boolean().map(makeResult)
private def makeResult(passed: Boolean): TestResult =
passed match {
case true => Right(())
case false =>
Left(TestFailure.TestRequestedFailure("Failed", NoSourcePosition))
}
val InputGenTestExecution: Datagen[TestExecution, Boolean] =
for
id <- GenTestExecutionId
permanentId <- GenPermanentId
tags <- GenTagList
spanId <- GenSpanId
duration <- GenTestDuration
yield (passed: Boolean) =>
TestExecution(
id = id,
permanentId = permanentId,
documentation = None,
tags = tags,
markers = Nil,
result = makeResult(passed),
spanId = spanId,
sourcePosition = NoSourcePosition,
duration = duration
)
val GenTestExecution: Gen[TestExecution] =
Gen.boolean().satisfy(InputGenTestExecution)
val GenTestExecutionPassed: Gen[TestExecution] =
InputGenTestExecution.toGen(true)
val GenTestExecutionFailed: Gen[TestExecution] =
InputGenTestExecution.toGen(false)
given Generated[TestExecution] = Generated.of(GenTestExecution)
val GenTestSuite: Gen[TestSuite] =
for
pid <- GenPermanentId
name <- Gen.string.alphaNumeric(Size.fixed(8))
yield TestSuite(pid, name, None)
given Generated[TestSuite] = Generated.of(GenTestSuite)
def testExecutionId(): TestExecution.Id = GenTestExecutionId.gen()
def permanentId(): PermanentId = GenPermanentId.gen()
def tag(): Tag = GenTag.gen()
def tags(): List[Tag] = GenTagList.gen()
def traceId(): String = GenTraceId.gen()
def spanId(): String = GenSpanId.gen()
def testDuration(): FiniteDuration = GenTestDuration.gen()
def testExecution(): TestExecution = GenTestExecution.gen()
def testExecutionPassed(): TestExecution =
GenTestExecutionPassed.gen()
def testExecutionFailed(): TestExecution =
GenTestExecutionFailed.gen()
def testSuite(): TestSuite = GenTestSuite.gen()
end Generators

View file

@ -0,0 +1,11 @@
package support
import java.util.concurrent.TimeUnit
import scala.concurrent.duration.FiniteDuration
object Durations:
val Zero: FiniteDuration = FiniteDuration(0L, TimeUnit.NANOSECONDS)
val OneMilli: FiniteDuration = FiniteDuration(1L, TimeUnit.MILLISECONDS)
end Durations

View file

@ -0,0 +1,27 @@
package support
import cats.effect.IO
import cats.effect.std.MapRef
final class SpanDb(
db: MapRef[IO, String, Option[List[TestSpan]]]
):
def get(spanName: String): IO[List[TestSpan]] =
db(spanName).get.map(_.getOrElse(Nil))
def putSpan(
spanName: String,
span: TestSpan
): IO[Unit] =
db(spanName).update {
case None => Some(List(span))
case Some(spans) => Some(span :: spans)
}
object SpanDb:
def initialize(): IO[SpanDb] =
MapRef[IO, String, List[TestSpan]].map(db => new SpanDb(db))
end SpanDb

View file

@ -0,0 +1,44 @@
package support
import cats.effect.IO
import cats.effect.kernel.Resource
import natchez.EntryPoint
import natchez.Kernel
import natchez.Span
import natchez.Span.Options
final class TestEntryPoint private (
val spanDb: SpanDb
) extends EntryPoint[IO]:
override def root(
name: String,
options: Options
): Resource[IO, Span[IO]] =
TestSpan
.provisionRoot(name, spanDb)
.evalTap(span => spanDb.putSpan(name, span))
override def continue(
name: String,
kernel: Kernel,
options: Options
): Resource[IO, Span[IO]] =
throw new IllegalStateException("Not allowed for testing.")
override def continueOrElseRoot(
name: String,
kernel: Kernel,
options: Options
): Resource[IO, Span[IO]] =
throw new IllegalStateException("Not allowed for testing.")
object TestEntryPoint:
def initialize(): IO[TestEntryPoint] =
SpanDb.initialize().map(db => new TestEntryPoint(db))
def provision(): Resource[IO, TestEntryPoint] =
Resource.make(initialize())(_ => IO.unit)
end TestEntryPoint

View file

@ -0,0 +1,93 @@
package support
import cats.effect.IO
import cats.effect.kernel.Resource
import cats.effect.std.MapRef
import java.net.URI
import java.util.UUID
import natchez.Kernel
import natchez.Span
import natchez.Span.Options
import natchez.TraceValue
final class TestSpan private (
val name: String,
val rawTraceId: String,
val rawSpanId: String,
baggage: MapRef[IO, String, Option[TraceValue]],
spanDb: SpanDb
) extends Span[IO]:
override def put(fields: (String, TraceValue)*): IO[Unit] =
fields.map { case (k, v) => baggage.setKeyValue(k, v) }.sequence.as(())
override def log(fields: (String, TraceValue)*): IO[Unit] = IO.unit
override def log(event: String): IO[Unit] = IO.unit
override def attachError(
err: Throwable,
fields: (String, TraceValue)*
): IO[Unit] = IO.unit
override def kernel: IO[Kernel] = IO(Kernel(Map.empty))
override def span(
name: String,
options: Options
): Resource[IO, Span[IO]] =
TestSpan
.provision(name, rawTraceId, TestSpan.makeSpanId(), spanDb)
.evalTap(span => spanDb.putSpan(name, span))
override def traceId: IO[Option[String]] = IO(Some(rawTraceId))
override def spanId: IO[Option[String]] = IO(Some(rawSpanId))
override def traceUri: IO[Option[URI]] = IO(None)
object TestSpan:
def initializeRoot(
name: String,
spanDb: SpanDb
): IO[TestSpan] =
initialize(name, makeTraceId(), makeSpanId(), spanDb)
def initialize(
name: String,
traceId: String,
spanId: String,
spanDb: SpanDb
): IO[TestSpan] =
MapRef.apply[IO, String, TraceValue].map { baggage =>
new TestSpan(
name = name,
rawTraceId = traceId,
rawSpanId = spanId,
baggage = baggage,
spanDb = spanDb
)
}
def provisionRoot(
name: String,
spanDb: SpanDb
): Resource[IO, TestSpan] =
provision(name, makeTraceId(), makeSpanId(), spanDb)
def provision(
name: String,
traceId: String,
spanId: String,
spanDb: SpanDb
): Resource[IO, TestSpan] =
Resource.make(initialize(name, traceId, spanId, spanDb))(_ => IO.unit)
private def makeTraceId(): String =
UUID.randomUUID().toString().filterNot(_ == '-')
private def makeSpanId(): String =
makeTraceId().take(16)
end TestSpan

View file

@ -1 +1 @@
sbt.version=1.10.2
sbt.version=1.11.6

View file

@ -28,6 +28,6 @@ externalResolvers := Seq(
"Garrity Software Releases" at "https://maven.garrity.co/gs"
)
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.1.0")
addSbtPlugin("gs" % "sbt-garrity-software" % "0.4.0")
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.3.1")
addSbtPlugin("gs" % "sbt-garrity-software" % "0.6.0")
addSbtPlugin("gs" % "sbt-gs-semver" % "0.3.0")