More work on the runtime.
This commit is contained in:
parent
c2a155ceab
commit
fb831ea7d3
13 changed files with 408 additions and 127 deletions
|
@ -59,11 +59,14 @@ lazy val `gs-test` = project
|
||||||
.aggregate(
|
.aggregate(
|
||||||
`test-support`,
|
`test-support`,
|
||||||
api,
|
api,
|
||||||
|
reporting,
|
||||||
runtime
|
runtime
|
||||||
)
|
)
|
||||||
.settings(noPublishSettings)
|
.settings(noPublishSettings)
|
||||||
.settings(name := s"${gsProjectName.value}-v${semVerMajor.value}")
|
.settings(name := s"${gsProjectName.value}-v${semVerMajor.value}")
|
||||||
|
|
||||||
|
/** Internal project used for unit tests.
|
||||||
|
*/
|
||||||
lazy val `test-support` = project
|
lazy val `test-support` = project
|
||||||
.in(file("modules/test-support"))
|
.in(file("modules/test-support"))
|
||||||
.settings(sharedSettings)
|
.settings(sharedSettings)
|
||||||
|
@ -79,6 +82,8 @@ lazy val `test-support` = project
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
/** Core API - the only dependency needed to write tests.
|
||||||
|
*/
|
||||||
lazy val api = project
|
lazy val api = project
|
||||||
.in(file("modules/api"))
|
.in(file("modules/api"))
|
||||||
.dependsOn(`test-support` % "test->test")
|
.dependsOn(`test-support` % "test->test")
|
||||||
|
@ -96,6 +101,8 @@ lazy val api = project
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
/** Reporting API and implementations.
|
||||||
|
*/
|
||||||
lazy val reporting = project
|
lazy val reporting = project
|
||||||
.in(file("modules/reporting"))
|
.in(file("modules/reporting"))
|
||||||
.dependsOn(`test-support` % "test->test")
|
.dependsOn(`test-support` % "test->test")
|
||||||
|
@ -111,6 +118,8 @@ lazy val reporting = project
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
/** Runtime - the dependency needed to _run_ tests.
|
||||||
|
*/
|
||||||
lazy val runtime = project
|
lazy val runtime = project
|
||||||
.in(file("modules/runtime"))
|
.in(file("modules/runtime"))
|
||||||
.dependsOn(`test-support` % "test->test")
|
.dependsOn(`test-support` % "test->test")
|
||||||
|
|
|
@ -24,4 +24,5 @@ final class GroupResult(
|
||||||
val seen: Long,
|
val seen: Long,
|
||||||
val passed: Long,
|
val passed: Long,
|
||||||
val failed: Long
|
val failed: Long
|
||||||
)
|
):
|
||||||
|
def millis: Long = duration.toMillis
|
||||||
|
|
|
@ -8,15 +8,17 @@ import scala.concurrent.duration.FiniteDuration
|
||||||
*
|
*
|
||||||
* @param id
|
* @param id
|
||||||
* Unique identifier for this execution.
|
* Unique identifier for this execution.
|
||||||
* @param suite
|
* @param testSuite
|
||||||
* Suite-level identifiers and metadata.
|
* Suite-level identifiers and metadata.
|
||||||
|
* @param traceId
|
||||||
|
* The 128-bit trace identifier used for this suite.
|
||||||
* @param duration
|
* @param duration
|
||||||
* Overall amount of time it took to execute the suite.
|
* Overall amount of time it took to execute the suite.
|
||||||
* @param countSeen
|
* @param seen
|
||||||
* Overall number of tests seen.
|
* Overall number of tests seen.
|
||||||
* @param countPassed
|
* @param passed
|
||||||
* Overall number of passed tests.
|
* Overall number of passed tests.
|
||||||
* @param countFailed
|
* @param failed
|
||||||
* Overall number of failed tests.
|
* Overall number of failed tests.
|
||||||
* @param executedAt
|
* @param executedAt
|
||||||
* Timestamp at which this suite was executed.
|
* Timestamp at which this suite was executed.
|
||||||
|
@ -24,9 +26,11 @@ import scala.concurrent.duration.FiniteDuration
|
||||||
case class SuiteExecution(
|
case class SuiteExecution(
|
||||||
id: UUID,
|
id: UUID,
|
||||||
testSuite: TestSuite,
|
testSuite: TestSuite,
|
||||||
|
traceId: String,
|
||||||
duration: FiniteDuration,
|
duration: FiniteDuration,
|
||||||
countSeen: Long,
|
seen: Long,
|
||||||
countPassed: Long,
|
passed: Long,
|
||||||
countFailed: Long,
|
failed: Long,
|
||||||
executedAt: Instant
|
executedAt: Instant
|
||||||
)
|
):
|
||||||
|
def millis: Long = duration.toMillis
|
||||||
|
|
|
@ -21,8 +21,8 @@ import scala.concurrent.duration.FiniteDuration
|
||||||
* Markers for the test that was executed.
|
* Markers for the test that was executed.
|
||||||
* @param result
|
* @param result
|
||||||
* The result of the test.
|
* The result of the test.
|
||||||
* @param traceId
|
* @param spanId
|
||||||
* The 128-bit trace identifier used for this test.
|
* The 64-bit span identifier used for this test (including before/after).
|
||||||
* @param sourcePosition
|
* @param sourcePosition
|
||||||
* The position, in source code, of the test that was executed.
|
* The position, in source code, of the test that was executed.
|
||||||
* @param duration
|
* @param duration
|
||||||
|
@ -36,10 +36,23 @@ case class TestExecution(
|
||||||
tags: List[Tag],
|
tags: List[Tag],
|
||||||
markers: List[Marker],
|
markers: List[Marker],
|
||||||
result: Either[TestFailure, Any],
|
result: Either[TestFailure, Any],
|
||||||
traceId: UUID,
|
spanId: String,
|
||||||
sourcePosition: SourcePosition,
|
sourcePosition: SourcePosition,
|
||||||
duration: FiniteDuration
|
duration: FiniteDuration
|
||||||
)
|
):
|
||||||
|
|
||||||
|
/** @return
|
||||||
|
* The string "passed" if the test passed, and "failed" otherwise.
|
||||||
|
*/
|
||||||
|
def textResult: String = result match {
|
||||||
|
case Left(_) => "failed"
|
||||||
|
case Right(_) => "passed"
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return
|
||||||
|
* The duration, in milliseconds, it took this test to execute.
|
||||||
|
*/
|
||||||
|
def millis: Long = duration.toMillis
|
||||||
|
|
||||||
object TestExecution:
|
object TestExecution:
|
||||||
|
|
||||||
|
|
|
@ -11,7 +11,7 @@ final class NoopReporter[F[_]: Applicative] extends Reporter[F]:
|
||||||
|
|
||||||
/** @inheritDocs
|
/** @inheritDocs
|
||||||
*/
|
*/
|
||||||
override def beginReporting(): F[Unit] = Applicative[F].unit
|
override def startReport(): F[Unit] = Applicative[F].unit
|
||||||
|
|
||||||
/** @inheritDocs
|
/** @inheritDocs
|
||||||
*/
|
*/
|
||||||
|
@ -27,4 +27,4 @@ final class NoopReporter[F[_]: Applicative] extends Reporter[F]:
|
||||||
|
|
||||||
/** @inheritDocs
|
/** @inheritDocs
|
||||||
*/
|
*/
|
||||||
override def endReporting(): F[Unit] = Applicative[F].unit
|
override def endReport(): F[Unit] = Applicative[F].unit
|
||||||
|
|
|
@ -6,6 +6,14 @@ import gs.test.v0.api.TestExecution
|
||||||
|
|
||||||
final class NoopResultFormatter extends ResultFormatter:
|
final class NoopResultFormatter extends ResultFormatter:
|
||||||
|
|
||||||
|
/** @inheritDocs
|
||||||
|
*/
|
||||||
|
override def prefix: String = ""
|
||||||
|
|
||||||
|
/** @inheritDocs
|
||||||
|
*/
|
||||||
|
override def suffix: String = ""
|
||||||
|
|
||||||
/** @inheritDocs
|
/** @inheritDocs
|
||||||
*/
|
*/
|
||||||
override def formatGroupResult(groupResult: GroupResult): String = ""
|
override def formatGroupResult(groupResult: GroupResult): String = ""
|
||||||
|
|
|
@ -0,0 +1,118 @@
|
||||||
|
package gs.test.v0.reporting
|
||||||
|
|
||||||
|
import cats.effect.Async
|
||||||
|
import cats.effect.Concurrent
|
||||||
|
import cats.effect.Resource
|
||||||
|
import cats.effect.kernel.Fiber
|
||||||
|
import cats.effect.std.Queue
|
||||||
|
import cats.effect.syntax.all.*
|
||||||
|
import cats.syntax.all.*
|
||||||
|
import fs2.text
|
||||||
|
import gs.test.v0.api.GroupResult
|
||||||
|
import gs.test.v0.api.SuiteExecution
|
||||||
|
import gs.test.v0.api.TestExecution
|
||||||
|
import java.io.OutputStream
|
||||||
|
|
||||||
|
/** Implementation of [[Reporter]] that writes bytes to an `OutputStream`.
|
||||||
|
*
|
||||||
|
* @param formatter
|
||||||
|
* The [[ResultFormatter]] used to render test results.
|
||||||
|
* @param state
|
||||||
|
* The internal state of the reporter.
|
||||||
|
*/
|
||||||
|
final class OutputStreamReporter[F[_]: Async] private (
|
||||||
|
formatter: ResultFormatter,
|
||||||
|
state: OutputStreamReporter.State[F]
|
||||||
|
) extends Reporter[F]:
|
||||||
|
|
||||||
|
/** @inheritDocs
|
||||||
|
*/
|
||||||
|
override def startReport(): F[Unit] =
|
||||||
|
write(formatter.prefix)
|
||||||
|
|
||||||
|
/** @inheritDocs
|
||||||
|
*/
|
||||||
|
override def reportGroup(
|
||||||
|
groupResult: GroupResult,
|
||||||
|
testExecutions: List[TestExecution]
|
||||||
|
): F[Unit] =
|
||||||
|
write(formatter.formatGroupResult(groupResult)) *>
|
||||||
|
testExecutions
|
||||||
|
.map(formatter.formatTestExecution)
|
||||||
|
.map(write)
|
||||||
|
.sequence
|
||||||
|
.as(())
|
||||||
|
|
||||||
|
/** @inheritDocs
|
||||||
|
*/
|
||||||
|
override def reportSuite(suiteExecution: SuiteExecution): F[Unit] =
|
||||||
|
write(formatter.formatSuiteExecution(suiteExecution))
|
||||||
|
|
||||||
|
/** @inheritDocs
|
||||||
|
*/
|
||||||
|
override def endReport(): F[Unit] =
|
||||||
|
write(formatter.suffix)
|
||||||
|
|
||||||
|
private def write(output: String): F[Unit] =
|
||||||
|
state.queue.offer(Some(output))
|
||||||
|
|
||||||
|
/** Produce an effect that, when executed, will cause the underlying stream to
|
||||||
|
* terminate. After executing this effect, the `OutputStreamReporter` will no
|
||||||
|
* longer be capable of writing more output.
|
||||||
|
*
|
||||||
|
* @return
|
||||||
|
* The effect that describes the stop operation.
|
||||||
|
*/
|
||||||
|
def stop(): F[Unit] = state.queue.offer(None)
|
||||||
|
|
||||||
|
object OutputStreamReporter:
|
||||||
|
|
||||||
|
/** Provision a new [[OutputStreamReporter]].
|
||||||
|
*
|
||||||
|
* @param formatter
|
||||||
|
* The [[ResultFormatter]] this reporter should use to render test results.
|
||||||
|
* @param output
|
||||||
|
* Resource which manages the `OutputStream` where bytes will be written.
|
||||||
|
* @return
|
||||||
|
* Resource which manages the [[OutputStreamReporter]].
|
||||||
|
*/
|
||||||
|
def provision[F[_]: Concurrent: Async](
|
||||||
|
formatter: ResultFormatter,
|
||||||
|
output: Resource[F, OutputStream]
|
||||||
|
): Resource[F, OutputStreamReporter[F]] =
|
||||||
|
output.flatMap { os =>
|
||||||
|
Resource.make(acquireReporter(formatter, os))(_.stop())
|
||||||
|
}
|
||||||
|
|
||||||
|
private def acquireReporter[F[_]: Concurrent: Async](
|
||||||
|
formatter: ResultFormatter,
|
||||||
|
output: OutputStream
|
||||||
|
): F[OutputStreamReporter[F]] =
|
||||||
|
for
|
||||||
|
queue <- Queue.unbounded[F, Option[String]]
|
||||||
|
process <- startProcess[F](queue, output)
|
||||||
|
yield new OutputStreamReporter[F](
|
||||||
|
formatter = formatter,
|
||||||
|
state = new State[F](queue, process)
|
||||||
|
)
|
||||||
|
|
||||||
|
private def startProcess[F[_]: Concurrent: Async](
|
||||||
|
queue: Queue[F, Option[String]],
|
||||||
|
output: OutputStream
|
||||||
|
): F[Fiber[F, Throwable, Unit]] =
|
||||||
|
fs2.Stream
|
||||||
|
.fromQueueNoneTerminated(queue)
|
||||||
|
.through(text.utf8.encode)
|
||||||
|
.through(
|
||||||
|
fs2.io.writeOutputStream(Async[F].delay(output), closeAfterUse = false)
|
||||||
|
)
|
||||||
|
.compile
|
||||||
|
.drain
|
||||||
|
.start
|
||||||
|
|
||||||
|
private class State[F[_]](
|
||||||
|
val queue: Queue[F, Option[String]],
|
||||||
|
val process: Fiber[F, Throwable, Unit]
|
||||||
|
)
|
||||||
|
|
||||||
|
end OutputStreamReporter
|
|
@ -0,0 +1,60 @@
|
||||||
|
package gs.test.v0.reporting
|
||||||
|
|
||||||
|
import cats.syntax.all.*
|
||||||
|
import gs.test.v0.api.GroupResult
|
||||||
|
import gs.test.v0.api.SuiteExecution
|
||||||
|
import gs.test.v0.api.TestExecution
|
||||||
|
import gs.test.v0.api.TestFailure
|
||||||
|
|
||||||
|
/** Implmentation of [[ResultFormatter]] that uses an unstructured text format.
|
||||||
|
*/
|
||||||
|
final class PlainResultFormatter extends ResultFormatter:
|
||||||
|
|
||||||
|
/** @inheritDocs
|
||||||
|
*/
|
||||||
|
override def prefix: String = ""
|
||||||
|
|
||||||
|
/** @inheritDocs
|
||||||
|
*/
|
||||||
|
override def suffix: String = ""
|
||||||
|
|
||||||
|
/** @inheritDocs
|
||||||
|
*/
|
||||||
|
override def formatGroupResult(groupResult: GroupResult): String =
|
||||||
|
def gr = groupResult
|
||||||
|
s"""
|
||||||
|
Group: '${gr.name.show}'
|
||||||
|
Stats: Seen=${gr.seen} Passed=${gr.passed} Failed=${gr.failed}
|
||||||
|
Duration: ${gr.millis}ms
|
||||||
|
Docs: ${gr.documentation.getOrElse("None")}
|
||||||
|
""".stripMargin
|
||||||
|
|
||||||
|
/** @inheritDocs
|
||||||
|
*/
|
||||||
|
override def formatTestExecution(testExecution: TestExecution): String =
|
||||||
|
def te = testExecution
|
||||||
|
s"""
|
||||||
|
Test: ${te.permanentId.show} (id=${te.id.show}) (span=${te.spanId})
|
||||||
|
Result: *${te.textResult}* in ${te.millis}ms
|
||||||
|
Tags: ${te.tags.mkString(", ")}
|
||||||
|
Docs: ${te.documentation.getOrElse("None")}${makeFailure(te.result)}
|
||||||
|
""".stripMargin
|
||||||
|
|
||||||
|
/** @inheritDocs
|
||||||
|
*/
|
||||||
|
override def formatSuiteExecution(suiteExecution: SuiteExecution): String =
|
||||||
|
def se = suiteExecution
|
||||||
|
s"""
|
||||||
|
Suite: '${se.testSuite.permanentId.show}' (id=${se.id.str}) (trace=${se.traceId})
|
||||||
|
Name: ${se.testSuite.name}
|
||||||
|
Stats: Seen=${se.seen} Passed=${se.passed} Failed=${se.failed}
|
||||||
|
Duration: ${se.millis}ms
|
||||||
|
""".stripMargin
|
||||||
|
|
||||||
|
private def makeFailure(result: Either[TestFailure, Any]): String =
|
||||||
|
result match
|
||||||
|
case Right(_) => ""
|
||||||
|
case Left(f) =>
|
||||||
|
s"""\n------
|
||||||
|
${f.message}
|
||||||
|
""".stripMargin
|
|
@ -22,7 +22,7 @@ trait Reporter[F[_]]:
|
||||||
* implementations to perform "setup" actions, such as opening a JSON object
|
* implementations to perform "setup" actions, such as opening a JSON object
|
||||||
* or writing a header.
|
* or writing a header.
|
||||||
*/
|
*/
|
||||||
def beginReporting(): F[Unit]
|
def startReport(): F[Unit]
|
||||||
|
|
||||||
/** Report the results of a single group.
|
/** Report the results of a single group.
|
||||||
*
|
*
|
||||||
|
@ -51,7 +51,7 @@ trait Reporter[F[_]]:
|
||||||
* to perform "finish" actions, such as closing a JSON object or writing a
|
* to perform "finish" actions, such as closing a JSON object or writing a
|
||||||
* footer.
|
* footer.
|
||||||
*/
|
*/
|
||||||
def endReporting(): F[Unit]
|
def endReport(): F[Unit]
|
||||||
|
|
||||||
object Reporter:
|
object Reporter:
|
||||||
|
|
||||||
|
|
|
@ -11,6 +11,16 @@ import gs.test.v0.api.TestExecution
|
||||||
* representations.
|
* representations.
|
||||||
*/
|
*/
|
||||||
trait ResultFormatter:
|
trait ResultFormatter:
|
||||||
|
/** @return
|
||||||
|
* The prefix for the format (if any).
|
||||||
|
*/
|
||||||
|
def prefix: String
|
||||||
|
|
||||||
|
/** @return
|
||||||
|
* The suffix for the format (if any).
|
||||||
|
*/
|
||||||
|
def suffix: String
|
||||||
|
|
||||||
/** Format a single [[GroupResult]] as a string.
|
/** Format a single [[GroupResult]] as a string.
|
||||||
*
|
*
|
||||||
* @param groupResult
|
* @param groupResult
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
package gs.test.v0.runtime.engine
|
package gs.test.v0.runtime.engine
|
||||||
|
|
||||||
|
import gs.uuid.v0.UUID
|
||||||
|
import java.time.Clock
|
||||||
|
|
||||||
/** Used to control the behavior of some [[TestEngine]]
|
/** Used to control the behavior of some [[TestEngine]]
|
||||||
*
|
*
|
||||||
* @param groupConcurrency
|
* @param groupConcurrency
|
||||||
|
@ -8,8 +11,17 @@ package gs.test.v0.runtime.engine
|
||||||
* @param testConcurrency
|
* @param testConcurrency
|
||||||
* [[ConcurrencySetting]] for tests; the number of tests allowed to execute
|
* [[ConcurrencySetting]] for tests; the number of tests allowed to execute
|
||||||
* at the same time within some group.
|
* at the same time within some group.
|
||||||
|
* @param clock
|
||||||
|
* The `Clock` instance used to inform all date/time operations.
|
||||||
|
* @param suiteIdGenerator
|
||||||
|
* UUID provider that is used at the suite level.
|
||||||
|
* @param testIdGenerator
|
||||||
|
* UUID provider that is used at the test level.
|
||||||
*/
|
*/
|
||||||
case class EngineConfiguration(
|
case class EngineConfiguration(
|
||||||
groupConcurrency: ConcurrencySetting,
|
groupConcurrency: ConcurrencySetting,
|
||||||
testConcurrency: ConcurrencySetting
|
testConcurrency: ConcurrencySetting,
|
||||||
|
clock: Clock,
|
||||||
|
suiteIdGenerator: UUID.Generator,
|
||||||
|
testIdGenerator: UUID.Generator
|
||||||
)
|
)
|
||||||
|
|
|
@ -4,9 +4,11 @@ object EngineConstants:
|
||||||
|
|
||||||
object Tracing:
|
object Tracing:
|
||||||
|
|
||||||
val RootSpan: String = "test-group"
|
val RootSpan: String = "suite"
|
||||||
|
val FullGroup: String = "full-group"
|
||||||
val BeforeGroup: String = "before-group"
|
val BeforeGroup: String = "before-group"
|
||||||
val AfterGroup: String = "after-group"
|
val AfterGroup: String = "after-group"
|
||||||
|
val FullTest: String = "full-test"
|
||||||
val BeforeTest: String = "before-test"
|
val BeforeTest: String = "before-test"
|
||||||
val AfterTest: String = "after-test"
|
val AfterTest: String = "after-test"
|
||||||
val InGroup: String = "in-group"
|
val InGroup: String = "in-group"
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package gs.test.v0.runtime.engine
|
package gs.test.v0.runtime.engine
|
||||||
|
|
||||||
import cats.effect.Async
|
import cats.effect.Async
|
||||||
|
import cats.effect.Resource
|
||||||
import cats.syntax.all.*
|
import cats.syntax.all.*
|
||||||
import gs.test.v0.api.GroupResult
|
import gs.test.v0.api.GroupResult
|
||||||
import gs.test.v0.api.SuiteExecution
|
import gs.test.v0.api.SuiteExecution
|
||||||
|
@ -11,8 +12,6 @@ import gs.test.v0.api.TestGroupDefinition
|
||||||
import gs.test.v0.api.TestSuite
|
import gs.test.v0.api.TestSuite
|
||||||
import gs.test.v0.reporting.Reporter
|
import gs.test.v0.reporting.Reporter
|
||||||
import gs.timing.v0.Timing
|
import gs.timing.v0.Timing
|
||||||
import gs.uuid.v0.UUID
|
|
||||||
import java.time.Clock
|
|
||||||
import java.time.Instant
|
import java.time.Instant
|
||||||
import natchez.EntryPoint
|
import natchez.EntryPoint
|
||||||
import natchez.Span
|
import natchez.Span
|
||||||
|
@ -47,75 +46,79 @@ import natchez.Span
|
||||||
*
|
*
|
||||||
* ## OpenTelemetry Support
|
* ## OpenTelemetry Support
|
||||||
*
|
*
|
||||||
* Each [[SuiteExecution]] produces a single trace per [[TestGroupDefinition]].
|
* Each [[SuiteExecution]] produces a single trace that encompasses all tests.
|
||||||
* This means that each group has a Trace ID and a tree of execution, with one
|
* Spans are used to designate different related portions of work.
|
||||||
* span per test.
|
|
||||||
*/
|
*/
|
||||||
final class TestEngine[F[_]: Async](
|
final class TestEngine[F[_]: Async](
|
||||||
val configuration: EngineConfiguration,
|
val configuration: EngineConfiguration,
|
||||||
reporter: Reporter[F],
|
reporter: Reporter[F],
|
||||||
timing: Timing[F],
|
entryPoint: EntryPoint[F],
|
||||||
suiteExecutionIdGenerator: UUID.Generator,
|
timing: Timing[F]
|
||||||
testExecutionIdGenerator: UUID.Generator,
|
|
||||||
clock: Clock,
|
|
||||||
val entryPoint: EntryPoint[F]
|
|
||||||
):
|
):
|
||||||
|
|
||||||
|
private def clock = configuration.clock
|
||||||
|
private def testIdGen = configuration.testIdGenerator
|
||||||
|
private def suiteIdGen = configuration.suiteIdGenerator
|
||||||
|
|
||||||
def runSuite(
|
def runSuite(
|
||||||
suite: TestSuite,
|
suite: TestSuite,
|
||||||
tests: fs2.Stream[F, TestGroupDefinition[F]]
|
tests: fs2.Stream[F, TestGroupDefinition[F]]
|
||||||
): F[SuiteExecution] =
|
): F[SuiteExecution] =
|
||||||
// TODO: REPORTING -- need interface
|
entryPoint.root(EngineConstants.Tracing.RootSpan).use { rootSpan =>
|
||||||
for
|
for
|
||||||
executedAt <- Async[F].delay(Instant.now(clock))
|
executedAt <- Async[F].delay(Instant.now(clock))
|
||||||
stats <- EngineStats.initialize[F]
|
stats <- EngineStats.initialize[F]
|
||||||
|
|
||||||
// Start reporting
|
// Start reporting
|
||||||
_ <- reporter.beginReporting()
|
_ <- reporter.startReport()
|
||||||
|
|
||||||
// TODO: Just do telemetry for the whole damn thing.
|
// Run all tests, group by group.
|
||||||
_ <- tests
|
_ <- tests
|
||||||
.mapAsync(configuration.groupConcurrency.toInt())(runGroup)
|
.mapAsync(configuration.groupConcurrency.toInt())(
|
||||||
.evalTap(
|
runGroup(rootSpan, _)
|
||||||
(
|
)
|
||||||
groupResult,
|
.evalTap(
|
||||||
testExecutions
|
(
|
||||||
) =>
|
groupResult,
|
||||||
for
|
testExecutions
|
||||||
// Update the overall statistics based on this group.
|
) =>
|
||||||
_ <- stats.updateForGroup(
|
for
|
||||||
duration = groupResult.duration,
|
// Update the overall statistics based on this group.
|
||||||
testExecutions = testExecutions
|
_ <- stats.updateForGroup(
|
||||||
)
|
duration = groupResult.duration,
|
||||||
|
testExecutions = testExecutions
|
||||||
|
)
|
||||||
|
|
||||||
// Report group level results for this group.
|
// Report group level results for this group.
|
||||||
_ <- reporter.reportGroup(
|
_ <- reporter.reportGroup(
|
||||||
groupResult = groupResult,
|
groupResult = groupResult,
|
||||||
testExecutions = testExecutions
|
testExecutions = testExecutions
|
||||||
)
|
)
|
||||||
yield ()
|
yield ()
|
||||||
)
|
)
|
||||||
.compile
|
.compile
|
||||||
.drain
|
.drain
|
||||||
|
|
||||||
// Calculate the final summary of execution at the suite level.
|
// Calculate the final summary of execution at the suite level.
|
||||||
suiteExecution <- makeSuiteExecution(suite, stats, executedAt)
|
suiteExecution <- makeSuiteExecution(rootSpan, suite, stats, executedAt)
|
||||||
|
|
||||||
// Report suite level results.
|
// Report suite level results.
|
||||||
_ <- reporter.reportSuite(suiteExecution)
|
_ <- reporter.reportSuite(suiteExecution)
|
||||||
|
|
||||||
// Finish reporting.
|
// Finish reporting.
|
||||||
_ <- reporter.endReporting()
|
_ <- reporter.endReport()
|
||||||
yield suiteExecution
|
yield suiteExecution
|
||||||
|
}
|
||||||
|
|
||||||
def runGroup(
|
def runGroup(
|
||||||
|
suiteSpan: Span[F],
|
||||||
group: TestGroupDefinition[F]
|
group: TestGroupDefinition[F]
|
||||||
): F[(GroupResult, List[TestExecution])] =
|
): F[(GroupResult, List[TestExecution])] =
|
||||||
entryPoint.root(EngineConstants.Tracing.RootSpan).use { rootSpan =>
|
suiteSpan.span(EngineConstants.Tracing.FullGroup).use { fullGroupSpan =>
|
||||||
for
|
for
|
||||||
groupStats <- EngineStats.initialize[F]
|
groupStats <- EngineStats.initialize[F]
|
||||||
// Augment the span with all group-level metadata.
|
// Augment the span with all group-level metadata.
|
||||||
_ <- rootSpan
|
_ <- fullGroupSpan
|
||||||
.put(EngineConstants.MetaData.TestGroupName -> group.name.show)
|
.put(EngineConstants.MetaData.TestGroupName -> group.name.show)
|
||||||
|
|
||||||
// Start the timer for the entire group.
|
// Start the timer for the entire group.
|
||||||
|
@ -124,17 +127,17 @@ final class TestEngine[F[_]: Async](
|
||||||
// Run the before-group logic (in its own span).
|
// Run the before-group logic (in its own span).
|
||||||
_ <- runSpan(
|
_ <- runSpan(
|
||||||
EngineConstants.Tracing.BeforeGroup,
|
EngineConstants.Tracing.BeforeGroup,
|
||||||
rootSpan,
|
fullGroupSpan,
|
||||||
group.beforeGroup.getOrElse(Async[F].unit)
|
group.beforeGroup.getOrElse(Async[F].unit)
|
||||||
)
|
)
|
||||||
|
|
||||||
// Execute all tests within this group.
|
// Execute all tests within this group.
|
||||||
testExecutions <- executeGroupTests(group, rootSpan)
|
testExecutions <- executeGroupTests(group, fullGroupSpan)
|
||||||
|
|
||||||
// Run the after-group logic (in its own span).
|
// Run the after-group logic (in its own span).
|
||||||
_ <- runSpan(
|
_ <- runSpan(
|
||||||
EngineConstants.Tracing.AfterGroup,
|
EngineConstants.Tracing.AfterGroup,
|
||||||
rootSpan,
|
fullGroupSpan,
|
||||||
group.afterGroup.getOrElse(Async[F].unit)
|
group.afterGroup.getOrElse(Async[F].unit)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -163,66 +166,67 @@ final class TestEngine[F[_]: Async](
|
||||||
|
|
||||||
private def executeGroupTests(
|
private def executeGroupTests(
|
||||||
group: TestGroupDefinition[F],
|
group: TestGroupDefinition[F],
|
||||||
rootSpan: Span[F]
|
fullGroupSpan: Span[F]
|
||||||
): F[List[TestExecution]] =
|
): F[List[TestExecution]] =
|
||||||
rootSpan.span(EngineConstants.Tracing.InGroup).use { groupSpan =>
|
fullGroupSpan.span(EngineConstants.Tracing.InGroup).use { groupSpan =>
|
||||||
for
|
for
|
||||||
// If, for some reason, the generated span has no Trace ID, this will
|
// If, for some reason, the generated span has no Trace ID, this will
|
||||||
// throw an exception.
|
// throw an exception.
|
||||||
traceId <- rootSpan.traceId.map(parseTraceId)
|
executions <- streamGroupTests(group, groupSpan).compile.toList
|
||||||
executions <- streamGroupTests(traceId, group, groupSpan).compile.toList
|
|
||||||
yield executions
|
yield executions
|
||||||
}
|
}
|
||||||
|
|
||||||
private def streamGroupTests(
|
private def streamGroupTests(
|
||||||
traceId: UUID,
|
|
||||||
group: TestGroupDefinition[F],
|
group: TestGroupDefinition[F],
|
||||||
groupSpan: Span[F]
|
groupSpan: Span[F]
|
||||||
): fs2.Stream[F, TestExecution] =
|
): fs2.Stream[F, TestExecution] =
|
||||||
fs2.Stream
|
fs2.Stream
|
||||||
.emits(group.tests)
|
.emits(group.tests)
|
||||||
.mapAsync(configuration.testConcurrency.toInt()) { test =>
|
.mapAsync(configuration.testConcurrency.toInt()) { test =>
|
||||||
for
|
groupSpan.span(EngineConstants.Tracing.FullTest).use { fullSpan =>
|
||||||
// Generate a unique TestExecutionId for this execution.
|
for
|
||||||
testExecutionId <- Async[F].delay(
|
// Generate a unique TestExecutionId for this execution.
|
||||||
TestExecution.Id(testExecutionIdGenerator.next())
|
testExecutionId <- Async[F].delay(
|
||||||
|
TestExecution.Id(testIdGen.next())
|
||||||
|
)
|
||||||
|
|
||||||
|
testSpanId <- fullSpan.spanId.map(parseSpanId)
|
||||||
|
|
||||||
|
// Start the timer for the test, including the before/after
|
||||||
|
// components.
|
||||||
|
timer <- timing.start()
|
||||||
|
|
||||||
|
// Run the before-test logic (in its own span).
|
||||||
|
_ <- runSpan(
|
||||||
|
EngineConstants.Tracing.BeforeTest,
|
||||||
|
groupSpan,
|
||||||
|
group.beforeEachTest.getOrElse(Async[F].unit)
|
||||||
|
)
|
||||||
|
|
||||||
|
// Run the test (in its own span).
|
||||||
|
result <- runSingleTest(testExecutionId, test, groupSpan)
|
||||||
|
|
||||||
|
// Run the after-test logic (in its own span).
|
||||||
|
_ <- runSpan(
|
||||||
|
EngineConstants.Tracing.AfterTest,
|
||||||
|
groupSpan,
|
||||||
|
group.afterEachTest.getOrElse(Async[F].unit)
|
||||||
|
)
|
||||||
|
|
||||||
|
// Calculate the overall elapsed time for this single test.
|
||||||
|
elapsed <- timer.checkpoint()
|
||||||
|
yield TestExecution(
|
||||||
|
id = testExecutionId,
|
||||||
|
permanentId = test.permanentId,
|
||||||
|
documentation = test.documentation,
|
||||||
|
tags = test.tags,
|
||||||
|
markers = test.markers,
|
||||||
|
result = result,
|
||||||
|
spanId = testSpanId,
|
||||||
|
sourcePosition = test.sourcePosition,
|
||||||
|
duration = elapsed.duration
|
||||||
)
|
)
|
||||||
|
}
|
||||||
// Start the timer for the test, including the before/after
|
|
||||||
// components.
|
|
||||||
timer <- timing.start()
|
|
||||||
|
|
||||||
// Run the before-test logic (in its own span).
|
|
||||||
_ <- runSpan(
|
|
||||||
EngineConstants.Tracing.BeforeTest,
|
|
||||||
groupSpan,
|
|
||||||
group.beforeEachTest.getOrElse(Async[F].unit)
|
|
||||||
)
|
|
||||||
|
|
||||||
// Run the test (in its own span).
|
|
||||||
result <- runSingleTest(testExecutionId, test, groupSpan)
|
|
||||||
|
|
||||||
// Run the after-test logic (in its own span).
|
|
||||||
_ <- runSpan(
|
|
||||||
EngineConstants.Tracing.AfterTest,
|
|
||||||
groupSpan,
|
|
||||||
group.afterEachTest.getOrElse(Async[F].unit)
|
|
||||||
)
|
|
||||||
|
|
||||||
// Calculate the overall elapsed time for this single test.
|
|
||||||
elapsed <- timer.checkpoint()
|
|
||||||
yield TestExecution(
|
|
||||||
id = testExecutionId,
|
|
||||||
permanentId = test.permanentId,
|
|
||||||
documentation = test.documentation,
|
|
||||||
tags = test.tags,
|
|
||||||
markers = test.markers,
|
|
||||||
result = result,
|
|
||||||
// TODO: TraceID isn't that useful here, need SpanID
|
|
||||||
traceId = traceId,
|
|
||||||
sourcePosition = test.sourcePosition,
|
|
||||||
duration = elapsed.duration
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private def runSingleTest(
|
private def runSingleTest(
|
||||||
|
@ -239,31 +243,38 @@ final class TestEngine[F[_]: Async](
|
||||||
yield result
|
yield result
|
||||||
}
|
}
|
||||||
|
|
||||||
private def parseTraceId(candidate: Option[String]): UUID =
|
private def parseTraceId(candidate: Option[String]): String =
|
||||||
candidate.flatMap(UUID.parse) match
|
candidate match
|
||||||
case Some(traceId) => traceId
|
case Some(traceId) => traceId
|
||||||
case None =>
|
case None =>
|
||||||
throw new IllegalArgumentException(
|
throw new IllegalArgumentException("Created a span without a Trace ID!")
|
||||||
"Created a span with an invalid Trace ID: " + candidate
|
|
||||||
)
|
private def parseSpanId(candidate: Option[String]): String =
|
||||||
|
candidate match
|
||||||
|
case Some(spanId) => spanId
|
||||||
|
case None =>
|
||||||
|
throw new IllegalArgumentException("Created a span without a Span ID!")
|
||||||
|
|
||||||
private def makeSuiteExecution(
|
private def makeSuiteExecution(
|
||||||
|
rootSpan: Span[F],
|
||||||
suite: TestSuite,
|
suite: TestSuite,
|
||||||
stats: EngineStats[F],
|
stats: EngineStats[F],
|
||||||
executedAt: Instant
|
executedAt: Instant
|
||||||
): F[SuiteExecution] =
|
): F[SuiteExecution] =
|
||||||
for
|
for
|
||||||
|
traceId <- rootSpan.traceId.map(parseTraceId)
|
||||||
overallDuration <- stats.duration
|
overallDuration <- stats.duration
|
||||||
countSeen <- stats.seen
|
seen <- stats.seen
|
||||||
countPassed <- stats.passed
|
passed <- stats.passed
|
||||||
countFailed <- stats.failed
|
failed <- stats.failed
|
||||||
yield SuiteExecution(
|
yield SuiteExecution(
|
||||||
id = suiteExecutionIdGenerator.next(),
|
id = suiteIdGen.next(),
|
||||||
testSuite = suite,
|
testSuite = suite,
|
||||||
|
traceId = traceId,
|
||||||
duration = overallDuration,
|
duration = overallDuration,
|
||||||
countSeen = countSeen,
|
seen = seen,
|
||||||
countPassed = countPassed,
|
passed = passed,
|
||||||
countFailed = countFailed,
|
failed = failed,
|
||||||
executedAt = executedAt
|
executedAt = executedAt
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -284,3 +295,36 @@ final class TestEngine[F[_]: Async](
|
||||||
f: F[A]
|
f: F[A]
|
||||||
): F[A] =
|
): F[A] =
|
||||||
root.span(name).use(_ => f)
|
root.span(name).use(_ => f)
|
||||||
|
|
||||||
|
object TestEngine:
|
||||||
|
|
||||||
|
/** Provision a new [[TestEngine]].
|
||||||
|
*
|
||||||
|
* @param configuration
|
||||||
|
* The [[EngineConfiguration]] used for this instance.
|
||||||
|
* @param reporter
|
||||||
|
* Resource which manages the [[Reporter]].
|
||||||
|
* @param entryPoint
|
||||||
|
* Resource which manages the telemetry entry point.
|
||||||
|
* @param timing
|
||||||
|
* Timing controller.
|
||||||
|
* @return
|
||||||
|
* Resource which manages the [[TestEngine]].
|
||||||
|
*/
|
||||||
|
def provision[F[_]: Async](
|
||||||
|
configuration: EngineConfiguration,
|
||||||
|
reporter: Resource[F, Reporter[F]],
|
||||||
|
entryPoint: Resource[F, EntryPoint[F]],
|
||||||
|
timing: Timing[F]
|
||||||
|
): Resource[F, TestEngine[F]] =
|
||||||
|
for
|
||||||
|
r <- reporter
|
||||||
|
ep <- entryPoint
|
||||||
|
yield new TestEngine(
|
||||||
|
configuration = configuration,
|
||||||
|
reporter = r,
|
||||||
|
entryPoint = ep,
|
||||||
|
timing = timing
|
||||||
|
)
|
||||||
|
|
||||||
|
end TestEngine
|
||||||
|
|
Loading…
Add table
Reference in a new issue