From 311ab17d5f6274faf3a0b5021578355448d0a84b Mon Sep 17 00:00:00 2001 From: Pat Garrity Date: Thu, 11 Sep 2025 21:12:32 -0500 Subject: [PATCH] WIP: writing tests for the runtime and improving test data --- build.sbt | 21 +++- .../test/v0/reporting/InMemoryReporter.scala | 50 ++++++++++ .../test/v0/runtime/engine/TestEngine.scala | 12 +++ .../v0/runtime/engine/EngineStatsTests.scala | 77 +++++++++++++++ .../src/test/scala/support/Generators.scala | 99 +++++++++++++++++++ .../src/test/scala/support/Durations.scala | 11 +++ project/build.properties | 2 +- 7 files changed, 270 insertions(+), 2 deletions(-) create mode 100644 modules/reporting/src/main/scala/gs/test/v0/reporting/InMemoryReporter.scala create mode 100644 modules/runtime/src/test/scala/gs/test/v0/runtime/engine/EngineStatsTests.scala create mode 100644 modules/test-data/src/test/scala/support/Generators.scala create mode 100644 modules/test-support/src/test/scala/support/Durations.scala diff --git a/build.sbt b/build.sbt index 3f0e9a5..a965705 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,4 @@ -val scala3: String = "3.7.1" +val scala3: String = "3.7.2" ThisBuild / scalaVersion := scala3 ThisBuild / versionScheme := Some("semver-spec") @@ -101,6 +101,24 @@ lazy val api = project ) ) +/** Internal project used for generating test data. + */ +lazy val `test-data` = project + .in(file("modules/test-data")) + .dependsOn(api) + .settings(sharedSettings) + .settings(testSettings) + .settings(noPublishSettings) + .settings( + name := s"${gsProjectName.value}-test-data" + ) + .settings( + libraryDependencies ++= Seq( + Deps.Cats.Core, + Deps.Cats.Effect + ) + ) + /** Reporting API and implementations. */ lazy val reporting = project @@ -123,6 +141,7 @@ lazy val reporting = project lazy val runtime = project .in(file("modules/runtime")) .dependsOn(`test-support` % "test->test") + .dependsOn(`test-data` % "test->test") .dependsOn(api, reporting) .settings(sharedSettings) .settings(testSettings) diff --git a/modules/reporting/src/main/scala/gs/test/v0/reporting/InMemoryReporter.scala b/modules/reporting/src/main/scala/gs/test/v0/reporting/InMemoryReporter.scala new file mode 100644 index 0000000..eb31865 --- /dev/null +++ b/modules/reporting/src/main/scala/gs/test/v0/reporting/InMemoryReporter.scala @@ -0,0 +1,50 @@ +package gs.test.v0.reporting + +import cats.effect.Async +import cats.effect.Ref +import cats.effect.std.Queue +import cats.syntax.all.* +import gs.test.v0.api.GroupResult +import gs.test.v0.api.SuiteExecution +import gs.test.v0.api.TestExecution + +final class InMemoryReporter[F[_]: Async] private ( + suiteExecution: Ref[F, Option[SuiteExecution]], + groupResults: Queue[F, (GroupResult, List[TestExecution])] +) extends Reporter[F]: + + /** @inheritDocs + */ + override def startReport(): F[Unit] = Async[F].unit + + /** @inheritDocs + */ + override def reportGroup( + groupResult: GroupResult, + testExecutions: List[TestExecution] + ): F[Unit] = groupResults.offer(groupResult -> testExecutions) + + /** @inheritDocs + */ + override def reportSuite(suiteExecution: SuiteExecution): F[Unit] = + this.suiteExecution.set(Some(suiteExecution)) + + /** @inheritDocs + */ + override def endReport(): F[Unit] = Async[F].unit + + def getSuiteExecution(): F[Option[SuiteExecution]] = suiteExecution.get + + // TODO: make stream to consume and reify to list + def getGroupResults(): F[List[(GroupResult, List[TestExecution])]] = + ??? + +object InMemoryReporter: + + def initialize[F[_]: Async]: F[InMemoryReporter[F]] = + for + se <- Ref.of[F, Option[SuiteExecution]](None) + gr <- Queue.unbounded[F, (GroupResult, List[TestExecution])] + yield new InMemoryReporter(se, gr) + +end InMemoryReporter diff --git a/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/TestEngine.scala b/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/TestEngine.scala index 7b65e2a..b102235 100644 --- a/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/TestEngine.scala +++ b/modules/runtime/src/main/scala/gs/test/v0/runtime/engine/TestEngine.scala @@ -60,6 +60,18 @@ final class TestEngine[F[_]: Async]( private def testIdGen = configuration.testIdGenerator private def suiteIdGen = configuration.suiteIdGenerator + /** Execute a suite of tests. + * + * This function only provides a summary output. Results are streamed using a + * [[Reporter]] instance. + * + * @param suite + * The metadata that describes the suite. + * @param tests + * The stream of groups that define the tests. + * @return + * Summary of the execution. + */ def runSuite( suite: TestSuite, tests: fs2.Stream[F, TestGroupDefinition[F]] diff --git a/modules/runtime/src/test/scala/gs/test/v0/runtime/engine/EngineStatsTests.scala b/modules/runtime/src/test/scala/gs/test/v0/runtime/engine/EngineStatsTests.scala new file mode 100644 index 0000000..a144fa0 --- /dev/null +++ b/modules/runtime/src/test/scala/gs/test/v0/runtime/engine/EngineStatsTests.scala @@ -0,0 +1,77 @@ +package gs.test.v0.runtime.engine + +import cats.effect.IO +import gs.datagen.v0.Gen +import gs.datagen.v0.generators.Size +import java.util.concurrent.TimeUnit +import munit.* +import scala.concurrent.duration.FiniteDuration +import support.* + +class EngineStatsTests extends IOSuite: + + iotest("should initialize empty stats") { + for + stats <- EngineStats.initialize[IO] + duration <- stats.duration + seen <- stats.seen + passed <- stats.passed + failed <- stats.failed + yield + assertEquals(duration, Durations.Zero) + assertEquals(seen, 0L) + assertEquals(passed, 0L) + assertEquals(failed, 0L) + } + + iotest("should update based on a group with no test executions") { + val expected = FiniteDuration(2L, TimeUnit.MILLISECONDS) + for + stats <- EngineStats.initialize[IO] + _ <- stats.updateForGroup(Durations.OneMilli, Nil) + _ <- stats.updateForGroup(Durations.OneMilli, Nil) + duration <- stats.duration + seen <- stats.seen + passed <- stats.passed + failed <- stats.failed + yield + assertEquals(duration, expected) + assertEquals(seen, 0L) + assertEquals(passed, 0L) + assertEquals(failed, 0L) + } + + iotest("should update based on a single test execution") { + for + stats <- EngineStats.initialize[IO] + _ <- stats.updateForTest(Generators.testExecutionPassed()) + _ <- stats.updateForTest(Generators.testExecutionFailed()) + duration <- stats.duration + seen <- stats.seen + passed <- stats.passed + failed <- stats.failed + yield + assertEquals(duration, Durations.Zero) + assertEquals(seen, 2L) + assertEquals(passed, 1L) + assertEquals(failed, 1L) + } + + iotest("should update based on a test group") { + val duration = Generators.testDuration() + val size = 4 + val executions = + Gen.list(Size.fixed(size), Generators.GenTestExecutionPassed).gen() + for + stats <- EngineStats.initialize[IO] + _ <- stats.updateForGroup(duration, executions) + duration <- stats.duration + seen <- stats.seen + passed <- stats.passed + failed <- stats.failed + yield + assertEquals(duration, duration) + assertEquals(seen, size.toLong) + assertEquals(passed, size.toLong) + assertEquals(failed, 0L) + } diff --git a/modules/test-data/src/test/scala/support/Generators.scala b/modules/test-data/src/test/scala/support/Generators.scala new file mode 100644 index 0000000..6d6bef6 --- /dev/null +++ b/modules/test-data/src/test/scala/support/Generators.scala @@ -0,0 +1,99 @@ +package support + +import gs.datagen.v0.* +import gs.datagen.v0.generators.Size +import gs.test.v0.api.* +import scala.concurrent.duration.FiniteDuration + +object Generators: + + val NoSourcePosition: SourcePosition = SourcePosition("TEST", 0) + + val GenTestExecutionId: Gen[TestExecution.Id] = + Gen.uuid.random().map(TestExecution.Id(_)) + + given Generated[TestExecution.Id] = Generated.of(GenTestExecutionId) + + val GenPermanentId: Gen[PermanentId] = + Gen.string.alphaNumeric(Size.Fixed(12)).map(x => PermanentId(s"pid-$x")) + + given Generated[PermanentId] = Generated.of(GenPermanentId) + + val GenTag: Gen[Tag] = + Gen.string.alphaNumeric(Size.Fixed(6)).map(x => Tag(s"tag-$x")) + + val GenTagList: Gen[List[Tag]] = Gen.list(Size.between(0, 8), GenTag) + + given Generated[Tag] = Generated.of(GenTag) + + val GenTraceId: Gen[String] = Gen.uuid.string().map(_.filterNot(_ == '-')) + + val GenSpanId: Gen[String] = GenTraceId.map(_.take(16)) + + val GenTestDuration: Gen[FiniteDuration] = + Gen.duration.finiteMilliseconds(1L, 100L) + + val GenTestResult: Gen[Either[TestFailure, Any]] = + Gen.boolean().map(makeResult) + + private def makeResult(passed: Boolean): TestResult = + passed match { + case true => Right(()) + case false => + Left(TestFailure.TestRequestedFailure("Failed", NoSourcePosition)) + } + + val InputGenTestExecution: Datagen[TestExecution, Boolean] = + for + id <- GenTestExecutionId + permanentId <- GenPermanentId + tags <- GenTagList + spanId <- GenSpanId + duration <- GenTestDuration + yield (passed: Boolean) => + TestExecution( + id = id, + permanentId = permanentId, + documentation = None, + tags = tags, + markers = Nil, + result = makeResult(passed), + spanId = spanId, + sourcePosition = NoSourcePosition, + duration = duration + ) + + val GenTestExecution: Gen[TestExecution] = + Gen.boolean().map(passed => InputGenTestExecution.generate(passed)) + + val GenTestExecutionPassed: Gen[TestExecution] = + Gen.single(true).map(InputGenTestExecution.generate) + + val GenTestExecutionFailed: Gen[TestExecution] = + Gen.single(false).map(InputGenTestExecution.generate) + + given Generated[TestExecution] = Generated.of(GenTestExecution) + + def testExecutionId(): TestExecution.Id = GenTestExecutionId.gen() + + def permanentId(): PermanentId = GenPermanentId.gen() + + def tag(): Tag = GenTag.gen() + + def tags(): List[Tag] = GenTagList.gen() + + def traceId(): String = GenTraceId.gen() + + def spanId(): String = GenSpanId.gen() + + def testDuration(): FiniteDuration = GenTestDuration.gen() + + def testExecution(): TestExecution = GenTestExecution.gen() + + def testExecutionPassed(): TestExecution = + InputGenTestExecution.generate(true) + + def testExecutionFailed(): TestExecution = + InputGenTestExecution.generate(false) + +end Generators diff --git a/modules/test-support/src/test/scala/support/Durations.scala b/modules/test-support/src/test/scala/support/Durations.scala new file mode 100644 index 0000000..5cd5715 --- /dev/null +++ b/modules/test-support/src/test/scala/support/Durations.scala @@ -0,0 +1,11 @@ +package support + +import java.util.concurrent.TimeUnit +import scala.concurrent.duration.FiniteDuration + +object Durations: + + val Zero: FiniteDuration = FiniteDuration(0L, TimeUnit.NANOSECONDS) + val OneMilli: FiniteDuration = FiniteDuration(1L, TimeUnit.MILLISECONDS) + +end Durations diff --git a/project/build.properties b/project/build.properties index bbb0b60..5e6884d 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.11.2 +sbt.version=1.11.6