From 910f14a3c6576591ec708852dce8e5f2fbdee3d1 Mon Sep 17 00:00:00 2001 From: Pat Garrity Date: Fri, 14 Nov 2025 22:12:47 -0600 Subject: [PATCH] json module functions properly and is documented --- .scalafmt.conf | 2 +- build.sbt | 4 +- docs/README.md | 2 +- .../gs/predicate/v0/json/JsonKeyExists.scala | 5 +- .../v0/json/query/CompiledQuery.scala | 311 ++++++++++++++++++ .../predicate/v0/json/query/EmptyQuery.scala | 19 ++ .../predicate/v0/json/query/JsonQuery.scala | 51 +++ .../query/QueryCompilationException.scala | 25 ++ .../json/query/CompiledQueryEvalTests.scala | 173 ++++++++++ project/build.properties | 2 +- 10 files changed, 587 insertions(+), 7 deletions(-) create mode 100644 modules/json/src/main/scala/gs/predicate/v0/json/query/CompiledQuery.scala create mode 100644 modules/json/src/main/scala/gs/predicate/v0/json/query/EmptyQuery.scala create mode 100644 modules/json/src/main/scala/gs/predicate/v0/json/query/JsonQuery.scala create mode 100644 modules/json/src/main/scala/gs/predicate/v0/json/query/QueryCompilationException.scala create mode 100644 modules/json/src/test/scala/gs/predicate/v0/json/query/CompiledQueryEvalTests.scala diff --git a/.scalafmt.conf b/.scalafmt.conf index be3b2cb..14ea2f7 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,5 +1,5 @@ // See: https://github.com/scalameta/scalafmt/tags for the latest tags. -version = 3.9.9 +version = 3.10.1 runner.dialect = scala3 maxColumn = 80 diff --git a/build.sbt b/build.sbt index 381de67..ea39342 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,4 @@ -val scala3: String = "3.7.3" +val scala3: String = "3.7.4" ThisBuild / scalaVersion := scala3 ThisBuild / versionScheme := Some("semver-spec") @@ -33,6 +33,7 @@ val Deps = new { val Circe = new { val Core: ModuleID = "io.circe" %% "circe-core" % "0.14.15" val Parser: ModuleID = "io.circe" %% "circe-parser" % "0.14.15" + val Optics: ModuleID = "io.circe" %% "circe-optics" % "0.15.1" } val Gs = new { @@ -131,6 +132,7 @@ lazy val json = project Deps.Cats.Effect, Deps.Circe.Core, Deps.Circe.Parser, + Deps.Circe.Optics, Deps.Gs.Uuid ) ) diff --git a/docs/README.md b/docs/README.md index 97ec101..6466fd5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3,4 +3,4 @@ - [Terminology](./terminology.md) - [Predicates: Standard](./predicates-standard.md) - [Predicates: Key-Value](./predicates-key-value.md) -- [Predicates: JSON](#) +- [Predicates: JSON](./predicates-json.md) diff --git a/modules/json/src/main/scala/gs/predicate/v0/json/JsonKeyExists.scala b/modules/json/src/main/scala/gs/predicate/v0/json/JsonKeyExists.scala index 7eb9ae1..7334ebe 100644 --- a/modules/json/src/main/scala/gs/predicate/v0/json/JsonKeyExists.scala +++ b/modules/json/src/main/scala/gs/predicate/v0/json/JsonKeyExists.scala @@ -15,14 +15,13 @@ import io.circe.Json */ final class JsonKeyExists[F[_]: Applicative]( val id: UUID, - val key: String + val queryString: String ) extends Predicate[F, Json]: /** @inheritDocs */ override def eval(input: Json): F[Predicate.Result] = - // \\ is recursive, we need a path-based lookup or something - Applicative[F].pure(Predicate.Result(input.\\(key).nonEmpty)) + Applicative[F].pure(Predicate.Result(false)) object JsonKeyExists: diff --git a/modules/json/src/main/scala/gs/predicate/v0/json/query/CompiledQuery.scala b/modules/json/src/main/scala/gs/predicate/v0/json/query/CompiledQuery.scala new file mode 100644 index 0000000..b3ce080 --- /dev/null +++ b/modules/json/src/main/scala/gs/predicate/v0/json/query/CompiledQuery.scala @@ -0,0 +1,311 @@ +package gs.predicate.v0.json.query + +import io.circe.Json +import io.circe.optics.JsonPath +import io.circe.optics.JsonPath._ +import io.circe.optics.JsonTraversalPath +import scala.util.Failure +import scala.util.Success +import scala.util.Try +import scala.util.matching.Regex + +/** Implementation of [[JsonQuery]] that is capable of running a query on + * submitted JSON values. + * + * ## Query Syntax + * + * ### The Empty Query + * + * An empty string is the empty query - the identity function. Always produces + * the exact input it was provided. + * + * ### Keys + * + * Queries are based on keys - the selection of a named property of the object. + * + * ### Dot Accessor + * + * The '.' (dot) character is an _accessor_. Given a key to the left and a key + * to the right, '.' accesses the right-side property of the left-side value. + * + * ``` + * { "x": { "y": "z" } } + * ``` + * + * In this example, `x.y` selects (string) value `"z"`. + * + * ### Array Traversal + * + * There are two ways to traverse arrays: `[any]` and `[all]`. These only + * differ when evaluating predicates - they have identical query behavior. + * + * #### Any + * + * The `[any]` modifier to a key implies that key refers to a JSON array. It + * means that the predicate will match if it satisfies _any_ array element. + * + * ``` + * { "x": [ { "y": 1 }, { "y": 2 } ] } + * ``` + * + * The query `x[any].y` will select the value at `y` from each object in the + * array. When evaluating a predicate, it will match if _any_ objects match: + * + * ``` + * == Json.fromInt(2) + * ``` + * + * This predicate would match the above example. + * + * Writing `x[any]` will select the full objects. + * + * #### All + * + * The `[all]` modifier to a key implies that key refers to a JSON array. It + * means that the predicate will match if it satisfies _all_ array elements. + * + * ``` + * { "x": [ { "y": 1 }, { "y": 2 } ] } + * ``` + * + * The query `x[all].y` will select the value at `y` from each object in the + * array. When evaluating a predicate, it will match if _all_ objects match: + * + * ``` + * == Json.fromInt(2) + * ``` + * + * This predicate would NOT match the above example. + * + * Writing `x[all]` will select the full objects. + * + * ### Array Indexing + * + * The modifier `[]`, where `` is some integer, will select the + * value at that index if one exists. + * + * ``` + * { "x": [ { "y": 1 }, { "y": 2 } ] } + * ``` + * + * The query `x[0].y` will select the value at `y` for the first object in the + * array -- in this case, the value `1`. + * + * ### Examples + * + * Select property `x` nested beneath two arrays of objects. When evaluating + * predicates, select `any` from `bar` and require `all` from `baz`: + * + * ``` + * foo.bar[any].baz[all].x + * ``` + * + * Select property `z` nested beneath two arrays of objects. When evaluating + * predicates, require `all` from `bar` and select `any` from `baz`: + * + * ``` + * foo.bar[any].baz[all].z + * ``` + * + * Select property `z` nested beneath the second object in the `baz` array, as + * held by the first object in the `bar` array: + * + * ``` + * foo.bar[0].baz[1].z + * ``` + * + * @param parts + * Internal ordered list of query parts. + * @param reader + * Pre-calculated reader function for lookups. + * @param maxQueryDepth + * Maximum allowed query depth. + */ +final class CompiledQuery private ( + private val parts: List[CompiledQuery.QueryPart], + private val reader: Json => List[Json], + private val maxQueryDepth: Int +) extends JsonQuery: + + import CompiledQuery.* + + /** @inheritDocs + */ + override def eval( + json: Json, + p: Json => Boolean + ): Boolean = + evalRec(parts, json, root, p, 0) + + private def evalRec( + ps: List[QueryPart], + json: Json, + path: JsonPath, + p: Json => Boolean, + depth: Int + ): Boolean = + if depth >= maxQueryDepth then + throw new RuntimeException( + s"Illegal query depth '$depth' exceeds maximum depth '$maxQueryDepth'" + ) + else + ps match + case Nil => + path.json.getOption(json).map(p).getOrElse(false) + case Single(key) :: rest => + evalRec(rest, json, path.selectDynamic(key), p, depth + 1) + case ArrayAny(key) :: rest => + path + .selectDynamic(key) + .each + .json + .exist(evalRec(rest, _, root, p, depth + 1)) + .apply(json) + case ArrayAll(key) :: rest => + path + .selectDynamic(key) + .each + .json + .all(evalRec(rest, _, root, p, depth + 1)) + .apply(json) + case ArrayIndex(key, index) :: rest => + evalRec( + rest, + json, + path.selectDynamic(key).index(index), + p, + depth + 1 + ) + + /** @inheritDocs + */ + override def read(json: Json): List[Json] = reader(json) + + override def toString(): String = parts.map(_.toString()).mkString(".") + +object CompiledQuery: + + private val queryPartRegex: Regex = """^([^\[]+)(?:\[([^\]]+)\])?$""".r + + sealed trait QueryPart: + def hasListTraversal: Boolean + + case class Single(key: String) extends QueryPart: + override def toString(): String = key + override def hasListTraversal: Boolean = false + + case class ArrayAny(key: String) extends QueryPart: + override def toString(): String = s"$key[any]" + override def hasListTraversal: Boolean = true + + case class ArrayAll(key: String) extends QueryPart: + override def toString(): String = s"$key[all]" + override def hasListTraversal: Boolean = true + + case class ArrayIndex( + key: String, + index: Int + ) extends QueryPart: + override def toString(): String = s"$key[$index]" + override def hasListTraversal: Boolean = false + + /** Compile a new [[JsonQuery]]. + * + * Throws an exception if the query is invalid and cannot be compiled. + * + * @param query + * The raw query string to compile to an executable representation. + * @param maxQueryDepth + * The maximum allowed query depth (number of query parts). + * @return + * The new query. + */ + def compile( + query: String, + maxQueryDepth: Int = 1024 + ): JsonQuery = + if query.isBlank() then JsonQuery.empty() + else + val queryParts = query.split("\\.").toList.map(parseQueryPart) + if queryParts.size > maxQueryDepth then + throw IllegalArgumentException( + s"Attempted to compile a query with ${queryParts.size} parts, which exceeds the maximum depth ($maxQueryDepth)." + ) + else if queryParts.exists(_.hasListTraversal) then + new CompiledQuery( + queryParts, + readerWithTraversal(queryParts), + maxQueryDepth + ) + else + new CompiledQuery( + queryParts, + readerWithoutTraversal(queryParts), + maxQueryDepth + ) + + private def readerWithoutTraversal(queryParts: List[QueryPart]) + : Json => List[Json] = + input => { + var lens: JsonPath = root + + queryParts.foreach { + case Single(key) => + lens = lens.selectDynamic(key) + case ArrayIndex(key, index) => + lens = lens.selectDynamic(key).index(index) + case _ => + throw QueryCompilationException.logicalError( + "Did not expect a traversal but encountered one - this is likely a query compiler bug." + ) + } + + lens.json.getOption(input).toList + } + + private def readerWithTraversal(queryParts: List[QueryPart]) + : Json => List[Json] = + input => { + var lens: JsonPath = root + var tp: JsonTraversalPath = null + var inArr: Boolean = false + + queryParts.foreach { + case Single(key) => + if inArr then tp = lens.each.selectDynamic(key) + else lens = lens.selectDynamic(key) + case ArrayAny(key) => + inArr = true + lens = lens.selectDynamic(key) + case ArrayAll(key) => + inArr = true + lens = lens.selectDynamic(key) + case ArrayIndex(key, index) => + lens = lens.selectDynamic(key).index(index) + } + + if tp == null then + throw QueryCompilationException.logicalError( + "Expected a traversal but did not encounter one - this is likely a compiler bug." + ) + else tp.json.getAll(input) + } + + private def parseQueryPart(qp: String): QueryPart = + qp match + case queryPartRegex(key, null) => Single(key) + case queryPartRegex(key, arr) => + if arr == "any" then ArrayAny(key) + else if arr == "all" then ArrayAll(key) + else ArrayIndex(key, parseArrayIndex(arr)) + case _ => throw QueryCompilationException.invalidPart(qp) + + private def parseArrayIndex(candidate: String): Int = + Try(candidate.toInt) match + case Success(value) => + if value >= 0 then value + else throw QueryCompilationException(candidate) + case Failure(cause) => + throw QueryCompilationException.invalidPart(candidate, cause) + +end CompiledQuery diff --git a/modules/json/src/main/scala/gs/predicate/v0/json/query/EmptyQuery.scala b/modules/json/src/main/scala/gs/predicate/v0/json/query/EmptyQuery.scala new file mode 100644 index 0000000..0564a4f --- /dev/null +++ b/modules/json/src/main/scala/gs/predicate/v0/json/query/EmptyQuery.scala @@ -0,0 +1,19 @@ +package gs.predicate.v0.json.query + +import io.circe.Json + +/** Implementation of [[JsonQuery]] that always refers to the input value as-is. + */ +object EmptyQuery extends JsonQuery: + + /** @inheritDocs + */ + override def eval( + json: Json, + p: Json => Boolean + ): Boolean = + p(json) + + /** @inheritDocs + */ + override def read(json: Json): List[Json] = List(json) diff --git a/modules/json/src/main/scala/gs/predicate/v0/json/query/JsonQuery.scala b/modules/json/src/main/scala/gs/predicate/v0/json/query/JsonQuery.scala new file mode 100644 index 0000000..d5f9356 --- /dev/null +++ b/modules/json/src/main/scala/gs/predicate/v0/json/query/JsonQuery.scala @@ -0,0 +1,51 @@ +package gs.predicate.v0.json.query + +import io.circe.Json + +/** Predicate-integrated queries for JSON objects. + */ +trait JsonQuery: + + /** Query the given JSON value and evaluate the predicate as part of that + * query. + * + * @param json + * The input JSON value. + * @param p + * The predicate to evaluate. + * @return + * True if the predicate matches, false otherwise. + */ + def eval( + json: Json, + p: Json => Boolean + ): Boolean + + /** Evaluate the query against the input JSON, returning the result. + * + * @param json + * The input JSON value. + * @return + * The list of extracted JSON values. + */ + def read(json: Json): List[Json] + +object JsonQuery: + + /** @return + * The empty query. + */ + def empty(): JsonQuery = EmptyQuery + + /** Compile a query. + * + * Throws [[QueryCompilationException]] if an invalid query is supplied. + * + * @param query + * The query string. + * @return + * The compiled query. + */ + def compile(query: String): JsonQuery = CompiledQuery.compile(query) + +end JsonQuery diff --git a/modules/json/src/main/scala/gs/predicate/v0/json/query/QueryCompilationException.scala b/modules/json/src/main/scala/gs/predicate/v0/json/query/QueryCompilationException.scala new file mode 100644 index 0000000..1df038e --- /dev/null +++ b/modules/json/src/main/scala/gs/predicate/v0/json/query/QueryCompilationException.scala @@ -0,0 +1,25 @@ +package gs.predicate.v0.json.query + +final class QueryCompilationException( + val message: String, + val cause: Option[Throwable] = None +) extends RuntimeException(message, cause.getOrElse(null)) + +object QueryCompilationException: + + def invalidPart(candidate: String): QueryCompilationException = + new QueryCompilationException(candidate, None) + + def invalidPart( + candidate: String, + cause: Throwable + ): QueryCompilationException = + new QueryCompilationException(candidate, Some(cause)) + + def logicalError(message: String): QueryCompilationException = + new QueryCompilationException(message, None) + + def invalidPartMessage(candidate: String): String = + s"Failed to compile query. Candidate query part '$candidate' is invalid." + +end QueryCompilationException diff --git a/modules/json/src/test/scala/gs/predicate/v0/json/query/CompiledQueryEvalTests.scala b/modules/json/src/test/scala/gs/predicate/v0/json/query/CompiledQueryEvalTests.scala new file mode 100644 index 0000000..1963258 --- /dev/null +++ b/modules/json/src/test/scala/gs/predicate/v0/json/query/CompiledQueryEvalTests.scala @@ -0,0 +1,173 @@ +package gs.predicate.v0.json.query + +import cats.Eq +import io.circe.Json + +class CompiledQueryEvalTests extends munit.FunSuite: + import CompiledQueryEvalTests.Data + + test("should handle a single, top-level key (matching case)") { + val query = compile("key") + val expectedValue = Json.fromString("value") + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, true) + } + + test("should handle a single, top-level key (non-matching case)") { + val query = compile("missing") + val expectedValue = Json.fromString("value") + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, false) + } + + test( + "should handle an array-all nested beneath an array-any (matching case)" + ) { + val query = compile("foo.bar[any].baz[all].x") + val expectedValue = Json.fromInt(10) + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, true) + } + + test( + "should handle an array-all nested beneath an array-any (non-matching case)" + ) { + val query = compile("foo.bar[any].baz[all].z") + val expectedValue = Json.fromInt(0) + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, false) + } + + test( + "should handle an array-any nested beneath an array-all (matching case)" + ) { + val query = compile("foo.bar[all].baz[any].y") + val expectedValue = Json.fromString("a") + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, true) + } + + test( + "should handle an array-any nested beneath an array-all (non-matching case)" + ) { + val query = compile("foo.bar[all].baz[any].z") + val expectedValue = Json.fromInt(0) + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, false) + } + + test("should handle nested index (matching case)") { + val query = compile("foo.bar[0].baz[2].z") + val expectedValue = Json.fromInt(2) + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, true) + } + + test("should handle nested index (non-matching case)") { + val query = compile("foo.bar[0].baz[1].z") + val expectedValue = Json.fromInt(2) + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, false) + } + + test("should handle index after all (matching case)") { + val query = compile("foo.bar[all].baz[0].x") + val expectedValue = Json.fromInt(10) + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, true) + } + + test("should handle index after all (non-matching case)") { + val query = compile("foo.bar[all].baz[0].z") + val expectedValue = Json.fromInt(0) + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, false) + } + + test("should handle [any] raw array values (matching case)") { + val query = compile("rawValuesAny[any]") + val expectedValue = Json.fromInt(1) + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, true) + } + + test("should handle [any] raw array values (non-matching case)") { + val query = compile("rawValuesAny[any]") + val expectedValue = Json.fromInt(6) + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, false) + } + + test("should handle [all] raw array values (matching case)") { + val query = compile("rawValuesAll[all]") + val expectedValue = Json.fromInt(1) + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, true) + } + + test("should handle [all] raw array values (non-matching case)") { + val query = compile("rawValuesAll[all]") + val expectedValue = Json.fromInt(6) + val p = (json: Json) => Eq[Json].eqv(json, expectedValue) + val result = query.eval(Data.parsedJson, p) + assertEquals(result, false) + } + + private def compile(query: String): JsonQuery = CompiledQuery.compile(query) + +object CompiledQueryEvalTests: + + object Data: + + /** Reference data that supports all tests. + */ + val rawJson: String = """ + { + "foo": { + "bar": [ + { + "baz": [ + { "x": 10, "y": "a", "z": 0 }, + { "x": 10, "y": "a", "z": 1 }, + { "x": 10, "y": "a", "z": 2 } + ] + }, + { + "baz": [ + { "x": 10, "y": "a", "z": 3 }, + { "x": 20, "y": "a", "z": 4 } + ] + } + ] + }, + "key": "value", + "rawValuesAny": [ 1, 2, 3, 4, 5 ], + "rawValuesAll": [ 1, 1, 1, 1, 1 ] + } + """.stripMargin + + /** Typically passed for predicate/query evaluation. + */ + val parsedJson: Json = parse(rawJson) + + end Data + + private def parse(json: String): Json = + io.circe.parser.parse(json) match + case Left(failure) => throw failure + case Right(value) => value + +end CompiledQueryEvalTests diff --git a/project/build.properties b/project/build.properties index 5e6884d..01a16ed 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.11.6 +sbt.version=1.11.7