diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3408040e..1300aafd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,9 +48,9 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: - files: sql/target/scala-2.13/coverage-report/cobertura.xml,client/testkit/target/scala-2.13/coverage-report/cobertura.xml,persistence/target/scala-2.13/coverage-report/cobertura.xml,java/target/scala-2.13/coverage-report/cobertura.xml,java/persistence/target/scala-2.13/coverage-report/cobertura.xml,teskit/target/scala-2.13/coverage-report/cobertura.xml + files: sql/target/scala-2.13/coverage-report/cobertura.xml,core/target/scala-2.13/coverage-report/cobertura.xml,persistence/target/scala-2.13/coverage-report/cobertura.xml,es6/sql-bridge/target/scala-2.13/coverage-report/cobertura.xml,es6/jest/target/scala-2.13/coverage-report/cobertura.xml,es6/jest/persistence/target/scala-2.13/coverage-report/cobertura.xml,es6/rest/target/scala-2.13/coverage-report/cobertura.xml,es6/rest/persistence/target/scala-2.13/coverage-report/cobertura.xml,es6/teskit/target/scala-2.13/coverage-report/cobertura.xml,sql/bridge/target-es7/scala-2.13/coverage-report/cobertura.xml,es7/rest/target/scala-2.13/coverage-report/cobertura.xml,es7/rest/persistence/target/scala-2.13/coverage-report/cobertura.xml,es7/teskit/target/scala-2.13/coverage-report/cobertura.xml,sql/bridge/target-es8/scala-2.13/coverage-report/cobertura.xml,es8/java/target/scala-2.13/coverage-report/cobertura.xml,es8/java/persistence/target/scala-2.13/coverage-report/cobertura.xml,es8/teskit/target/scala-2.13/coverage-report/cobertura.xml,sql/bridge/target-es9/scala-2.13/coverage-report/cobertura.xml,es9/java/target/scala-2.13/coverage-report/cobertura.xml,es9/java/persistence/target/scala-2.13/coverage-report/cobertura.xml,es9/teskit/target/scala-2.13/coverage-report/cobertura.xml flags: unittests fail_ci_if_error: false verbose: true - name: Publish - run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g -Dfile.encoding=UTF-8" sbt publish + run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g -Dfile.encoding=UTF-8" sbt '+ publish' diff --git a/.gitignore b/.gitignore index bbd2f713..4b3e0970 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ target .metals *.log .bsp -.bloop \ No newline at end of file +.bloop +actions-runner +target-es* \ No newline at end of file diff --git a/build.sbt b/build.sbt index bf070da4..89bb80a5 100644 --- a/build.sbt +++ b/build.sbt @@ -1,11 +1,12 @@ +import SoftClient4es.* import app.softnetwork.* ///////////////////////////////// // Defaults ///////////////////////////////// +lazy val scala212 = "2.12.20" lazy val scala213 = "2.13.16" -lazy val javacCompilerVersion = "17" lazy val scalacCompilerOptions = Seq( "-deprecation", "-feature", @@ -14,9 +15,9 @@ lazy val scalacCompilerOptions = Seq( ThisBuild / organization := "app.softnetwork" -name := "elastic" +name := "softclient4es" -ThisBuild / version := Versions.elasticSearch +ThisBuild / version := "0.1-SNAPSHOT" ThisBuild / scalaVersion := scala213 @@ -28,7 +29,7 @@ ThisBuild / dependencyOverrides ++= Seq( ) lazy val moduleSettings = Seq( - crossScalaVersions := Seq(scala213), + crossScalaVersions := Seq(scala212, scala213), scalacOptions ++= { CrossVersion.partialVersion(scalaVersion.value) match { case Some((2, 12)) => scalacCompilerOptions :+ "-Ypartial-unification" @@ -38,7 +39,7 @@ lazy val moduleSettings = Seq( } ) -ThisBuild / javacOptions ++= Seq("-source", javacCompilerVersion, "-target", javacCompilerVersion) +ThisBuild / javacOptions ++= Seq("-source", "1.8", "-target", "1.8") ThisBuild / resolvers ++= Seq( "Softnetwork Server" at "https://softnetwork.jfrog.io/artifactory/releases/", @@ -55,14 +56,6 @@ val logging = Seq( "org.slf4j" % "jul-to-slf4j" % Versions.slf4j ) -val jacksonExclusions = Seq( - ExclusionRule(organization = "com.fasterxml.jackson.core"), - ExclusionRule(organization = "com.fasterxml.jackson.dataformat"), - ExclusionRule(organization = "com.fasterxml.jackson.datatype"), - ExclusionRule(organization = "com.fasterxml.jackson.module"), - ExclusionRule(organization = "org.codehaus.jackson") -) - val json4s = Seq( "org.json4s" %% "json4s-jackson" % Versions.json4s, "org.json4s" %% "json4s-ext" % Versions.json4s @@ -78,9 +71,12 @@ Test / parallelExecution := false lazy val sql = project.in(file("sql")) .configs(IntegrationTest) - .settings(Defaults.itSettings) + .settings( + Defaults.itSettings, + moduleSettings, + ) -lazy val client = project.in(file("client")) +lazy val core = project.in(file("core")) .configs(IntegrationTest) .settings( Defaults.itSettings, @@ -97,39 +93,324 @@ lazy val persistence = project.in(file("persistence")) moduleSettings ) .dependsOn( - client % "compile->compile;test->test;it->it" + core % "compile->compile;test->test;it->it" ) -lazy val java = project.in(file("java")) +lazy val es6bridge = project.in(file("es6/sql-bridge")) .configs(IntegrationTest) .settings( Defaults.itSettings, - moduleSettings + moduleSettings, + elasticSearchVersion := Versions.es6, ) .dependsOn( - client % "compile->compile;test->test;it->it" + sql % "compile->compile;test->test;it->it" ) -lazy val javaPersistence = project.in(file("java/persistence")) +lazy val es6rest = project.in(file("es6/rest")) .configs(IntegrationTest) - .settings(Defaults.itSettings) + .settings( + Defaults.itSettings, + moduleSettings, + elasticSearchVersion := Versions.es6, + ) .dependsOn( - java % "compile->compile;test->test;it->it", + core % "compile->compile;test->test;it->it" + ) + .dependsOn( + es6bridge % "compile->compile;test->test;it->it" + ) + +lazy val es6restp = project.in(file("es6/rest/persistence")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + elasticSearchVersion := Versions.es6, ) .dependsOn( persistence % "compile->compile;test->test;it->it" ) + .dependsOn( + es6rest % "compile->compile;test->test;it->it" + ) -lazy val testKit = project.in(file("testkit")) +lazy val es6jest = project.in(file("es6/jest")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + elasticSearchVersion := Versions.es6, + ) + .dependsOn( + core % "compile->compile;test->test;it->it" + ) + .dependsOn( + es6bridge % "compile->compile;test->test;it->it" + ) + +lazy val es6jestp = project.in(file("es6/jest/persistence")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + elasticSearchVersion := Versions.es6, + ) + .dependsOn( + persistence % "compile->compile;test->test;it->it" + ) + .dependsOn( + es6jest % "compile->compile;test->test;it->it" + ) + +lazy val es6testkit = project.in(file("es6/testkit")) .configs(IntegrationTest) .settings( Defaults.itSettings, app.softnetwork.Info.infoSettings, - moduleSettings + moduleSettings, + elasticSearchVersion := Versions.es6, + buildInfoKeys += BuildInfoKey("elasticVersion" -> elasticSearchVersion.value) + ) + .enablePlugins(BuildInfoPlugin) + .dependsOn( + es6restp % "compile->compile;test->test;it->it" + ) + .dependsOn( + es6jestp % "compile->compile;test->test;it->it" + ) + +lazy val es6 = project.in(file("es6")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + Publish.noPublishSettings, + crossScalaVersions := Nil, + elasticSearchVersion := Versions.es6 + ) + .aggregate( + es6bridge, + es6rest, + es6restp, + es6jest, + es6jestp, + es6testkit + ) + +lazy val es7bridge = project.in(file("sql/bridge")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + elasticSearchVersion := Versions.es7, + ) + .dependsOn( + sql % "compile->compile;test->test;it->it" + ) + +lazy val es7rest = project.in(file("es7/rest")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + elasticSearchVersion := Versions.es7, + ) + .dependsOn( + core % "compile->compile;test->test;it->it" + ) + .dependsOn( + es7bridge % "compile->compile;test->test;it->it" + ) + +lazy val es7restp = project.in(file("es7/rest/persistence")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + elasticSearchVersion := Versions.es7, + ) + .dependsOn( + persistence % "compile->compile;test->test;it->it" + ) + .dependsOn( + es7rest % "compile->compile;test->test;it->it" + ) + +lazy val es7testkit = project.in(file("es7/testkit")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + app.softnetwork.Info.infoSettings, + moduleSettings, + elasticSearchVersion := Versions.es7, + buildInfoKeys += BuildInfoKey("elasticVersion" -> elasticSearchVersion.value) + ) + .enablePlugins(BuildInfoPlugin) + .dependsOn( + es7restp % "compile->compile;test->test;it->it" + ) + +lazy val es7 = project.in(file("es7")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + Publish.noPublishSettings, + crossScalaVersions := Nil, + elasticSearchVersion := Versions.es7 + ) + .aggregate( + es7bridge, + es7rest, + es7restp, + es7testkit + ) + +lazy val es8bridge = project.in(file("sql/bridge")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + elasticSearchVersion := Versions.es8, + ) + .dependsOn( + sql % "compile->compile;test->test;it->it" + ) + +lazy val es8java = project.in(file("es8/java")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + elasticSearchVersion := Versions.es8, + ) + .dependsOn( + core % "compile->compile;test->test;it->it" + ) + .dependsOn( + es8bridge % "compile->compile;test->test;it->it" + ) + +lazy val es8javap = project.in(file("es8/java/persistence")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + elasticSearchVersion := Versions.es8, + ) + .dependsOn( + persistence % "compile->compile;test->test;it->it" + ) + .dependsOn( + es8java % "compile->compile;test->test;it->it" + ) + +lazy val es8testkit = project.in(file("es8/testkit")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + app.softnetwork.Info.infoSettings, + moduleSettings, + elasticSearchVersion := Versions.es8, + buildInfoKeys += BuildInfoKey("elasticVersion" -> elasticSearchVersion.value) ) .enablePlugins(BuildInfoPlugin) .dependsOn( - javaPersistence % "compile->compile;test->test;it->it" + es8javap % "compile->compile;test->test;it->it" + ) + +lazy val es8 = project.in(file("es8")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + Publish.noPublishSettings, + crossScalaVersions := Nil, + elasticSearchVersion := Versions.es8 + ) + .aggregate( + es8bridge, + es8java, + es8javap, + es8testkit + ) + +lazy val es9bridge = project.in(file("sql/bridge")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + scalaVersion := scala213, + crossScalaVersions := Seq(scala213), + elasticSearchVersion := Versions.es9, + javacOptions ++= Seq("-source", "17", "-target", "17") + ) + .dependsOn( + sql % "compile->compile;test->test;it->it" + ) + +lazy val es9java = project.in(file("es9/java")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + scalaVersion := scala213, + crossScalaVersions := Seq(scala213), + elasticSearchVersion := Versions.es9, + javacOptions ++= Seq("-source", "17", "-target", "17") + ) + .dependsOn( + core % "compile->compile;test->test;it->it" + ) + .dependsOn( + es9bridge % "compile->compile;test->test;it->it" + ) + +lazy val es9javap = project.in(file("es9/java/persistence")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + moduleSettings, + scalaVersion := scala213, + crossScalaVersions := Seq(scala213), + elasticSearchVersion := Versions.es9, + javacOptions ++= Seq("-source", "17", "-target", "17") + ) + .dependsOn( + persistence % "compile->compile;test->test;it->it" + ) + .dependsOn( + es9java % "compile->compile;test->test;it->it" + ) + +lazy val es9testkit = project.in(file("es9/testkit")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + app.softnetwork.Info.infoSettings, + moduleSettings, + scalaVersion := scala213, + crossScalaVersions := Seq(scala213), + elasticSearchVersion := Versions.es9, + javacOptions ++= Seq("-source", "17", "-target", "17"), + buildInfoKeys += BuildInfoKey("elasticVersion" -> elasticSearchVersion.value) + ) + .enablePlugins(BuildInfoPlugin) + .dependsOn( + es9javap % "compile->compile;test->test;it->it" + ) + +lazy val es9 = project.in(file("es9")) + .configs(IntegrationTest) + .settings( + Defaults.itSettings, + Publish.noPublishSettings, + crossScalaVersions := Nil, + elasticSearchVersion := Versions.es9 + ) + .aggregate( + es9bridge, + es9java, + es9javap, + es9testkit ) lazy val root = project.in(file(".")) @@ -139,4 +420,10 @@ lazy val root = project.in(file(".")) Publish.noPublishSettings, crossScalaVersions := Nil ) - .aggregate(sql, client, persistence, java, javaPersistence, testKit) + .aggregate( + sql, + core, + persistence, + es8, + es9 + ) diff --git a/client/build.sbt b/client/build.sbt deleted file mode 100644 index e0c9c010..00000000 --- a/client/build.sbt +++ /dev/null @@ -1,25 +0,0 @@ -organization := "app.softnetwork.elastic" - -name := "elastic-client" - -val configDependencies = Seq( - "com.typesafe" % "config" % Versions.typesafeConfig, - "com.github.kxbmap" %% "configs" % Versions.kxbmap -) - -val jacksonExclusions = Seq( - ExclusionRule(organization = "com.fasterxml.jackson.core"), - ExclusionRule(organization = "com.fasterxml.jackson.dataformat"), - ExclusionRule(organization = "com.fasterxml.jackson.datatype"), - ExclusionRule(organization = "com.fasterxml.jackson.module"), - ExclusionRule(organization = "org.codehaus.jackson") -) - -val json4s = Seq( - "org.json4s" %% "json4s-jackson" % Versions.json4s, - "org.json4s" %% "json4s-ext" % Versions.json4s -).map(_.excludeAll(jacksonExclusions: _*)) - -libraryDependencies ++= configDependencies ++ json4s :+ - ("app.softnetwork.persistence" %% "persistence-core" % Versions.genericPersistence excludeAll(jacksonExclusions: _*)) :+ - "com.google.code.gson" % "gson" % Versions.gson diff --git a/core/build.sbt b/core/build.sbt new file mode 100644 index 00000000..09867b89 --- /dev/null +++ b/core/build.sbt @@ -0,0 +1,19 @@ +import SoftClient4es._ + +organization := "app.softnetwork.elastic" + +name := "softclient4es-core" + +val configDependencies = Seq( + "com.typesafe" % "config" % Versions.typesafeConfig, + "com.github.kxbmap" %% "configs" % Versions.kxbmap +) + +val json4s = Seq( + "org.json4s" %% "json4s-jackson" % Versions.json4s, + "org.json4s" %% "json4s-ext" % Versions.json4s +).map(_.excludeAll(jacksonExclusions: _*)) + +libraryDependencies ++= configDependencies ++ + json4s :+ "com.google.code.gson" % "gson" % Versions.gson :+ + ("app.softnetwork.persistence" %% "persistence-core" % Versions.genericPersistence excludeAll(jacksonExclusions: _*)) diff --git a/client/src/main/resources/softnetwork-elastic.conf b/core/src/main/resources/softnetwork-elastic.conf similarity index 100% rename from client/src/main/resources/softnetwork-elastic.conf rename to core/src/main/resources/softnetwork-elastic.conf diff --git a/client/src/main/scala/app/softnetwork/elastic/client/AggregateResult.scala b/core/src/main/scala/app/softnetwork/elastic/client/AggregateResult.scala similarity index 100% rename from client/src/main/scala/app/softnetwork/elastic/client/AggregateResult.scala rename to core/src/main/scala/app/softnetwork/elastic/client/AggregateResult.scala diff --git a/client/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala similarity index 91% rename from client/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala rename to core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index b46c76d5..f6889bd0 100644 --- a/client/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -8,12 +8,12 @@ import _root_.akka.stream.{FlowShape, Materializer} import akka.stream.scaladsl._ import app.softnetwork.persistence.model.Timestamped import app.softnetwork.serialization._ -import app.softnetwork.elastic.sql.SQLQuery +import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} import com.google.gson.JsonParser import com.typesafe.config.{Config, ConfigFactory} -import com.typesafe.scalalogging.StrictLogging import org.json4s.{DefaultFormats, Formats} import org.json4s.jackson.JsonMethods._ +import org.slf4j.Logger import java.util.UUID import scala.concurrent.{Await, ExecutionContext, Future} @@ -221,11 +221,9 @@ trait SettingsApi { _: IndicesApi => def loadSettings(index: String): String } -trait MappingApi extends IndicesApi with RefreshApi with StrictLogging { - @deprecated("Use setMapping(index: String, mapping: String) instead", "7.17.29") - def setMapping(index: String, indexType: String, mapping: String): Boolean = { - this.setMapping(index, mapping) - } +trait MappingApi extends IndicesApi with RefreshApi { + + protected def logger: Logger /** Set the mapping of an index. * @param index @@ -236,10 +234,6 @@ trait MappingApi extends IndicesApi with RefreshApi with StrictLogging { * true if the mapping was set successfully, false otherwise */ def setMapping(index: String, mapping: String): Boolean - @deprecated("Use getMapping(index: String) instead", "7.17.29") - def getMapping(index: String, indexType: String): String = { - this.getMapping(index) - } /** Get the mapping of an index. * @param index @@ -489,11 +483,6 @@ trait IndexApi { _: RefreshApi => ) } - @deprecated("Use index(index: String, id: String, source: String) instead", "7.17.29") - def index(index: String, indexType: String, id: String, source: String): Boolean = { - this.index(index, id, source) - } - /** Index an entity in the given index. * @param index * - the name of the index to index the entity in @@ -525,13 +514,6 @@ trait IndexApi { _: RefreshApi => indexAsync(index.getOrElse(indexType), entity.uuid, serialization.write[U](entity)) } - @deprecated("Use indexAsync(index: String, id: String, source: String) instead", "7.17.29") - def indexAsync(index: String, indexType: String, id: String, source: String)(implicit - ec: ExecutionContext - ): Future[Boolean] = { - this.indexAsync(index, id, source) - } - /** Index an entity in the given index asynchronously. * @param index * - the name of the index to index the entity in @@ -580,20 +562,6 @@ trait UpdateApi { _: RefreshApi => ) } - @deprecated( - "Use update(index: String, id: String, source: String, upsert: Boolean) instead", - "7.17.29" - ) - def update( - index: String, - indexType: String, - id: String, - source: String, - upsert: Boolean - ): Boolean = { - this.update(index, id, source, upsert) - } - /** Update an entity in the given index. * @param index * - the name of the index to update the entity in @@ -636,14 +604,6 @@ trait UpdateApi { _: RefreshApi => ) } - @deprecated( - "Use updateAsync(index: String, id: String, source: String, upsert: Boolean) instead", - "7.17.29" - ) - def updateAsync(index: String, indexType: String, id: String, source: String, upsert: Boolean)( - implicit ec: ExecutionContext - ): Future[Boolean] = this.updateAsync(index, id, source, upsert) - /** Update an entity in the given index asynchronously. * @param index * - the name of the index to update the entity in @@ -686,11 +646,6 @@ trait DeleteApi { _: RefreshApi => delete(entity.uuid, index.getOrElse(indexType)) } - @deprecated("Use delete(uuid: String, index: String) instead", "7.17.29") - def delete(uuid: String, index: String, indexType: String): Boolean = { - this.delete(uuid, index) - } - /** Delete an entity from the given index. * @param uuid * - the id of the entity to delete @@ -720,13 +675,6 @@ trait DeleteApi { _: RefreshApi => deleteAsync(entity.uuid, index.getOrElse(indexType)) } - @deprecated("Use deleteAsync(uuid: String, index: String) instead", "7.17.29") - def deleteAsync(uuid: String, index: String, indexType: String)(implicit - ec: ExecutionContext - ): Future[Boolean] = { - this.deleteAsync(uuid, index) - } - /** Delete an entity from the given index asynchronously. * @param uuid * - the id of the entity to delete @@ -1042,6 +990,38 @@ trait GetApi { } trait SearchApi { + implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String + + implicit def sqlQueryToJSONQuery(sqlQuery: SQLQuery): JSONQuery = { + sqlQuery.request match { + case Some(Left(value)) => + JSONQuery(value.copy(score = sqlQuery.score), collection.immutable.Seq(value.sources: _*)) + case _ => + throw new IllegalArgumentException( + s"SQL query ${sqlQuery.query} does not contain a valid search request" + ) + } + } + + implicit def sqlQueryToJSONQueries(sqlQuery: SQLQuery): JSONQueries = { + sqlQuery.request match { + case Some(Right(value)) => + JSONQueries( + value.requests + .map(request => + JSONQuery( + request.copy(score = sqlQuery.score), + collection.immutable.Seq(request.sources: _*) + ) + ) + .toList + ) + case _ => + throw new IllegalArgumentException( + s"SQL query ${sqlQuery.query} does not contain a valid search request" + ) + } + } /** Search for entities matching the given JSON query. * @param jsonQuery @@ -1066,15 +1046,7 @@ trait SearchApi { * a list of entities matching the query */ def search[U](sqlQuery: SQLQuery)(implicit m: Manifest[U], formats: Formats): List[U] = { - sqlQuery.search match { - case Some(searchRequest) => - val indices = collection.immutable.Seq(searchRequest.sources: _*) - search[U](JSONQuery(searchRequest.query, indices)) - case None => - throw new IllegalArgumentException( - s"SQL query ${sqlQuery.query} does not contain a valid search request" - ) - } + search[U](implicitly[JSONQuery](sqlQuery))(m, formats) } /** Search for entities matching the given SQL query asynchronously. @@ -1112,16 +1084,7 @@ trait SearchApi { m2: Manifest[I], formats: Formats ): List[(U, List[I])] = { - sqlQuery.search match { - case Some(searchRequest) => - val indices = collection.immutable.Seq(searchRequest.sources: _*) - val jsonQuery = JSONQuery(searchRequest.query, indices) - searchWithInnerHits(jsonQuery, innerField)(m1, m2, formats) - case None => - throw new IllegalArgumentException( - s"SQL query ${sqlQuery.query} does not contain a valid search request" - ) - } + searchWithInnerHits[U, I](implicitly[JSONQuery](sqlQuery), innerField)(m1, m2, formats) } /** Search for entities matching the given JSON query with inner hits. @@ -1157,21 +1120,7 @@ trait SearchApi { def multiSearch[U]( sqlQuery: SQLQuery )(implicit m: Manifest[U], formats: Formats): List[List[U]] = { - sqlQuery.multiSearch match { - case Some(multiSearchRequest) => - val jsonQueries: JSONQueries = JSONQueries( - collection.immutable - .Seq(multiSearchRequest.requests.map { searchRequest => - JSONQuery(searchRequest.query, collection.immutable.Seq(searchRequest.sources: _*)) - }: _*) - .toList - ) - multiSearch[U](jsonQueries) - case None => - throw new IllegalArgumentException( - s"SQL query ${sqlQuery.query} does not contain a valid search request" - ) - } + multiSearch[U](implicitly[JSONQueries](sqlQuery))(m, formats) } /** Perform a multi-search operation with the given JSON queries. @@ -1207,21 +1156,7 @@ trait SearchApi { m2: Manifest[I], formats: Formats ): List[List[(U, List[I])]] = { - sqlQuery.multiSearch match { - case Some(multiSearchRequest) => - val jsonQueries: JSONQueries = JSONQueries( - collection.immutable - .Seq(multiSearchRequest.requests.map { searchRequest => - JSONQuery(searchRequest.query, collection.immutable.Seq(searchRequest.sources: _*)) - }: _*) - .toList - ) - multiSearchWithInnerHits[U, I](jsonQueries, innerField) - case None => - throw new IllegalArgumentException( - s"SQL query ${sqlQuery.query} does not contain a valid search request" - ) - } + multiSearchWithInnerHits[U, I](implicitly[JSONQueries](sqlQuery), innerField)(m1, m2, formats) } /** Perform a multi-search operation with the given JSON queries and inner hits. diff --git a/client/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala b/core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala similarity index 100% rename from client/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala rename to core/src/main/scala/app/softnetwork/elastic/client/MappingComparator.scala diff --git a/client/src/main/scala/app/softnetwork/elastic/client/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/package.scala similarity index 98% rename from client/src/main/scala/app/softnetwork/elastic/client/package.scala rename to core/src/main/scala/app/softnetwork/elastic/client/package.scala index 30753177..b1798d1f 100644 --- a/client/src/main/scala/app/softnetwork/elastic/client/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/package.scala @@ -6,9 +6,10 @@ import app.softnetwork.elastic.client.BulkAction.BulkAction import app.softnetwork.serialization._ import com.google.gson.{Gson, JsonElement, JsonObject} import com.typesafe.config.{Config, ConfigFactory} -import com.typesafe.scalalogging.{Logger, StrictLogging} +import com.typesafe.scalalogging.StrictLogging import configs.ConfigReader import org.json4s.Formats +import org.slf4j.Logger import scala.collection.immutable.Seq import scala.collection.mutable diff --git a/core/testkit/build.sbt b/core/testkit/build.sbt new file mode 100644 index 00000000..c6d03ddc --- /dev/null +++ b/core/testkit/build.sbt @@ -0,0 +1,7 @@ +import SoftClient4es._ + +organization := "app.softnetwork.elastic" + +name := "softclient4es-core-testkit" + +libraryDependencies += "org.testcontainers" % "elasticsearch" % Versions.testContainers excludeAll (jacksonExclusions: _*) diff --git a/es6/build.sbt b/es6/build.sbt new file mode 100644 index 00000000..28252611 --- /dev/null +++ b/es6/build.sbt @@ -0,0 +1,7 @@ +import SoftClient4es.* +organization := "app.softnetwork.elastic" +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}" +publish / skip := true +Compile / sources := Nil +Test / sources := Nil + diff --git a/es6/jest/build.sbt b/es6/jest/build.sbt new file mode 100644 index 00000000..8fca8f44 --- /dev/null +++ b/es6/jest/build.sbt @@ -0,0 +1,8 @@ +import SoftClient4es.* + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-jest-client" + +libraryDependencies ++= jestClientDependencies(elasticSearchVersion.value) ++ + elastic4sDependencies(elasticSearchVersion.value) diff --git a/es6/jest/persistence/build.sbt b/es6/jest/persistence/build.sbt new file mode 100644 index 00000000..87b0d37a --- /dev/null +++ b/es6/jest/persistence/build.sbt @@ -0,0 +1,6 @@ +import SoftClient4es.{elasticSearchMajorVersion, elasticSearchVersion} + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-jest-persistence" + diff --git a/es6/jest/persistence/src/main/scala/app/softnetwork/persistence/query/JestProvider.scala b/es6/jest/persistence/src/main/scala/app/softnetwork/persistence/query/JestProvider.scala new file mode 100644 index 00000000..926b13c5 --- /dev/null +++ b/es6/jest/persistence/src/main/scala/app/softnetwork/persistence/query/JestProvider.scala @@ -0,0 +1,12 @@ +package app.softnetwork.persistence.query + +import app.softnetwork.elastic.client.jest.JestClientApi +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.model.Timestamped + +/** Created by smanciot on 20/05/2021. + */ +trait JestProvider[T <: Timestamped] extends ElasticProvider[T] with JestClientApi { + _: ManifestWrapper[T] => +} diff --git a/es6/jest/persistence/src/main/scala/app/softnetwork/persistence/query/State2ElasticProcessorStreamWithJestProvider.scala b/es6/jest/persistence/src/main/scala/app/softnetwork/persistence/query/State2ElasticProcessorStreamWithJestProvider.scala new file mode 100644 index 00000000..f7c862a5 --- /dev/null +++ b/es6/jest/persistence/src/main/scala/app/softnetwork/persistence/query/State2ElasticProcessorStreamWithJestProvider.scala @@ -0,0 +1,9 @@ +package app.softnetwork.persistence.query + +import app.softnetwork.elastic.persistence.query.State2ElasticProcessorStream +import app.softnetwork.persistence.message.CrudEvent +import app.softnetwork.persistence.model.Timestamped + +trait State2ElasticProcessorStreamWithJestProvider[T <: Timestamped, E <: CrudEvent] + extends State2ElasticProcessorStream[T, E] + with JestProvider[T] { _: JournalProvider with OffsetProvider => } diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala new file mode 100644 index 00000000..82c9590c --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientApi.scala @@ -0,0 +1,832 @@ +package app.softnetwork.elastic.client.jest + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Flow +import app.softnetwork.elastic.client._ +import app.softnetwork.elastic.sql +import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.persistence.model.Timestamped +import app.softnetwork.serialization._ +import com.google.gson.{Gson, JsonParser} +import io.searchbox.action.BulkableAction +import io.searchbox.core._ +import io.searchbox.core.search.aggregation.RootAggregation +import io.searchbox.indices._ +import io.searchbox.indices.aliases.{AddAliasMapping, ModifyAliases, RemoveAliasMapping} +import io.searchbox.indices.mapping.{GetMapping, PutMapping} +import io.searchbox.indices.reindex.Reindex +import io.searchbox.indices.settings.{GetSettings, UpdateSettings} +import io.searchbox.params.Parameters +import org.json4s.Formats + +import scala.collection.JavaConverters._ +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.language.implicitConversions +import scala.util.{Failure, Success, Try} + +/** Created by smanciot on 20/05/2021. + */ +trait JestClientApi + extends ElasticClientApi + with JestIndicesApi + with JestAliasApi + with JestSettingsApi + with JestMappingApi + with JestRefreshApi + with JestFlushApi + with JestCountApi + with JestSingleValueAggregateApi + with JestIndexApi + with JestUpdateApi + with JestDeleteApi + with JestGetApi + with JestSearchApi + with JestBulkApi + +trait JestIndicesApi extends IndicesApi with JestRefreshApi with JestClientCompanion { + override def createIndex(index: String, settings: String = defaultSettings): Boolean = + tryOrElse( + apply() + .execute( + new CreateIndex.Builder(index).settings(settings).build() + ) + .isSucceeded, + false + )(logger) + + override def deleteIndex(index: String): Boolean = + tryOrElse( + apply() + .execute( + new DeleteIndex.Builder(index).build() + ) + .isSucceeded, + false + )(logger) + + override def closeIndex(index: String): Boolean = + tryOrElse( + apply() + .execute( + new CloseIndex.Builder(index).build() + ) + .isSucceeded, + false + )(logger) + + override def openIndex(index: String): Boolean = + tryOrElse( + apply() + .execute( + new OpenIndex.Builder(index).build() + ) + .isSucceeded, + false + )(logger) + + /** Reindex from source index to target index. + * + * @param sourceIndex + * - the name of the source index + * @param targetIndex + * - the name of the target index + * @param refresh + * - true to refresh the target index after reindexing, false otherwise + * @return + * true if the reindexing was successful, false otherwise + */ + override def reindex(sourceIndex: String, targetIndex: String, refresh: Boolean): Boolean = { + tryOrElse( + { + apply() + .execute( + new Reindex.Builder(s"""{"index": "$sourceIndex"}""", s"""{"index": "$targetIndex"}""") + .build() + ) + .isSucceeded && { + if (refresh) { + this.refresh(targetIndex) + } else { + true + } + } + }, + false + )(logger) + } + + /** Check if an index exists. + * + * @param index + * - the name of the index to check + * @return + * true if the index exists, false otherwise + */ + override def indexExists(index: String): Boolean = + tryOrElse( + apply() + .execute( + new IndicesExists.Builder(index).build() + ) + .isSucceeded, + false + )(logger) +} + +trait JestAliasApi extends AliasApi with JestClientCompanion { + override def addAlias(index: String, alias: String): Boolean = { + tryOrElse( + apply() + .execute( + new ModifyAliases.Builder( + new AddAliasMapping.Builder(index, alias).build() + ).build() + ) + .isSucceeded, + false + )(logger) + } + + override def removeAlias(index: String, alias: String): Boolean = { + tryOrElse( + apply() + .execute( + new ModifyAliases.Builder( + new RemoveAliasMapping.Builder(index, alias).build() + ).build() + ) + .isSucceeded, + false + )(logger) + } +} + +trait JestSettingsApi extends SettingsApi with JestClientCompanion { + _: IndicesApi => + override def updateSettings(index: String, settings: String = defaultSettings): Boolean = + closeIndex(index) && + tryOrElse( + apply() + .execute( + new UpdateSettings.Builder(settings).addIndex(index).build() + ) + .isSucceeded, + false + )(logger) && + openIndex(index) + + override def loadSettings(index: String): String = + tryOrElse( + apply() + .execute( + new GetSettings.Builder().addIndex(index).build() + ) + .getJsonString, + s"""{"$index": {"settings": {"index": {}}}}""" + )(logger) +} + +trait JestMappingApi extends MappingApi with JestClientCompanion { + _: IndicesApi => + override def setMapping(index: String, mapping: String): Boolean = + tryOrElse( + apply() + .execute( + new PutMapping.Builder(index, "_doc", mapping).build() + ) + .isSucceeded, + false + )(logger) + + override def getMapping(index: String): String = + tryOrElse( + apply() + .execute( + new GetMapping.Builder().addIndex(index).addType("_doc").build() + ) + .getJsonString, + s""""{$index: {"mappings": {"_doc":{"properties": {}}}}}""" // empty mapping + )(logger) + + /** Get the mapping properties of an index. + * + * @param index + * - the name of the index to get the mapping properties for + * @return + * the mapping properties of the index as a JSON string + */ + override def getMappingProperties(index: String): String = { + tryOrElse( + new Gson().toJson( + new JsonParser() + .parse(getMapping(index)) + .getAsJsonObject + .get(index) + .getAsJsonObject + .get("mappings") + .getAsJsonObject + .get("_doc") + .getAsJsonObject + ), + "{\"properties\": {}}" + )(logger) + } +} + +trait JestRefreshApi extends RefreshApi with JestClientCompanion { + override def refresh(index: String): Boolean = + tryOrElse( + apply() + .execute( + new Refresh.Builder().addIndex(index).build() + ) + .isSucceeded, + false + )(logger) +} + +trait JestFlushApi extends FlushApi with JestClientCompanion { + override def flush(index: String, force: Boolean = true, wait: Boolean = true): Boolean = + tryOrElse( + apply() + .execute( + new Flush.Builder().addIndex(index).force(force).waitIfOngoing(wait).build() + ) + .isSucceeded, + false + )(logger) +} + +trait JestCountApi extends CountApi with JestClientCompanion { + override def countAsync( + jsonQuery: JSONQuery + )(implicit ec: ExecutionContext): Future[Option[Double]] = { + import JestClientResultHandler._ + import jsonQuery._ + val count = new Count.Builder().query(query) + for (indice <- indices) count.addIndex(indice) + for (t <- types) count.addType(t) + val promise = Promise[Option[Double]]() + apply().executeAsyncPromise(count.build()) onComplete { + case Success(result) => + if (!result.isSucceeded) + logger.error(result.getErrorMessage) + promise.success(Option(result.getCount)) + case Failure(f) => + logger.error(f.getMessage, f) + promise.failure(f) + } + promise.future + } + + override def count(jsonQuery: JSONQuery): Option[Double] = { + import jsonQuery._ + val count = new Count.Builder().query(query) + for (indice <- indices) count.addIndex(indice) + for (t <- types) count.addType(t) + Try { + apply().execute(count.build()) + } match { + case Success(result) => + if (!result.isSucceeded) + logger.error(result.getErrorMessage) + Option(result.getCount) + case Failure(f) => + logger.error(f.getMessage, f) + None + } + } +} + +trait JestSingleValueAggregateApi extends SingleValueAggregateApi with JestCountApi { + override def aggregate( + sqlQuery: SQLQuery + )(implicit ec: ExecutionContext): Future[Seq[SingleValueAggregateResult]] = { + val aggregations: Seq[ElasticAggregation] = sqlQuery + val futures = for (aggregation <- aggregations) yield { + val promise: Promise[SingleValueAggregateResult] = Promise() + val field = aggregation.field + val sourceField = aggregation.sourceField + val aggType = aggregation.aggType + val aggName = aggregation.aggName + val query = aggregation.query + val sources = aggregation.sources + sourceField match { + case "_id" if aggType.sql == "count" => + countAsync( + JSONQuery( + query, + collection.immutable.Seq(sources: _*), + collection.immutable.Seq.empty[String] + ) + ).onComplete { + case Success(result) => + promise.success( + SingleValueAggregateResult( + field, + aggType, + result.map(r => NumericValue(r.doubleValue())).getOrElse(EmptyValue), + None + ) + ) + case Failure(f) => + logger.error(f.getMessage, f.fillInStackTrace()) + promise.success( + SingleValueAggregateResult(field, aggType, EmptyValue, Some(f.getMessage)) + ) + } + promise.future + case _ => + import JestClientApi._ + import JestClientResultHandler._ + apply() + .executeAsyncPromise( + JSONQuery( + query, + collection.immutable.Seq(sources: _*), + collection.immutable.Seq.empty[String] + ).search + ) + .onComplete { + case Success(result) => + val agg = aggName.split("\\.").last + + val itAgg = aggName.split("\\.").iterator + + var root = + if (aggregation.nested) + result.getAggregations.getAggregation(itAgg.next(), classOf[RootAggregation]) + else + result.getAggregations + + if (aggregation.filtered) { + root = root.getAggregation(itAgg.next(), classOf[RootAggregation]) + } + + promise.success( + SingleValueAggregateResult( + field, + aggType, + aggType match { + case sql.Count => + if (aggregation.distinct) + NumericValue( + root.getCardinalityAggregation(agg).getCardinality.doubleValue() + ) + else { + NumericValue( + root.getValueCountAggregation(agg).getValueCount.doubleValue() + ) + } + case sql.Sum => + NumericValue(root.getSumAggregation(agg).getSum) + case sql.Avg => + NumericValue(root.getAvgAggregation(agg).getAvg) + case sql.Min => + NumericValue(root.getMinAggregation(agg).getMin) + case sql.Max => + NumericValue(root.getMaxAggregation(agg).getMax) + case _ => EmptyValue + }, + None + ) + ) + + case Failure(f) => + logger.error(f.getMessage, f.fillInStackTrace()) + promise.success( + SingleValueAggregateResult(field, aggType, EmptyValue, Some(f.getMessage)) + ) + } + + promise.future + } + } + Future.sequence(futures) + } +} + +trait JestIndexApi extends IndexApi with JestClientCompanion { + _: RefreshApi => + override def index(index: String, id: String, source: String): Boolean = { + Try( + apply().execute( + new Index.Builder(source).index(index).`type`("_doc").id(id).build() + ) + ) match { + case Success(s) => + if (!s.isSucceeded) + logger.error(s.getErrorMessage) + s.isSucceeded && this.refresh(index) + case Failure(f) => + logger.error(f.getMessage, f) + false + } + } + + override def indexAsync(index: String, id: String, source: String)(implicit + ec: ExecutionContext + ): Future[Boolean] = { + import JestClientResultHandler._ + val promise: Promise[Boolean] = Promise() + apply().executeAsyncPromise( + new Index.Builder(source).index(index).`type`("_doc").id(id).build() + ) onComplete { + case Success(s) => promise.success(s.isSucceeded && this.refresh(index)) + case Failure(f) => + logger.error(f.getMessage, f) + promise.failure(f) + } + promise.future + } + +} + +trait JestUpdateApi extends UpdateApi with JestClientCompanion { + _: RefreshApi => + override def update( + index: String, + id: String, + source: String, + upsert: Boolean + ): Boolean = { + Try( + apply().execute( + new Update.Builder( + if (upsert) + docAsUpsert(source) + else + source + ).index(index).`type`("_doc").id(id).build() + ) + ) match { + case Success(s) => + if (!s.isSucceeded) + logger.error(s.getErrorMessage) + s.isSucceeded && this.refresh(index) + case Failure(f) => + logger.error(f.getMessage, f) + false + } + } + + override def updateAsync( + index: String, + id: String, + source: String, + upsert: Boolean + )(implicit ec: ExecutionContext): Future[Boolean] = { + import JestClientResultHandler._ + val promise: Promise[Boolean] = Promise() + apply().executeAsyncPromise( + new Update.Builder( + if (upsert) + docAsUpsert(source) + else + source + ).index(index).`type`("_doc").id(id).build() + ) onComplete { + case Success(s) => + if (!s.isSucceeded) + logger.error(s.getErrorMessage) + promise.success(s.isSucceeded && this.refresh(index)) + case Failure(f) => + logger.error(f.getMessage, f) + promise.failure(f) + } + promise.future + } + +} + +trait JestDeleteApi extends DeleteApi with JestClientCompanion { + _: RefreshApi => + override def delete(uuid: String, index: String): Boolean = { + Try( + apply() + .execute( + new Delete.Builder(uuid).index(index).`type`("_doc").build() + ) + ) match { + case Success(result) => + if (!result.isSucceeded) + logger.error(result.getErrorMessage) + result.isSucceeded && this.refresh(index) + case Failure(f) => + logger.error(f.getMessage, f) + false + } + } + + override def deleteAsync(uuid: String, index: String)(implicit + ec: ExecutionContext + ): Future[Boolean] = { + import JestClientResultHandler._ + val promise: Promise[Boolean] = Promise() + apply().executeAsyncPromise( + new Delete.Builder(uuid).index(index).`type`("_doc").build() + ) onComplete { + case Success(s) => + if (!s.isSucceeded) + logger.error(s.getErrorMessage) + promise.success(s.isSucceeded && this.refresh(index)) + case Failure(f) => + logger.error(f.getMessage, f) + promise.failure(f) + } + promise.future + } + +} + +trait JestGetApi extends GetApi with JestClientCompanion { + + // GetApi + override def get[U <: Timestamped]( + id: String, + index: Option[String] = None, + maybeType: Option[String] = None + )(implicit m: Manifest[U], formats: Formats): Option[U] = { + val result = apply().execute( + new Get.Builder( + index.getOrElse( + maybeType.getOrElse( + m.runtimeClass.getSimpleName.toLowerCase + ) + ), + id + ).build() + ) + if (result.isSucceeded) { + Some(serialization.read[U](result.getSourceAsString)) + } else { + logger.error(result.getErrorMessage) + None + } + } + + override def getAsync[U <: Timestamped]( + id: String, + index: Option[String] = None, + maybeType: Option[String] = None + )(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats): Future[Option[U]] = { + import JestClientResultHandler._ + val promise: Promise[Option[U]] = Promise() + apply().executeAsyncPromise( + new Get.Builder( + index.getOrElse( + maybeType.getOrElse( + m.runtimeClass.getSimpleName.toLowerCase + ) + ), + id + ).build() + ) onComplete { + case Success(result) => + if (result.isSucceeded) + promise.success(Some(serialization.read[U](result.getSourceAsString))) + else { + logger.error(result.getErrorMessage) + promise.success(None) + } + case Failure(f) => + logger.error(f.getMessage, f) + promise.failure(f) + } + promise.future + } + +} + +trait JestSearchApi extends SearchApi with JestClientCompanion { + + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + implicitly[ElasticSearchRequest](sqlSearch).query + + import JestClientApi._ + + override def search[U]( + jsonQuery: JSONQuery + )(implicit m: Manifest[U], formats: Formats): List[U] = { + import jsonQuery._ + val search = new Search.Builder(query) + for (indice <- indices) search.addIndex(indice) + for (t <- types) search.addType(t) + Try( + apply() + .execute(search.build()) + .getSourceAsStringList + .asScala + .map(source => serialization.read[U](source)) + .toList + ) match { + case Success(s) => s + case Failure(f) => + logger.error(f.getMessage, f) + List.empty + } + } + + override def searchAsync[U]( + sqlQuery: SQLQuery + )(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats): Future[List[U]] = { + val promise = Promise[List[U]]() + val search: Option[Search] = sqlQuery.jestSearch + search match { + case Some(s) => + import JestClientResultHandler._ + apply().executeAsyncPromise(s) onComplete { + case Success(searchResult) => + promise.success( + searchResult.getSourceAsStringList.asScala + .map(source => serialization.read[U](source)) + .toList + ) + case Failure(f) => + promise.failure(f) + } + case _ => promise.success(List.empty) + } + promise.future + } + + override def searchWithInnerHits[U, I](jsonQuery: JSONQuery, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[(U, List[I])] = { + Try(apply().execute(jsonQuery.search)).toOption match { + case Some(result) => + if (!result.isSucceeded) { + logger.error(result.getErrorMessage) + return List.empty + } + Try(result.getJsonObject ~> [U, I] innerField) match { + case Success(s) => s + case Failure(f) => + logger.error(f.getMessage, f) + List.empty + } + case _ => List.empty + } + } + + override def multiSearch[U]( + jsonQueries: JSONQueries + )(implicit m: Manifest[U], formats: Formats): List[List[U]] = { + tryOrElse( + { + val multiSearchResult = + apply().execute(new MultiSearch.Builder(jsonQueries.queries.map(_.search).asJava).build()) + multiSearchResult.getResponses.asScala + .map(searchResponse => + searchResponse.searchResult.getSourceAsStringList.asScala + .map(source => serialization.read[U](source)) + .toList + ) + .toList + }, + List.empty + )(logger) + } + + override def multiSearchWithInnerHits[U, I](jsonQueries: JSONQueries, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[List[(U, List[I])]] = { + val multiSearch = new MultiSearch.Builder(jsonQueries.queries.map(_.search).asJava).build() + Try(apply().execute(multiSearch)).toOption match { + case Some(multiSearchResult) => + if (!multiSearchResult.isSucceeded) { + logger.error(multiSearchResult.getErrorMessage) + return List.empty + } + multiSearchResult.getResponses.asScala + .map(searchResponse => { + Try(searchResponse.searchResult.getJsonObject ~> [U, I] innerField) match { + case Success(s) => s + case Failure(f) => + logger.error(f.getMessage, f) + List.empty[(U, List[I])] + } + }) + .toList + case _ => List.empty + } + } + +} + +trait JestBulkApi + extends JestRefreshApi + with JestSettingsApi + with JestIndicesApi + with BulkApi + with JestClientCompanion { + override type A = BulkableAction[DocumentResult] + override type R = BulkResult + + override implicit def toBulkElasticAction(a: A): BulkElasticAction = + new BulkElasticAction { + override def index: String = a.getIndex + } + + private[this] def toBulkElasticResultItem(i: BulkResult#BulkResultItem): BulkElasticResultItem = + new BulkElasticResultItem { + override def index: String = i.index + } + + override implicit def toBulkElasticResult(r: R): BulkElasticResult = + new BulkElasticResult { + override def items: List[BulkElasticResultItem] = + r.getItems.asScala.toList.map(toBulkElasticResultItem) + } + + override def bulk(implicit + bulkOptions: BulkOptions, + system: ActorSystem + ): Flow[Seq[A], R, NotUsed] = { + import JestClientResultHandler._ + val parallelism = Math.max(1, bulkOptions.balance) + + Flow[Seq[BulkableAction[DocumentResult]]] + .named("bulk") + .mapAsyncUnordered[BulkResult](parallelism)(items => { + logger.info(s"Starting to write batch of ${items.size}...") + val init = + new Bulk.Builder().defaultIndex(bulkOptions.index).defaultType(bulkOptions.documentType) + val bulkQuery = items.foldLeft(init) { (current, query) => + current.addAction(query) + } + apply().executeAsyncPromise(bulkQuery.build()) + }) + } + + override def bulkResult: Flow[R, Set[String], NotUsed] = + Flow[BulkResult] + .named("result") + .map(result => { + val items = result.getItems + val indices = items.asScala.map(_.index).toSet + logger.info(s"Finished to write batch of ${items.size} within ${indices.mkString(",")}.") + indices + }) + + override def toBulkAction(bulkItem: BulkItem): A = { + val builder = bulkItem.action match { + case BulkAction.DELETE => new Delete.Builder(bulkItem.body) + case BulkAction.UPDATE => new Update.Builder(docAsUpsert(bulkItem.body)) + case _ => new Index.Builder(bulkItem.body) + } + bulkItem.id.foreach(builder.id) + builder.index(bulkItem.index) + bulkItem.parent.foreach(s => builder.setParameter(Parameters.PARENT, s)) + builder.build() + } + +} + +object JestClientApi { + + implicit def requestToSearch(elasticSelect: ElasticSearchRequest): Search = { + import elasticSelect._ + Console.println(query) + val search = new Search.Builder(query) + for (source <- sources) search.addIndex(source) + search.build() + } + + implicit class SearchSQLQuery(sqlQuery: SQLQuery) { + def jestSearch: Option[Search] = { + sqlQuery.request match { + case Some(Left(value)) => + val request: ElasticSearchRequest = value + Some(request) + case _ => None + } + } + } + + implicit class SearchJSONQuery(jsonQuery: JSONQuery) { + def search: Search = { + import jsonQuery._ + val _search = new Search.Builder(query) + for (indice <- indices) _search.addIndex(indice) + for (t <- types) _search.addType(t) + _search.build() + } + } + + implicit class SearchResults(searchResult: SearchResult) { + def apply[M: Manifest]()(implicit formats: Formats): List[M] = { + searchResult.getSourceAsStringList.asScala.map(source => serialization.read[M](source)).toList + } + } + + implicit class JestBulkAction(bulkableAction: BulkableAction[DocumentResult]) { + def index: String = bulkableAction.getIndex + } +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientCompanion.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientCompanion.scala new file mode 100644 index 00000000..37278153 --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientCompanion.scala @@ -0,0 +1,159 @@ +package app.softnetwork.elastic.client.jest + +import app.softnetwork.elastic.client.{ElasticConfig, ElasticCredentials} +import com.sksamuel.exts.Logging +import io.searchbox.action.Action +import io.searchbox.client.config.HttpClientConfig +import io.searchbox.client.{JestClient, JestClientFactory, JestResult, JestResultHandler} +import org.apache.http.HttpHost + +import java.io.IOException +import java.util +import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ +import scala.language.reflectiveCalls +import scala.util.{Failure, Success, Try} + +/** Created by smanciot on 20/05/2021. + */ +trait JestClientCompanion extends Logging { + + def elasticConfig: ElasticConfig + + private[this] var jestClient: Option[InnerJestClient] = None + + private[this] val factory = new JestClientFactory() + + private[this] var httpClientConfig: HttpClientConfig = _ + + private[this] class InnerJestClient(private var _jestClient: JestClient) extends JestClient { + private[this] var nbFailures: Int = 0 + + override def shutdownClient(): Unit = { + close() + } + + private def checkClient(): Unit = { + Option(_jestClient) match { + case None => + factory.setHttpClientConfig(httpClientConfig) + _jestClient = Try(factory.getObject) match { + case Success(s) => + s + case Failure(f) => + logger.error(f.getMessage, f) + throw f + } + case _ => + } + } + + override def executeAsync[J <: JestResult]( + clientRequest: Action[J], + jestResultHandler: JestResultHandler[_ >: J] + ): Unit = { + Try(checkClient()) + Option(_jestClient) match { + case Some(s) => s.executeAsync[J](clientRequest, jestResultHandler) + case _ => + close() + jestResultHandler.failed(new Exception("JestClient not initialized")) + } + } + + override def execute[J <: JestResult](clientRequest: Action[J]): J = { + Try(checkClient()) + Option(_jestClient) match { + case Some(j) => + Try(j.execute[J](clientRequest)) match { + case Success(s) => + nbFailures = 0 + s + case Failure(f) => + f match { + case e: IOException => + nbFailures += 1 + logger.error(e.getMessage, e) + close() + if (nbFailures < 10) { + Thread.sleep(1000 * nbFailures) + execute(clientRequest) + } else { + throw f + } + case e: IllegalStateException => + nbFailures += 1 + logger.error(e.getMessage, e) + close() + if (nbFailures < 10) { + Thread.sleep(1000 * nbFailures) + execute(clientRequest) + } else { + throw f + } + case _ => + close() + throw f + } + } + case _ => + close() + throw new Exception("JestClient not initialized") + } + } + + override def setServers(servers: util.Set[String]): Unit = { + Try(checkClient()) + Option(_jestClient).foreach(_.setServers(servers)) + } + + override def close(): Unit = { + Option(_jestClient).foreach(_.close()) + _jestClient = null + } + } + + private[this] def getHttpHosts(esUrl: String): Set[HttpHost] = { + esUrl + .split(",") + .map(u => { + val url = new java.net.URL(u) + new HttpHost(url.getHost, url.getPort, url.getProtocol) + }) + .toSet + } + + def apply(): JestClient = { + apply( + elasticConfig.credentials, + multithreaded = elasticConfig.multithreaded, + discoveryEnabled = elasticConfig.discoveryEnabled + ) + } + + def apply( + esCredentials: ElasticCredentials, + multithreaded: Boolean = true, + timeout: Int = 60000, + discoveryEnabled: Boolean = false, + discoveryFrequency: Long = 60L, + discoveryFrequencyTimeUnit: TimeUnit = TimeUnit.SECONDS + ): JestClient = { + jestClient match { + case Some(s) => s + case None => + httpClientConfig = new HttpClientConfig.Builder(esCredentials.url) + .defaultCredentials(esCredentials.username, esCredentials.password) + .preemptiveAuthTargetHosts(getHttpHosts(esCredentials.url).asJava) + .multiThreaded(multithreaded) + .discoveryEnabled(discoveryEnabled) + .discoveryFrequency(discoveryFrequency, discoveryFrequencyTimeUnit) + .connTimeout(timeout) + .readTimeout(timeout) + .build() + factory.setHttpClientConfig(httpClientConfig) + jestClient = Some(new InnerJestClient(factory.getObject)) + jestClient.get + } + } +} diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientResultHandler.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientResultHandler.scala new file mode 100644 index 00000000..fc05e2e2 --- /dev/null +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestClientResultHandler.scala @@ -0,0 +1,44 @@ +package app.softnetwork.elastic.client.jest + +import io.searchbox.action.Action +import io.searchbox.client.{JestClient, JestResult, JestResultHandler} +import io.searchbox.core.BulkResult + +import scala.concurrent.{Future, Promise} + +/** Created by smanciot on 28/04/17. + */ +private class JestClientResultHandler[T <: JestResult] extends JestResultHandler[T] { + + protected val promise: Promise[T] = Promise() + + override def completed(result: T): Unit = + if (!result.isSucceeded) + promise.failure(new Exception(s"${result.getErrorMessage} - ${result.getJsonString}")) + else { + result match { + case r: BulkResult if !r.getFailedItems.isEmpty => + promise.failure( + new Exception(s"We don't allow any failed item while indexing ${result.getJsonString}") + ) + case _ => promise.success(result) + + } + } + + override def failed(exception: Exception): Unit = promise.failure(exception) + + def future: Future[T] = promise.future + +} + +object JestClientResultHandler { + + implicit class PromiseJestClient(jestClient: JestClient) { + def executeAsyncPromise[T <: JestResult](clientRequest: Action[T]): Future[T] = { + val resultHandler = new JestClientResultHandler[T]() + jestClient.executeAsync(clientRequest, resultHandler) + resultHandler.future + } + } +} diff --git a/es6/rest/build.sbt b/es6/rest/build.sbt new file mode 100644 index 00000000..1c65c2ce --- /dev/null +++ b/es6/rest/build.sbt @@ -0,0 +1,8 @@ +import SoftClient4es.* + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-rest-client" + +libraryDependencies ++= restClientDependencies(elasticSearchVersion.value) ++ + elastic4sDependencies(elasticSearchVersion.value) diff --git a/es6/rest/persistence/build.sbt b/es6/rest/persistence/build.sbt new file mode 100644 index 00000000..e1dc23b4 --- /dev/null +++ b/es6/rest/persistence/build.sbt @@ -0,0 +1,6 @@ +import SoftClient4es.{elasticSearchMajorVersion, elasticSearchVersion} + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-rest-persistence" + diff --git a/es6/rest/persistence/src/main/scala/app/softnetwork/persistence/query/RestHighLevelClientProvider.scala b/es6/rest/persistence/src/main/scala/app/softnetwork/persistence/query/RestHighLevelClientProvider.scala new file mode 100644 index 00000000..879c5879 --- /dev/null +++ b/es6/rest/persistence/src/main/scala/app/softnetwork/persistence/query/RestHighLevelClientProvider.scala @@ -0,0 +1,13 @@ +package app.softnetwork.persistence.query + +import app.softnetwork.elastic.client.rest.RestHighLevelClientApi +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.model.Timestamped + +trait RestHighLevelClientProvider[T <: Timestamped] + extends ElasticProvider[T] + with RestHighLevelClientApi { + _: ManifestWrapper[T] => + +} diff --git a/es6/rest/persistence/src/main/scala/app/softnetwork/persistence/query/State2ElasticProcessorStreamWithRestProvider.scala b/es6/rest/persistence/src/main/scala/app/softnetwork/persistence/query/State2ElasticProcessorStreamWithRestProvider.scala new file mode 100644 index 00000000..c114c063 --- /dev/null +++ b/es6/rest/persistence/src/main/scala/app/softnetwork/persistence/query/State2ElasticProcessorStreamWithRestProvider.scala @@ -0,0 +1,9 @@ +package app.softnetwork.persistence.query + +import app.softnetwork.elastic.persistence.query.State2ElasticProcessorStream +import app.softnetwork.persistence.message.CrudEvent +import app.softnetwork.persistence.model.Timestamped + +trait State2ElasticProcessorStreamWithRestProvider[T <: Timestamped, E <: CrudEvent] + extends State2ElasticProcessorStream[T, E] + with RestHighLevelClientProvider[T] { _: JournalProvider with OffsetProvider => } diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala new file mode 100644 index 00000000..15ee3f28 --- /dev/null +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -0,0 +1,918 @@ +package app.softnetwork.elastic.client.rest + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Flow +import app.softnetwork.elastic.client._ +import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.{client, sql} +import app.softnetwork.persistence.model.Timestamped +import app.softnetwork.serialization.serialization +import com.google.gson.JsonParser +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions +import org.elasticsearch.action.admin.indices.close.CloseIndexRequest +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest +import org.elasticsearch.action.admin.indices.flush.FlushRequest +import org.elasticsearch.action.admin.indices.open.OpenIndexRequest +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest +import org.elasticsearch.action.bulk.{BulkItemResponse, BulkRequest, BulkResponse} +import org.elasticsearch.action.delete.{DeleteRequest, DeleteResponse} +import org.elasticsearch.action.get.{GetRequest, GetResponse} +import org.elasticsearch.action.index.{IndexRequest, IndexResponse} +import org.elasticsearch.action.search.{MultiSearchRequest, SearchRequest, SearchResponse} +import org.elasticsearch.action.update.{UpdateRequest, UpdateResponse} +import org.elasticsearch.action.{ActionListener, DocWriteRequest} +import org.elasticsearch.client.{Request, RequestOptions} +import org.elasticsearch.client.core.{CountRequest, CountResponse} +import org.elasticsearch.client.indices.{ + CreateIndexRequest, + GetIndexRequest, + GetMappingsRequest, + PutMappingRequest +} +import org.elasticsearch.common.io.stream.InputStreamStreamInput +import org.elasticsearch.common.xcontent.{DeprecationHandler, XContentType} +import org.elasticsearch.rest.RestStatus +import org.elasticsearch.search.aggregations.bucket.filter.Filter +import org.elasticsearch.search.aggregations.bucket.nested.Nested +import org.elasticsearch.search.aggregations.metrics.avg.Avg +import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality +import org.elasticsearch.search.aggregations.metrics.max.Max +import org.elasticsearch.search.aggregations.metrics.min.Min +import org.elasticsearch.search.aggregations.metrics.sum.Sum +import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount +import org.elasticsearch.search.builder.SearchSourceBuilder +import org.json4s.Formats + +import java.io.ByteArrayInputStream +import scala.collection.JavaConverters.mapAsScalaMapConverter +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.language.implicitConversions +import scala.util.{Failure, Success, Try} + +trait RestHighLevelClientApi + extends ElasticClientApi + with RestHighLevelClientIndicesApi + with RestHighLevelClientAliasApi + with RestHighLevelClientSettingsApi + with RestHighLevelClientMappingApi + with RestHighLevelClientRefreshApi + with RestHighLevelClientFlushApi + with RestHighLevelClientCountApi + with RestHighLevelClientSingleValueAggregateApi + with RestHighLevelClientIndexApi + with RestHighLevelClientUpdateApi + with RestHighLevelClientDeleteApi + with RestHighLevelClientGetApi + with RestHighLevelClientSearchApi + with RestHighLevelClientBulkApi + +trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientCompanion { + override def createIndex(index: String, settings: String): Boolean = { + tryOrElse( + apply() + .indices() + .create( + new CreateIndexRequest(index) + .settings(settings, XContentType.JSON), + RequestOptions.DEFAULT + ) + .isAcknowledged, + false + )(logger) + } + + override def deleteIndex(index: String): Boolean = { + tryOrElse( + apply() + .indices() + .delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT) + .isAcknowledged, + false + )(logger) + } + + override def openIndex(index: String): Boolean = { + tryOrElse( + apply().indices().open(new OpenIndexRequest(index), RequestOptions.DEFAULT).isAcknowledged, + false + )(logger) + } + + override def closeIndex(index: String): Boolean = { + tryOrElse( + apply().indices().close(new CloseIndexRequest(index), RequestOptions.DEFAULT).isAcknowledged, + false + )(logger) + } + + /** Reindex from source index to target index. + * + * @param sourceIndex + * - the name of the source index + * @param targetIndex + * - the name of the target index + * @param refresh + * - true to refresh the target index after reindexing, false otherwise + * @return + * true if the reindexing was successful, false otherwise + */ + override def reindex(sourceIndex: String, targetIndex: String, refresh: Boolean): Boolean = { + val request = new Request("POST", s"/_reindex?refresh=$refresh") + request.setJsonEntity( + s""" + |{ + | "source": { + | "index": "$sourceIndex" + | }, + | "dest": { + | "index": "$targetIndex" + | } + |} + """.stripMargin + ) + tryOrElse( + apply().getLowLevelClient.performRequest(request).getStatusLine.getStatusCode < 400, + false + )(logger) + } + + /** Check if an index exists. + * + * @param index + * - the name of the index to check + * @return + * true if the index exists, false otherwise + */ + override def indexExists(index: String): Boolean = { + tryOrElse( + apply().indices().exists(new GetIndexRequest(index), RequestOptions.DEFAULT), + false + )(logger) + } + +} + +trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientCompanion { + override def addAlias(index: String, alias: String): Boolean = { + tryOrElse( + apply() + .indices() + .updateAliases( + new IndicesAliasesRequest() + .addAliasAction( + new AliasActions(AliasActions.Type.ADD) + .index(index) + .alias(alias) + ), + RequestOptions.DEFAULT + ) + .isAcknowledged, + false + )(logger) + } + + override def removeAlias(index: String, alias: String): Boolean = { + tryOrElse( + apply() + .indices() + .updateAliases( + new IndicesAliasesRequest() + .addAliasAction( + new AliasActions(AliasActions.Type.REMOVE) + .index(index) + .alias(alias) + ), + RequestOptions.DEFAULT + ) + .isAcknowledged, + false + )(logger) + } +} + +trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClientCompanion { + _: RestHighLevelClientIndicesApi => + + override def updateSettings(index: String, settings: String): Boolean = { + tryOrElse( + apply() + .indices() + .putSettings( + new UpdateSettingsRequest(index) + .settings(settings, XContentType.JSON), + RequestOptions.DEFAULT + ) + .isAcknowledged, + false + )(logger) + } + + override def loadSettings(index: String): String = { + tryOrElse( + apply() + .indices() + .getSettings( + new GetSettingsRequest().indices(index), + RequestOptions.DEFAULT + ) + .toString, + s"""{"$index": {"settings": {"index": {}}}}""" + )(logger) + } +} + +trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientCompanion { + override def setMapping(index: String, mapping: String): Boolean = { + tryOrElse( + apply() + .indices() + .putMapping( + new PutMappingRequest(index) + .source(mapping, XContentType.JSON), + RequestOptions.DEFAULT + ) + .isAcknowledged, + false + )(logger) + } + + override def getMapping(index: String): String = { + tryOrElse( + apply() + .indices() + .getMapping( + new GetMappingsRequest().indices(index), + RequestOptions.DEFAULT + ) + .mappings() + .asScala + .get(index) + .map(metadata => metadata.source().string()), + None + )(logger).getOrElse(s""""{$index: {"mappings": {}}}""") + } +} + +trait RestHighLevelClientRefreshApi extends RefreshApi with RestHighLevelClientCompanion { + override def refresh(index: String): Boolean = { + tryOrElse( + apply() + .indices() + .refresh( + new RefreshRequest(index), + RequestOptions.DEFAULT + ) + .getStatus + .getStatus < 400, + false + )(logger) + } +} + +trait RestHighLevelClientFlushApi extends FlushApi with RestHighLevelClientCompanion { + override def flush(index: String, force: Boolean = true, wait: Boolean = true): Boolean = { + tryOrElse( + apply() + .indices() + .flush( + new FlushRequest(index).force(force).waitIfOngoing(wait), + RequestOptions.DEFAULT + ) + .getStatus == RestStatus.OK, + false + )(logger) + } +} + +trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientCompanion { + override def countAsync( + query: client.JSONQuery + )(implicit ec: ExecutionContext): Future[Option[Double]] = { + val promise = Promise[Option[Double]]() + apply().countAsync( + new CountRequest().indices(query.indices: _*).types(query.types: _*), + RequestOptions.DEFAULT, + new ActionListener[CountResponse] { + override def onResponse(response: CountResponse): Unit = + promise.success(Option(response.getCount.toDouble)) + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } + + override def count(query: client.JSONQuery): Option[Double] = { + tryOrElse( + Option( + apply() + .count( + new CountRequest().indices(query.indices: _*).types(query.types: _*), + RequestOptions.DEFAULT + ) + .getCount + .toDouble + ), + None + )(logger) + } +} + +trait RestHighLevelClientSingleValueAggregateApi + extends SingleValueAggregateApi + with RestHighLevelClientCountApi { + override def aggregate( + sqlQuery: SQLQuery + )(implicit ec: ExecutionContext): Future[Seq[SingleValueAggregateResult]] = { + val aggregations: Seq[ElasticAggregation] = sqlQuery + val futures = for (aggregation <- aggregations) yield { + val promise: Promise[SingleValueAggregateResult] = Promise() + val field = aggregation.field + val sourceField = aggregation.sourceField + val aggType = aggregation.aggType + val aggName = aggregation.aggName + val query = aggregation.query.getOrElse("") + val sources = aggregation.sources + sourceField match { + case "_id" if aggType.sql == "count" => + countAsync( + JSONQuery( + query, + collection.immutable.Seq(sources: _*), + collection.immutable.Seq.empty[String] + ) + ).onComplete { + case Success(result) => + promise.success( + SingleValueAggregateResult( + field, + aggType, + result.map(r => NumericValue(r.doubleValue())).getOrElse(EmptyValue), + None + ) + ) + case Failure(f) => + logger.error(f.getMessage, f.fillInStackTrace()) + promise.success( + SingleValueAggregateResult(field, aggType, EmptyValue, Some(f.getMessage)) + ) + } + promise.future + case _ => + val jsonQuery = JSONQuery( + query, + collection.immutable.Seq(sources: _*), + collection.immutable.Seq.empty[String] + ) + import jsonQuery._ + // Create a parser for the query + val xContentParser = XContentType.JSON + .xContent() + .createParser( + namedXContentRegistry, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + jsonQuery.query + ) + apply().searchAsync( + new SearchRequest(indices: _*) + .types(types: _*) + .source( + SearchSourceBuilder.fromXContent(xContentParser) + ), + RequestOptions.DEFAULT, + new ActionListener[SearchResponse] { + override def onResponse(response: SearchResponse): Unit = { + val agg = aggName.split("\\.").last + + val itAgg = aggName.split("\\.").iterator + + var root = + if (aggregation.nested) { + response.getAggregations.get(itAgg.next()).asInstanceOf[Nested].getAggregations + } else { + response.getAggregations + } + + if (aggregation.filtered) { + root = root.get(itAgg.next()).asInstanceOf[Filter].getAggregations + } + + promise.success( + SingleValueAggregateResult( + field, + aggType, + aggType match { + case sql.Count => + if (aggregation.distinct) { + NumericValue(root.get(agg).asInstanceOf[Cardinality].value()) + } else { + NumericValue(root.get(agg).asInstanceOf[ValueCount].value()) + } + case sql.Sum => + NumericValue(root.get(agg).asInstanceOf[Sum].value()) + case sql.Avg => + NumericValue(root.get(agg).asInstanceOf[Avg].value()) + case sql.Min => + NumericValue(root.get(agg).asInstanceOf[Min].value()) + case sql.Max => + NumericValue(root.get(agg).asInstanceOf[Max].value()) + case _ => EmptyValue + }, + None + ) + ) + } + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } + } + Future.sequence(futures) + } +} + +trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientCompanion { + _: RestHighLevelClientRefreshApi => + override def index(index: String, id: String, source: String): Boolean = { + tryOrElse( + apply() + .index( + new IndexRequest(index) + .`type`("_doc") + .id(id) + .source(source, XContentType.JSON), + RequestOptions.DEFAULT + ) + .status() + .getStatus < 400, + false + )(logger) + } + + override def indexAsync(index: String, id: String, source: String)(implicit + ec: ExecutionContext + ): Future[Boolean] = { + val promise: Promise[Boolean] = Promise() + apply().indexAsync( + new IndexRequest(index) + .`type`("_doc") + .id(id) + .source(source, XContentType.JSON), + RequestOptions.DEFAULT, + new ActionListener[IndexResponse] { + override def onResponse(response: IndexResponse): Unit = + promise.success(response.status().getStatus < 400) + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } +} + +trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientCompanion { + _: RestHighLevelClientRefreshApi => + override def update( + index: String, + id: String, + source: String, + upsert: Boolean + ): Boolean = { + tryOrElse( + apply() + .update( + new UpdateRequest(index, "_doc", id) + .doc(source, XContentType.JSON) + .docAsUpsert(upsert), + RequestOptions.DEFAULT + ) + .status() + .getStatus < 400, + false + )(logger) + } + + override def updateAsync( + index: String, + id: String, + source: String, + upsert: Boolean + )(implicit ec: ExecutionContext): Future[Boolean] = { + val promise: Promise[Boolean] = Promise() + apply().updateAsync( + new UpdateRequest(index, "_doc", id) + .doc(source, XContentType.JSON) + .docAsUpsert(upsert), + RequestOptions.DEFAULT, + new ActionListener[UpdateResponse] { + override def onResponse(response: UpdateResponse): Unit = + promise.success(response.status().getStatus < 400) + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } +} + +trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientCompanion { + _: RestHighLevelClientRefreshApi => + + override def delete(uuid: String, index: String): Boolean = { + tryOrElse( + apply() + .delete( + new DeleteRequest(index, "_doc", uuid), + RequestOptions.DEFAULT + ) + .status() + .getStatus < 400, + false + )(logger) + } + + override def deleteAsync(uuid: String, index: String)(implicit + ec: ExecutionContext + ): Future[Boolean] = { + val promise: Promise[Boolean] = Promise() + apply().deleteAsync( + new DeleteRequest(index, "_doc", uuid), + RequestOptions.DEFAULT, + new ActionListener[DeleteResponse] { + override def onResponse(response: DeleteResponse): Unit = + promise.success(response.status().getStatus < 400) + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } +} + +trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientCompanion { + def get[U <: Timestamped]( + id: String, + index: Option[String] = None, + maybeType: Option[String] = None + )(implicit m: Manifest[U], formats: Formats): Option[U] = { + Try( + apply().get( + new GetRequest( + index.getOrElse( + maybeType.getOrElse( + m.runtimeClass.getSimpleName.toLowerCase + ) + ), + maybeType.getOrElse("_all"), + id + ), + RequestOptions.DEFAULT + ) + ) match { + case Success(response) => + if (response.isExists) { + val source = response.getSourceAsString + logger.info(s"Deserializing response $source for id: $id, index: ${index + .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}") + // Deserialize the source string to the expected type + // Note: This assumes that the source is a valid JSON representation of U + // and that the serialization library is capable of handling it. + Try(serialization.read[U](source)) match { + case Success(value) => Some(value) + case Failure(f) => + logger.error( + s"Failed to deserialize response $source for id: $id, index: ${index + .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}", + f + ) + None + } + } else { + None + } + case Failure(f) => + logger.error( + s"Failed to get document with id: $id, index: ${index + .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}", + f + ) + None + } + } + + override def getAsync[U <: Timestamped]( + id: String, + index: Option[String] = None, + maybeType: Option[String] = None + )(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats): Future[Option[U]] = { + val promise = Promise[Option[U]]() + apply().getAsync( + new GetRequest( + index.getOrElse( + maybeType.getOrElse( + m.runtimeClass.getSimpleName.toLowerCase + ) + ), + maybeType.getOrElse("_all"), + id + ), + RequestOptions.DEFAULT, + new ActionListener[GetResponse] { + override def onResponse(response: GetResponse): Unit = { + if (response.isExists) { + promise.success(Some(serialization.read[U](response.getSourceAsString))) + } else { + promise.success(None) + } + } + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } +} + +trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientCompanion { + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + implicitly[ElasticSearchRequest](sqlSearch).query + + override def search[U]( + jsonQuery: JSONQuery + )(implicit m: Manifest[U], formats: Formats): List[U] = { + import jsonQuery._ + logger.info(s"Searching with query: $query on indices: ${indices.mkString(", ")}") + // Create a parser for the query + val xContentParser = XContentType.JSON + .xContent() + .createParser( + namedXContentRegistry, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + query + ) + val response = apply().search( + new SearchRequest(indices: _*) + .types(types: _*) + .source( + SearchSourceBuilder.fromXContent(xContentParser) + ), + RequestOptions.DEFAULT + ) + if (response.getHits.getTotalHits > 0) { + response.getHits.getHits.toList.map { hit => + logger.info(s"Deserializing hit: ${hit.getSourceAsString}") + serialization.read[U](hit.getSourceAsString) + } + } else { + List.empty[U] + } + } + + override def searchAsync[U]( + sqlQuery: SQLQuery + )(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats): Future[List[U]] = { + val jsonQuery: JSONQuery = sqlQuery + import jsonQuery._ + val promise = Promise[List[U]]() + logger.info(s"Searching with query: $query on indices: ${indices.mkString(", ")}") + // Create a parser for the query + val xContentParser = XContentType.JSON + .xContent() + .createParser( + namedXContentRegistry, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + query + ) + // Execute the search asynchronously + apply().searchAsync( + new SearchRequest(indices: _*) + .types(types: _*) + .source( + SearchSourceBuilder.fromXContent(xContentParser) + ), + RequestOptions.DEFAULT, + new ActionListener[SearchResponse] { + override def onResponse(response: SearchResponse): Unit = { + if (response.getHits.getTotalHits > 0) { + promise.success(response.getHits.getHits.toList.map { hit => + serialization.read[U](hit.getSourceAsString) + }) + } else { + promise.success(List.empty[U]) + } + } + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } + + override def searchWithInnerHits[U, I](jsonQuery: JSONQuery, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[(U, List[I])] = { + import jsonQuery._ + // Create a parser for the query + val xContentParser = XContentType.JSON + .xContent() + .createParser( + namedXContentRegistry, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + jsonQuery.query + ) + val response = apply().search( + new SearchRequest(indices: _*) + .types(types: _*) + .source( + SearchSourceBuilder.fromXContent(xContentParser) + ), + RequestOptions.DEFAULT + ) + Try(new JsonParser().parse(response.toString).getAsJsonObject ~> [U, I] innerField) match { + case Success(s) => s + case Failure(f) => + logger.error(f.getMessage, f) + List.empty + } + } + + override def multiSearch[U]( + jsonQueries: JSONQueries + )(implicit m: Manifest[U], formats: Formats): List[List[U]] = { + import jsonQueries._ + val request = new MultiSearchRequest() + for (query <- queries) { + request.add( + new SearchRequest(query.indices: _*) + .types(query.types: _*) + .source( + new SearchSourceBuilder( + new InputStreamStreamInput( + new ByteArrayInputStream( + query.query.getBytes() + ) + ) + ) + ) + ) + } + val responses = apply().msearch(request, RequestOptions.DEFAULT) + responses.getResponses.toList.map { response => + if (response.isFailure) { + logger.error(s"Error in multi search: ${response.getFailureMessage}") + List.empty[U] + } else { + response.getResponse.getHits.getHits.toList.map { hit => + serialization.read[U](hit.getSourceAsString) + } + } + } + } + + override def multiSearchWithInnerHits[U, I](jsonQueries: JSONQueries, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[List[(U, List[I])]] = { + import jsonQueries._ + val request = new MultiSearchRequest() + for (query <- queries) { + request.add( + new SearchRequest(query.indices: _*) + .types(query.types: _*) + .source( + new SearchSourceBuilder( + new InputStreamStreamInput( + new ByteArrayInputStream( + query.query.getBytes() + ) + ) + ) + ) + ) + } + val responses = apply().msearch(request, RequestOptions.DEFAULT) + responses.getResponses.toList.map { response => + if (response.isFailure) { + logger.error(s"Error in multi search: ${response.getFailureMessage}") + List.empty[(U, List[I])] + } else { + Try( + new JsonParser().parse(response.getResponse.toString).getAsJsonObject ~> [U, I] innerField + ) match { + case Success(s) => s + case Failure(f) => + logger.error(f.getMessage, f) + List.empty + } + } + } + } + +} + +trait RestHighLevelClientBulkApi + extends RestHighLevelClientRefreshApi + with RestHighLevelClientSettingsApi + with RestHighLevelClientIndicesApi + with BulkApi { + override type A = DocWriteRequest[_] + override type R = BulkResponse + + override def toBulkAction(bulkItem: BulkItem): A = { + import bulkItem._ + val request = action match { + case BulkAction.UPDATE => + val r = new UpdateRequest(index, null, if (id.isEmpty) null else id.get) + .doc(body, XContentType.JSON) + .docAsUpsert(true) + parent.foreach(r.parent) + r + case BulkAction.DELETE => + val r = new DeleteRequest(index).id(id.getOrElse("_all")) + parent.foreach(r.parent) + r + case _ => + val r = new IndexRequest(index).source(body, XContentType.JSON) + id.foreach(r.id) + parent.foreach(r.parent) + r + } + request + } + + override def bulkResult: Flow[R, Set[String], NotUsed] = + Flow[BulkResponse] + .named("result") + .map(result => { + val items = result.getItems + val grouped = items.groupBy(_.getIndex) + val indices = grouped.keys.toSet + for (index <- indices) { + logger + .info(s"Bulk operation succeeded for index $index with ${grouped(index).length} items.") + } + indices + }) + + override def bulk(implicit + bulkOptions: BulkOptions, + system: ActorSystem + ): Flow[Seq[A], R, NotUsed] = { + val parallelism = Math.max(1, bulkOptions.balance) + Flow[Seq[A]] + .named("bulk") + .mapAsyncUnordered[R](parallelism) { items => + val request = new BulkRequest(bulkOptions.index, bulkOptions.documentType) + items.foreach(request.add) + val promise: Promise[R] = Promise[R]() + apply().bulkAsync( + request, + RequestOptions.DEFAULT, + new ActionListener[BulkResponse] { + override def onResponse(response: BulkResponse): Unit = { + if (response.hasFailures) { + logger.error(s"Bulk operation failed: ${response.buildFailureMessage()}") + } else { + logger.info(s"Bulk operation succeeded with ${response.getItems.length} items.") + } + promise.success(response) + } + + override def onFailure(e: Exception): Unit = { + logger.error("Bulk operation failed", e) + promise.failure(e) + } + } + ) + promise.future + } + } + + private[this] def toBulkElasticResultItem(i: BulkItemResponse): BulkElasticResultItem = + new BulkElasticResultItem { + override def index: String = i.getIndex + } + + override implicit def toBulkElasticAction(a: DocWriteRequest[_]): BulkElasticAction = { + new BulkElasticAction { + override def index: String = a.index + } + } + + override implicit def toBulkElasticResult(r: BulkResponse): BulkElasticResult = { + new BulkElasticResult { + override def items: List[BulkElasticResultItem] = + r.getItems.toList.map(toBulkElasticResultItem) + } + } +} diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala new file mode 100644 index 00000000..1dd158a2 --- /dev/null +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala @@ -0,0 +1,53 @@ +package app.softnetwork.elastic.client.rest + +import app.softnetwork.elastic.client.ElasticConfig +import com.sksamuel.exts.Logging +import org.apache.http.HttpHost +import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.client.{RestClient, RestClientBuilder, RestHighLevelClient} +import org.elasticsearch.common.settings.Settings +import org.elasticsearch.common.xcontent.NamedXContentRegistry +import org.elasticsearch.plugins.SearchPlugin +import org.elasticsearch.search.SearchModule + +trait RestHighLevelClientCompanion extends Logging { + + def elasticConfig: ElasticConfig + + private var client: Option[RestHighLevelClient] = None + + lazy val namedXContentRegistry: NamedXContentRegistry = { + import scala.collection.JavaConverters._ + val searchModule = new SearchModule(Settings.EMPTY, false, List.empty[SearchPlugin].asJava) + new NamedXContentRegistry(searchModule.getNamedXContents) + } + + def apply(): RestHighLevelClient = { + client match { + case Some(c) => c + case _ => + val credentialsProvider = new BasicCredentialsProvider() + if (elasticConfig.credentials.username.nonEmpty) { + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials( + elasticConfig.credentials.username, + elasticConfig.credentials.password + ) + ) + } + val restClientBuilder: RestClientBuilder = RestClient + .builder( + HttpHost.create(elasticConfig.credentials.url) + ) + .setHttpClientConfigCallback((httpAsyncClientBuilder: HttpAsyncClientBuilder) => + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + ) + val c = new RestHighLevelClient(restClientBuilder) + client = Some(c) + c + } + } +} diff --git a/es6/sql-bridge/build.sbt b/es6/sql-bridge/build.sbt new file mode 100644 index 00000000..4de62eb1 --- /dev/null +++ b/es6/sql-bridge/build.sbt @@ -0,0 +1,8 @@ +import SoftClient4es.* + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-sql-bridge" + +libraryDependencies ++= elasticDependencies(elasticSearchVersion.value) ++ + elastic4sDependencies(elasticSearchVersion.value) diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala new file mode 100644 index 00000000..6b7b578a --- /dev/null +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -0,0 +1,120 @@ +package app.softnetwork.elastic.sql.bridge + +import app.softnetwork.elastic.sql.{ + AggregateFunction, + Avg, + Count, + ElasticBoolQuery, + Max, + Min, + SQLAggregate, + Sum +} +import com.sksamuel.elastic4s.ElasticApi.{ + avgAgg, + cardinalityAgg, + filterAgg, + matchAllQuery, + maxAgg, + minAgg, + nestedAggregation, + sumAgg, + valueCountAgg +} +import com.sksamuel.elastic4s.searches.aggs.Aggregation + +import scala.language.implicitConversions + +case class ElasticAggregation( + aggName: String, + field: String, + sourceField: String, + sources: Seq[String] = Seq.empty, + query: Option[String] = None, + distinct: Boolean = false, + nested: Boolean = false, + filtered: Boolean = false, + aggType: AggregateFunction, + agg: Aggregation +) + +object ElasticAggregation { + def apply(sqlAgg: SQLAggregate): ElasticAggregation = { + import sqlAgg._ + val sourceField = identifier.columnName + + val field = alias match { + case Some(alias) => alias.alias + case _ => sourceField + } + + val distinct = identifier.distinct.isDefined + + val agg = + if (distinct) + s"${function}_distinct_${sourceField.replace(".", "_")}" + else + s"${function}_${sourceField.replace(".", "_")}" + + var aggPath = Seq[String]() + + val _agg = + function match { + case Count => + if (distinct) + cardinalityAgg(agg, sourceField) + else { + valueCountAgg(agg, sourceField) + } + case Min => minAgg(agg, sourceField) + case Max => maxAgg(agg, sourceField) + case Avg => avgAgg(agg, sourceField) + case Sum => sumAgg(agg, sourceField) + } + + def _filtered: Aggregation = filter match { + case Some(f) => + val boolQuery = Option(ElasticBoolQuery(group = true)) + val filteredAgg = s"filtered_agg" + aggPath ++= Seq(filteredAgg) + filterAgg( + filteredAgg, + f.criteria + .map( + _.asFilter(boolQuery) + .query(Set(identifier.innerHitsName).flatten, boolQuery) + ) + .getOrElse(matchAllQuery()) + ) subaggs { + aggPath ++= Seq(agg) + _agg + } + case _ => + aggPath ++= Seq(agg) + _agg + } + + val aggregation = + if (identifier.nested) { + val path = sourceField.split("\\.").head + val nestedAgg = s"nested_$agg" + aggPath ++= Seq(nestedAgg) + nestedAggregation(nestedAgg, path) subaggs { + _filtered + } + } else { + _filtered + } + + ElasticAggregation( + aggPath.mkString("."), + field, + sourceField, + distinct = distinct, + nested = identifier.nested, + filtered = filter.nonEmpty, + aggType = function, + agg = aggregation + ) + } +} diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala new file mode 100644 index 00000000..bf6ebe38 --- /dev/null +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -0,0 +1,15 @@ +package app.softnetwork.elastic.sql.bridge + +import app.softnetwork.elastic.sql.SQLCriteria +import com.sksamuel.elastic4s.searches.queries.Query + +case class ElasticCriteria(criteria: SQLCriteria) { + + def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty): Query = { + val query = criteria.boolQuery.copy(group = group) + query + .filter(criteria.asFilter(Option(query))) + .unfilteredMatchCriteria() + .query(innerHitsNames, Option(query)) + } +} diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticMultiSearchRequest.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticMultiSearchRequest.scala new file mode 100644 index 00000000..61925b7b --- /dev/null +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticMultiSearchRequest.scala @@ -0,0 +1,11 @@ +package app.softnetwork.elastic.sql.bridge + +import com.sksamuel.elastic4s.http.search.MultiSearchBuilderFn +import com.sksamuel.elastic4s.searches.MultiSearchRequest + +case class ElasticMultiSearchRequest( + requests: Seq[ElasticSearchRequest], + multiSearch: MultiSearchRequest +) { + def query: String = MultiSearchBuilderFn(multiSearch).replace("\"version\":true,", "") /*FIXME*/ +} diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala new file mode 100644 index 00000000..d49cf08c --- /dev/null +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -0,0 +1,74 @@ +package app.softnetwork.elastic.sql.bridge + +import app.softnetwork.elastic.sql.{ + ElasticBoolQuery, + ElasticChild, + ElasticFilter, + ElasticGeoDistance, + ElasticMatch, + ElasticNested, + ElasticParent, + SQLBetween, + SQLExpression, + SQLIn, + SQLIsNotNull, + SQLIsNull +} +import com.sksamuel.elastic4s.ElasticApi.{bool, _} +import com.sksamuel.elastic4s.searches.queries.Query + +case class ElasticQuery(filter: ElasticFilter) { + def query( + innerHitsNames: Set[String] = Set.empty, + currentQuery: Option[ElasticBoolQuery] + ): Query = { + filter match { + case boolQuery: ElasticBoolQuery => + import boolQuery._ + bool( + mustFilters.map(implicitly[ElasticQuery](_).query(innerHitsNames, currentQuery)), + shouldFilters.map(implicitly[ElasticQuery](_).query(innerHitsNames, currentQuery)), + notFilters.map(implicitly[ElasticQuery](_).query(innerHitsNames, currentQuery)) + ) + .filter(innerFilters.map(_.query(innerHitsNames, currentQuery))) + case nested: ElasticNested => + import nested._ + if (innerHitsNames.contains(innerHitsName.getOrElse(""))) { + criteria.asFilter(currentQuery).query(innerHitsNames, currentQuery) + } else { + val boolQuery = Option(ElasticBoolQuery(group = true)) + nestedQuery( + relationType.getOrElse(""), + criteria + .asFilter(boolQuery) + .query(innerHitsNames + innerHitsName.getOrElse(""), boolQuery) + ) /*.scoreMode(ScoreMode.None)*/ + .inner( + innerHits(innerHitsName.getOrElse("")).from(0).size(limit.map(_.limit).getOrElse(3)) + ) + } + case child: ElasticChild => + import child._ + hasChildQuery( + relationType.getOrElse(""), + criteria.asQuery(group = group, innerHitsNames = innerHitsNames) + ) + case parent: ElasticParent => + import parent._ + hasParentQuery( + relationType.getOrElse(""), + criteria.asQuery(group = group, innerHitsNames = innerHitsNames), + score = false + ) + case expression: SQLExpression => expression + case isNull: SQLIsNull => isNull + case isNotNull: SQLIsNotNull => isNotNull + case in: SQLIn[_, _] => in + case between: SQLBetween => between + case geoDistance: ElasticGeoDistance => geoDistance + case matchExpression: ElasticMatch => matchExpression + case other => + throw new IllegalArgumentException(s"Unsupported filter type: ${other.getClass.getName}") + } + } +} diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala new file mode 100644 index 00000000..83310eaa --- /dev/null +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -0,0 +1,25 @@ +package app.softnetwork.elastic.sql.bridge + +import app.softnetwork.elastic.sql.{SQLCriteria, SQLExcept, SQLField} +import com.sksamuel.elastic4s.searches.SearchRequest +import com.sksamuel.elastic4s.http.search.SearchBodyBuilderFn + +case class ElasticSearchRequest( + fields: Seq[SQLField], + except: Option[SQLExcept], + sources: Seq[String], + criteria: Option[SQLCriteria], + limit: Option[Int], + search: SearchRequest, + aggregations: Seq[ElasticAggregation] = Seq.empty +) { + def minScore(score: Option[Double]): ElasticSearchRequest = { + score match { + case Some(s) => this.copy(search = search minScore s) + case _ => this + } + } + + def query: String = + SearchBodyBuilderFn(search).string.replace("\"version\":true,", "") /*FIXME*/ +} diff --git a/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala new file mode 100644 index 00000000..98ad716f --- /dev/null +++ b/es6/sql-bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -0,0 +1,312 @@ +package app.softnetwork.elastic.sql + +import com.sksamuel.elastic4s.ElasticApi +import com.sksamuel.elastic4s.ElasticApi._ +import com.sksamuel.elastic4s.http.ElasticDsl.BuildableTermsNoOp +import com.sksamuel.elastic4s.http.search.SearchBodyBuilderFn +import com.sksamuel.elastic4s.searches.queries.Query +import com.sksamuel.elastic4s.searches.{MultiSearchRequest, SearchRequest} +import com.sksamuel.elastic4s.searches.sort.FieldSort + +import scala.language.implicitConversions + +package object bridge { + implicit def requestToElasticSearchRequest(request: SQLSearchRequest): ElasticSearchRequest = + ElasticSearchRequest( + request.select.fields, + request.select.except, + request.sources, + request.where.flatMap(_.criteria), + request.limit.map(_.limit), + request, + request.aggregates.map(ElasticAggregation(_)) + ).minScore(request.score) + + implicit def requestToSearchRequest(request: SQLSearchRequest): SearchRequest = { + import request._ + val aggregations = aggregates.map(ElasticAggregation(_)) + var _search: SearchRequest = search("") query { + where.flatMap(_.criteria.map(_.asQuery())).getOrElse(matchAllQuery()) + } sourceInclude fields + + _search = excludes match { + case Nil => _search + case excludes => _search sourceExclude excludes + } + + _search = aggregations match { + case Nil => _search + case _ => _search aggregations { aggregations.map(_.agg) } + } + + _search = orderBy match { + case Some(o) => + _search sortBy o.sorts.map(sort => + sort.order match { + case Some(Desc) => FieldSort(sort.field).desc() + case _ => FieldSort(sort.field).asc() + } + ) + case _ => _search + } + + if (aggregations.nonEmpty && fields.isEmpty) { + _search size 0 + } else { + limit match { + case Some(l) => _search limit l.limit from 0 + case _ => _search + } + } + } + + implicit def requestToMultiSearchRequest( + request: SQLMultiSearchRequest + ): MultiSearchRequest = { + MultiSearchRequest( + request.requests.map(implicitly[SearchRequest](_)) + ) + } + + implicit def expressionToQuery(expression: SQLExpression): Query = { + import expression._ + value match { + case n: SQLNumeric[Any] @unchecked => + operator match { + case _: Ge.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) lt n.sql + case _ => + rangeQuery(identifier.columnName) gte n.sql + } + case _: Gt.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) lte n.sql + case _ => + rangeQuery(identifier.columnName) gt n.sql + } + case _: Le.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) gt n.sql + case _ => + rangeQuery(identifier.columnName) lte n.sql + } + case _: Lt.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) gte n.sql + case _ => + rangeQuery(identifier.columnName) lt n.sql + } + case _: Eq.type => + maybeNot match { + case Some(_) => + not(termQuery(identifier.columnName, n.sql)) + case _ => + termQuery(identifier.columnName, n.sql) + } + case _: Ne.type => + maybeNot match { + case Some(_) => + termQuery(identifier.columnName, n.sql) + case _ => + not(termQuery(identifier.columnName, n.sql)) + } + case _ => matchAllQuery() + } + case l: SQLLiteral => + operator match { + case _: Like.type => + maybeNot match { + case Some(_) => + not(regexQuery(identifier.columnName, toRegex(l.value))) + case _ => + regexQuery(identifier.columnName, toRegex(l.value)) + } + case _: Ge.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) lt l.value + case _ => + rangeQuery(identifier.columnName) gte l.value + } + case _: Gt.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) lte l.value + case _ => + rangeQuery(identifier.columnName) gt l.value + } + case _: Le.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) gt l.value + case _ => + rangeQuery(identifier.columnName) lte l.value + } + case _: Lt.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) gte l.value + case _ => + rangeQuery(identifier.columnName) lt l.value + } + case _: Eq.type => + maybeNot match { + case Some(_) => + not(termQuery(identifier.columnName, l.value)) + case _ => + termQuery(identifier.columnName, l.value) + } + case _: Ne.type => + maybeNot match { + case Some(_) => + termQuery(identifier.columnName, l.value) + case _ => + not(termQuery(identifier.columnName, l.value)) + } + case _ => matchAllQuery() + } + case b: SQLBoolean => + operator match { + case _: Eq.type => + maybeNot match { + case Some(_) => + not(termQuery(identifier.columnName, b.value)) + case _ => + termQuery(identifier.columnName, b.value) + } + case _: Ne.type => + maybeNot match { + case Some(_) => + termQuery(identifier.columnName, b.value) + case _ => + not(termQuery(identifier.columnName, b.value)) + } + case _ => matchAllQuery() + } + case _ => matchAllQuery() + } + } + + implicit def isNullToQuery( + isNull: SQLIsNull + ): Query = { + import isNull._ + not(existsQuery(identifier.columnName)) + } + + implicit def isNotNullToQuery( + isNotNull: SQLIsNotNull + ): Query = { + import isNotNull._ + existsQuery(identifier.columnName) + } + + implicit def inToQuery[R, T <: SQLValue[R]](in: SQLIn[R, T]): Query = { + import in._ + val _values: Seq[Any] = values.innerValues + val t = + _values.headOption match { + case Some(_: Double) => + termsQuery(identifier.columnName, _values.asInstanceOf[Seq[Double]]) + case Some(_: Integer) => + termsQuery(identifier.columnName, _values.asInstanceOf[Seq[Integer]]) + case Some(_: Long) => + termsQuery(identifier.columnName, _values.asInstanceOf[Seq[Long]]) + case _ => termsQuery(identifier.columnName, _values.map(_.toString)) + } + maybeNot match { + case Some(_) => not(t) + case _ => t + } + } + + implicit def betweenToQuery( + between: SQLBetween + ): Query = { + import between._ + val r = rangeQuery(identifier.columnName) gte from.value lte to.value + maybeNot match { + case Some(_) => not(r) + case _ => r + } + } + + implicit def geoDistanceToQuery( + geoDistance: ElasticGeoDistance + ): Query = { + import geoDistance._ + geoDistanceQuery(identifier.columnName, lat.value, lon.value) distance distance.value + } + + implicit def matchToQuery( + matchExpression: ElasticMatch + ): Query = { + import matchExpression._ + matchQuery(identifier.columnName, value.value) + } + + implicit def criteriaToElasticCriteria( + criteria: SQLCriteria + ): ElasticCriteria = { + ElasticCriteria( + criteria + ) + } + + implicit def filterToQuery( + filter: ElasticFilter + ): ElasticQuery = { + ElasticQuery(filter) + } + + implicit def sqlQueryToAggregations( + query: SQLQuery + ): Seq[ElasticAggregation] = { + import query._ + request + .map { + case Left(l) => + l.aggregates + .map(ElasticAggregation(_)) + .map(aggregation => { + val queryFiltered = + l.where + .flatMap(_.criteria.map(ElasticCriteria(_).asQuery())) + .getOrElse(matchAllQuery()) + + aggregation.copy( + sources = l.sources, + query = Some( + (aggregation.aggType match { + case Count if aggregation.sourceField.equalsIgnoreCase("_id") => + SearchBodyBuilderFn( + ElasticApi.search("") query { + queryFiltered + } + ) + case _ => + SearchBodyBuilderFn( + ElasticApi.search("") query { + queryFiltered + } + aggregations { + aggregation.agg + } + size 0 + ) + }).string().replace("\"version\":true,", "") /*FIXME*/ + ) + ) + }) + + case _ => Seq.empty + + } + .getOrElse(Seq.empty) + } +} diff --git a/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala new file mode 100644 index 00000000..9f938334 --- /dev/null +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -0,0 +1,875 @@ +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import com.sksamuel.elastic4s.ElasticApi.matchAllQuery +import com.sksamuel.elastic4s.http.search.SearchBodyBuilderFn +import com.sksamuel.elastic4s.searches.SearchRequest +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** Created by smanciot on 13/04/17. + */ +class SQLCriteriaSpec extends AnyFlatSpec with Matchers { + + import Queries._ + + import scala.language.implicitConversions + + def asQuery(sql: String): String = { + import SQLImplicits._ + val criteria: Option[SQLCriteria] = sql + val result = SearchBodyBuilderFn( + SearchRequest("*") query criteria.map(_.asQuery()).getOrElse(matchAllQuery()) + ).string + println(result) + result + } + + "SQLCriteria" should "filter numerical eq" in { + asQuery(numericalEq) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"term" : { + | "identifier" : { + | "value" : "1.0" + | } + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter numerical ne" in { + asQuery(numericalNe) shouldBe """{ + + |"query":{ + | "bool":{ + | "filter":[{"bool":{"must_not":[ + | { + | "term":{ + | "identifier":{ + | "value":"1" + | } + | } + | } + | ] + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter numerical lt" in { + asQuery(numericalLt) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"range" : { + | "identifier" : { + | "lt" : "1" + | } + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter numerical le" in { + asQuery(numericalLe) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"range" : { + | "identifier" : { + | "lte" : "1" + | } + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter numerical gt" in { + asQuery(numericalGt) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"range" : { + | "identifier" : { + | "gt" : "1" + | } + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter numerical ge" in { + asQuery(numericalGe) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"range" : { + | "identifier" : { + | "gte" : "1" + | } + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter literal eq" in { + asQuery(literalEq) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"term" : { + | "identifier" : { + | "value" : "un" + | } + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter literal ne" in { + asQuery(literalNe) shouldBe """{ + + |"query":{ + | "bool" : { + | "filter":[{"bool":{"must_not" : [ + | { + | "term" : { + | "identifier" : { + | "value" : "un" + | } + | } + | } + | ] + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter literal like" in { + asQuery(literalLike) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"regexp" : { + | "identifier" : { + | "value" : ".*?un.*?" + | } + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter literal not like" in { + asQuery(literalNotLike) shouldBe """{ + |"query":{ + | "bool": { + | "filter":[{"bool":{"must_not": [{ + | "regexp": { + | "identifier": { + | "value": ".*?un.*?" + | } + | } + | }] + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter between" in { + asQuery(betweenExpression) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"range" : { + | "identifier" : { + | "gte" : "1", + | "lte" : "2" + | } + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter and predicate" in { + asQuery(andPredicate) shouldBe """{ + + |"query":{ + | "bool":{ + | "filter" : [ + | { + | "term" : { + | "identifier1" : { + | "value" : "1" + | } + | } + | }, + | { + | "range" : { + | "identifier2" : { + | "gt" : "2" + | } + | } + | } + | ] + | } + | } + |}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter or predicate" in { + asQuery(orPredicate) shouldBe """{ + + |"query":{ + | "bool":{ + | "should" : [ + | { + | "term" : { + | "identifier1" : { + | "value" : "1" + | } + | } + | }, + | { + | "range" : { + | "identifier2" : { + | "gt" : "2" + | } + | } + | } + | ] + | } + | } + |}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter left predicate with criteria" in { + asQuery(leftPredicate) shouldBe """{ + + |"query":{ + | "bool":{ + | "should" : [ + | { + | "bool" : { + | "filter" : [ + | { + | "term" : { + | "identifier1" : { + | "value" : "1" + | } + | } + | }, + | { + | "range" : { + | "identifier2" : { + | "gt" : "2" + | } + | } + | } + | ] + | } + | }, + | { + | "term" : { + | "identifier3" : { + | "value" : "3" + | } + | } + | } + | ] + | } + | } + |}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter right predicate with criteria" in { + asQuery(rightPredicate) shouldBe """{ + + |"query":{ + | "bool":{ + | "filter" : [ + | { + | "term" : { + | "identifier1" : { + | "value" : "1" + | } + | } + | }, + | { + | "bool" : { + | "should" : [ + | { + | "range" : { + | "identifier2" : { + | "gt" : "2" + | } + | } + | }, + | { + | "term" : { + | "identifier3" : { + | "value" : "3" + | } + | } + | } + | ] + | } + | } + | ] + | } + | } + |}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter multiple predicates" in { + asQuery(predicates) shouldBe """{ + + |"query":{ + | "bool":{ + | "should" : [ + | { + | "bool" : { + | "filter" : [ + | { + | "term" : { + | "identifier1" : { + | "value" : "1" + | } + | } + | }, + | { + | "range" : { + | "identifier2" : { + | "gt" : "2" + | } + | } + | } + | ] + | } + | }, + | { + | "bool" : { + | "filter" : [ + | { + | "term" : { + | "identifier3" : { + | "value" : "3" + | } + | } + | }, + | { + | "term" : { + | "identifier4" : { + | "value" : "4" + | } + | } + | } + | ] + | } + | } + | ] + | } + | } + |}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter in literal expression" in { + asQuery(inLiteralExpression) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"terms" : { + | "identifier" : [ + | "val1", + | "val2", + | "val3" + | ] + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter in numerical expression with Int values" in { + asQuery(inNumericalExpressionWithIntValues) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"terms" : { + | "identifier" : [ + | 1, + | 2, + | 3 + | ] + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter in numerical expression with Double values" in { + asQuery(inNumericalExpressionWithDoubleValues) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"terms" : { + | "identifier" : [ + | 1.0, + | 2.1, + | 3.4 + | ] + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter nested predicate" in { + asQuery(nestedPredicate) shouldBe """{ + + |"query":{ + | "bool":{ + | "filter" : [ + | { + | "term" : { + | "identifier1" : { + | "value" : "1" + | } + | } + | }, + | { + | "nested" : { + | "path" : "nested", + | "query" : { + | "bool" : { + | "should" : [ + | { + | "range" : { + | "nested.identifier2" : { + | "gt" : "2" + | } + | } + | }, + | { + | "term" : { + | "nested.identifier3" : { + | "value" : "3" + | } + | } + | } + | ] + | } + | }, + | "inner_hits":{"name":"nested","from":0,"size":3} + | } + | } + | ] + | } + | } + |}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter nested criteria" in { + asQuery(nestedCriteria) shouldBe """{ + + |"query":{ + | "bool":{ + | "filter" : [ + | { + | "term" : { + | "identifier1" : { + | "value" : "1" + | } + | } + | }, + | { + | "nested" : { + | "path" : "nested", + | "query" : { + | "term" : { + | "nested.identifier3" : { + | "value" : "3" + | } + | } + | }, + | "inner_hits":{"name":"nested","from":0,"size":3} + | } + | } + | ] + | } + | } + |}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter child predicate" in { + asQuery(childPredicate) shouldBe + """ + |{ + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "identifier1": { + | "value": "1" + | } + | } + | }, + | { + | "has_child": { + | "type": "child", + | "score_mode": "none", + | "query": { + | "bool": { + | "filter": [ + | { + | "bool": { + | "should": [ + | { + | "range": { + | "child.identifier2": { + | "gt": "2" + | } + | } + | }, + | { + | "term": { + | "child.identifier3": { + | "value": "3" + | } + | } + | } + | ] + | } + | } + | ] + | } + | } + | } + | } + | ] + | } + | } + |}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter child criteria" in { + asQuery(childCriteria) shouldBe + """ + |{ + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "identifier1": { + | "value": "1" + | } + | } + | }, + | { + | "has_child": { + | "type": "child", + | "score_mode": "none", + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "child.identifier3": { + | "value": "3" + | } + | } + | } + | ] + | } + | } + | } + | } + | ] + | } + | } + |}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter parent predicate" in { + asQuery(parentPredicate) shouldBe + """ + |{ + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "identifier1": { + | "value": "1" + | } + | } + | }, + | { + | "has_parent": { + | "parent_type": "parent", + | "query": { + | "bool": { + | "filter": [ + | { + | "bool": { + | "should": [ + | { + | "range": { + | "parent.identifier2": { + | "gt": "2" + | } + | } + | }, + | { + | "term": { + | "parent.identifier3": { + | "value": "3" + | } + | } + | } + | ] + | } + | } + | ] + | } + | } + | } + | } + | ] + | } + | } + |}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter parent criteria" in { + asQuery(parentCriteria) shouldBe + """ + |{ + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "identifier1": { + | "value": "1" + | } + | } + | }, + | { + | "has_parent": { + | "parent_type": "parent", + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "parent.identifier3": { + | "value": "3" + | } + | } + | } + | ] + | } + | } + | } + | } + | ] + | } + | } + |}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter nested with between" in { + asQuery(nestedWithBetween) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"nested" : { + | "path" : "ciblage", + | "query" : { + | "bool" : { + | "filter" : [ + | { + | "range" : { + | "ciblage.Archivage_CreationDate" : { + | "gte" : "now-3M/M", + | "lte" : "now" + | } + | } + | }, + | { + | "term" : { + | "ciblage.statutComportement" : { + | "value" : "1" + | } + | } + | } + | ] + | } + | }, + | "inner_hits":{"name":"ciblage","from":0,"size":3} + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter boolean eq" in { + asQuery(boolEq) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"term" : { + | "identifier" : { + | "value" : true + | } + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter boolean ne" in { + asQuery(boolNe) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"bool" : { + | "must_not" : [ + | { + | "term" : { + | "identifier" : { + | "value" : false + | } + | } + | } + | ] + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter is null" in { + asQuery(isNull) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"bool" : { + | "must_not" : [ + | { + | "exists" : { + | "field" : "identifier" + | } + | } + | ] + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter is not null" in { + asQuery(isNotNull) shouldBe """{ + + |"query":{ + | "bool":{"filter":[{"exists" : { + | "field" : "identifier" + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter geo distance criteria" in { + asQuery(geoDistanceCriteria) shouldBe + """{ + + |"query": { + | "bool":{"filter":[{"geo_distance": { + | "distance": "5km", + | "profile.location": [ + | 40.0, + | -70.0 + | ] + | } + | } + |]}}}""".stripMargin.replaceAll("\\s", "") + } + + it should "filter match criteria" in { + asQuery(matchCriteria) shouldBe + """{ + | "query":{ + | "bool":{ + | "filter":[ + | { + | "match":{ + | "identifier":{ + | "query":"value" + | } + | } + | } + | ] + | } + | } + | }""".stripMargin.replaceAll("\\s", "") + } + + it should "filter complex queries" in { + val query = + """select * from Table + |where (identifier is not null and identifier = 1) or + |( + | (identifier is null or identifier2 > 2) + | and identifier3 = 3 + |)""".stripMargin + asQuery(query) shouldBe + """ + |{ + | "query": { + | "bool": { + | "should": [ + | { + | "bool": { + | "filter": [ + | { + | "exists": { + | "field": "identifier" + | } + | }, + | { + | "term": { + | "identifier": { + | "value": "1" + | } + | } + | } + | ] + | } + | }, + | { + | "bool": { + | "filter": [ + | { + | "bool": { + | "should": [ + | { + | "bool": { + | "must_not": [ + | { + | "exists": { + | "field": "identifier" + | } + | } + | ] + | } + | }, + | { + | "range": { + | "identifier2": { + | "gt": "2" + | } + | } + | } + | ] + | } + | }, + | { + | "term": { + | "identifier3": { + | "value": "3" + | } + | } + | } + | ] + | } + | } + | ] + | } + | } + |} + |""".stripMargin.replaceAll("\\s", "") + } + +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala similarity index 95% rename from sql/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala rename to es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala index a6ec75b6..8d2e2271 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala +++ b/es6/sql-bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -1,11 +1,12 @@ package app.softnetwork.elastic.sql +import app.softnetwork.elastic.sql.bridge._ import app.softnetwork.elastic.sql.Queries._ import com.google.gson.{JsonArray, JsonObject, JsonParser, JsonPrimitive} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import scala.collection.JavaConverters.asScalaIteratorConverter +import scala.jdk.CollectionConverters.IteratorHasAsScala /** Created by smanciot on 13/04/17. */ @@ -13,9 +14,20 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { import scala.language.implicitConversions + implicit def sqlQueryToRequest(sqlQuery: SQLQuery): ElasticSearchRequest = { + sqlQuery.request match { + case Some(Left(value)) => + value.copy(score = sqlQuery.score) + case None => + throw new IllegalArgumentException( + s"SQL query ${sqlQuery.query} does not contain a valid search request" + ) + } + } + "SQLQuery" should "perform native count" in { - val results = - SQLQuery("select count(t.id) c2 from Table t where t.nom = \"Nom\"").aggregations + val results: Seq[ElasticAggregation] = + SQLQuery("select count(t.id) c2 from Table t where t.nom = \"Nom\"") results.size shouldBe 1 val result = results.head result.nested shouldBe false @@ -50,8 +62,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } it should "perform count distinct" in { - val results = - SQLQuery("select count(distinct t.id) as c2 from Table as t where nom = \"Nom\"").aggregations + val results: Seq[ElasticAggregation] = + SQLQuery("select count(distinct t.id) as c2 from Table as t where nom = \"Nom\"") results.size shouldBe 1 val result = results.head result.nested shouldBe false @@ -86,10 +98,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } it should "perform nested count" in { - val results = + val results: Seq[ElasticAggregation] = SQLQuery( "select count(inner_emails.value) as email from index i, unnest(emails) as inner_emails where i.nom = \"Nom\"" - ).aggregations + ) results.size shouldBe 1 val result = results.head result.nested shouldBe true @@ -131,10 +143,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } it should "perform nested count with nested criteria" in { - val results = + val results: Seq[ElasticAggregation] = SQLQuery( "select count(inner_emails.value) as count_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))" - ).aggregations + ) results.size shouldBe 1 val result = results.head result.nested shouldBe true @@ -190,10 +202,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } it should "perform nested count with filter" in { - val results = + val results: Seq[ElasticAggregation] = SQLQuery( "select count(inner_emails.value) as count_emails filter[inner_emails.context = \"profile\"] from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))" - ).aggregations + ) results.size shouldBe 1 val result = results.head result.nested shouldBe true @@ -260,10 +272,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } it should "perform nested count with \"and not\" operator" in { - val results = + val results: Seq[ElasticAggregation] = SQLQuery( "select count(distinct inner_emails.value) as count_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where ((inner_profiles.postalCode = \"33600\") and (inner_profiles.postalCode <> \"75001\"))" - ).aggregations + ) results.size shouldBe 1 val result = results.head result.nested shouldBe true @@ -336,10 +348,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } it should "perform nested count with date filtering" in { - val results = + val results: Seq[ElasticAggregation] = SQLQuery( "select count(distinct inner_emails.value) as count_distinct_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where inner_profiles.postalCode = \"33600\" and inner_profiles.createdDate <= \"now-35M/M\"" - ).aggregations + ) results.size shouldBe 1 val result = results.head result.nested shouldBe true @@ -404,7 +416,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } it should "perform nested select" in { - val select = + val select: ElasticSearchRequest = SQLQuery(""" |SELECT |profileId, @@ -419,10 +431,8 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { |((profile_ccm.postalCode BETWEEN "10" AND "99999") |AND |(profile_ccm.birthYear <= 2000)) - |limit 100""".stripMargin).search - select.isDefined shouldBe true - val result = select.get - val query = result.query + |limit 100""".stripMargin) + val query = select.query val queryWithoutSource = query.substring(0, query.indexOf("_source") - 2) + "}" queryWithoutSource shouldBe """{ @@ -489,13 +499,11 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } it should "exclude fields from select" in { - val select = + val select: ElasticSearchRequest = SQLQuery( except - ).search - select.isDefined shouldBe true - val result = select.get - result.query shouldBe + ) + select.query shouldBe """ |{ | "query":{ @@ -509,7 +517,7 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { } it should "perform complex query" in { - val select = + val select: ElasticSearchRequest = SQLQuery( s"""SELECT | inner_products.name, @@ -548,11 +556,10 @@ class SQLQuerySpec extends AnyFlatSpec with Matchers { | ) |ORDER BY preparationTime ASC, nbOrders DESC |LIMIT 100""".stripMargin - ).minScore(1.0).search - select.isDefined shouldBe true - val result = select.get - println(result.query) - result.query shouldBe + ).minScore(1.0) + val query = select.query + println(query) + query shouldBe """ |{ | "query": { diff --git a/es6/testkit/build.sbt b/es6/testkit/build.sbt new file mode 100644 index 00000000..94b22a86 --- /dev/null +++ b/es6/testkit/build.sbt @@ -0,0 +1,42 @@ +import SoftClient4es.* + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-testkit" + +libraryDependencies ++= elastic4sTestkitDependencies(elasticSearchVersion.value) ++ Seq( + "org.apache.logging.log4j" % "log4j-api" % Versions.log4j, + // "org.apache.logging.log4j" % "log4j-slf4j-impl" % Versions.log4j, + "org.apache.logging.log4j" % "log4j-core" % Versions.log4j, + "app.softnetwork.persistence" %% "persistence-core-testkit" % Versions.genericPersistence, + "org.testcontainers" % "elasticsearch" % Versions.testContainers excludeAll (jacksonExclusions: _*) +) + +val testJavaOptions = { + val heapSize = sys.env.getOrElse("HEAP_SIZE", "1g") + val extraTestJavaArgs = Seq( + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED" + ).mkString(" ") + s"-Xmx$heapSize -Xss4m -XX:ReservedCodeCacheSize=128m -Dfile.encoding=UTF-8 $extraTestJavaArgs" + .split(" ") + .toSeq +} + +Test / javaOptions ++= testJavaOptions + +// Required by the Test container framework +Test / fork := true diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/es6/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala similarity index 100% rename from testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala rename to es6/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala diff --git a/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala b/es6/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala similarity index 100% rename from testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala rename to es6/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala diff --git a/es6/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala b/es6/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala new file mode 100644 index 00000000..7fff1f3f --- /dev/null +++ b/es6/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala @@ -0,0 +1,353 @@ +package app.softnetwork.elastic.scalatest + +import app.softnetwork.concurrent.scalatest.CompletionTestKit +import app.softnetwork.elastic.Softclient4es6TestkitBuildInfo +import com.sksamuel.elastic4s.{IndexAndTypes, Indexes} +import com.sksamuel.elastic4s.http.index.admin.RefreshIndexResponse +import com.sksamuel.elastic4s.http.{ElasticClient, ElasticDsl, ElasticProperties} +import com.typesafe.config.{Config, ConfigFactory} +import org.elasticsearch.ResourceAlreadyExistsException +import org.elasticsearch.transport.RemoteTransportException +import org.scalatest.{BeforeAndAfterAll, Suite} +import org.scalatest.matchers.{MatchResult, Matcher} +import org.slf4j.Logger + +import java.util.UUID +import scala.util.{Failure, Success, Try} + +/** Created by smanciot on 18/05/2021. + */ +trait ElasticTestKit extends ElasticDsl with CompletionTestKit with BeforeAndAfterAll { _: Suite => + + def log: Logger + + def elasticVersion: String = Softclient4es6TestkitBuildInfo.elasticVersion + + def elasticURL: String + + lazy val elasticConfig: Config = ConfigFactory + .parseString(elasticConfigAsString) + .withFallback(ConfigFactory.load("softnetwork-elastic.conf")) + + lazy val elasticConfigAsString: String = + s""" + |elastic { + | credentials { + | url = "$elasticURL" + | } + | multithreaded = false + | discovery-enabled = false + |} + |""".stripMargin + + lazy val clusterName: String = s"test-${UUID.randomUUID()}" + + lazy val elasticClient: ElasticClient = ElasticClient(ElasticProperties(elasticURL)) + + def start(): Unit = () + + def stop(): Unit = () + + override def beforeAll(): Unit = { + start() + elasticClient + .execute { + createIndexTemplate("all_templates", "*").settings( + Map("number_of_shards" -> 1, "number_of_replicas" -> 0) + ) + } + .complete() match { + case Success(_) => () + case Failure(f) => throw f + } + } + + override def afterAll(): Unit = { + elasticClient.close() + stop() + } + + // Rewriting methods from IndexMatchers in elastic4s with the ElasticClient + def haveCount(expectedCount: Int): Matcher[String] = + (left: String) => { + elasticClient.execute(search(left).size(0)).complete() match { + case Success(s) => + val count = s.result.totalHits + MatchResult( + count == expectedCount, + s"Index $left had count $count but expected $expectedCount", + s"Index $left had document count $expectedCount" + ) + case Failure(f) => throw f + } + } + + def containDoc(expectedId: String): Matcher[String] = + (left: String) => { + elasticClient.execute(get(expectedId).from(left)).complete() match { + case Success(s) => + val exists = s.result.exists + MatchResult( + exists, + s"Index $left did not contain expected document $expectedId", + s"Index $left contained document $expectedId" + ) + case Failure(f) => throw f + } + } + + def beCreated(): Matcher[String] = + (left: String) => { + elasticClient.execute(indexExists(left)).complete() match { + case Success(s) => + val exists = s.result.isExists + MatchResult( + exists, + s"Index $left did not exist", + s"Index $left exists" + ) + case Failure(f) => throw f + } + } + + def beEmpty(): Matcher[String] = + (left: String) => { + elasticClient.execute(search(left).size(0)).complete() match { + case Success(s) => + val count = s.result.totalHits + MatchResult( + count == 0, + s"Index $left was not empty", + s"Index $left was empty" + ) + case Failure(f) => throw f + } + } + + // Copy/paste methos HttpElasticSugar as it is not available yet + + // refresh all indexes + def refreshAll(): RefreshIndexResponse = refresh(Indexes.All) + + // refreshes all specified indexes + def refresh(indexes: Indexes): RefreshIndexResponse = { + elasticClient + .execute { + refreshIndex(indexes) + } + .complete() match { + case Success(s) => s.result + case Failure(f) => throw f + } + } + + def blockUntilGreen(): Unit = { + blockUntil("Expected cluster to have green status") { () => + elasticClient + .execute { + clusterHealth() + } + .complete() match { + case Success(s) => s.result.status.toUpperCase == "GREEN" + case Failure(f) => throw f + } + } + } + + def blockUntil(explain: String)(predicate: () => Boolean): Unit = { + blockUntil(explain, 16, 200)(predicate) + } + + def ensureIndexExists(index: String): Unit = { + elasticClient + .execute { + createIndex(index) + } + .complete() match { + case Success(_) => () + case Failure(f) => + f match { + case _: ResourceAlreadyExistsException => // Ok, ignore. + case _: RemoteTransportException => // Ok, ignore. + case other => throw other + } + } + } + + def doesIndexExists(name: String): Boolean = { + elasticClient + .execute { + indexExists(name) + } + .complete() match { + case Success(s) => s.result.isExists + case _ => false + } + } + + def isIndexOpened(name: String): Boolean = { + elasticClient + .execute { + indexStats(name) + } + .complete() match { + case Success(s) => + Try(s.result.indices.contains(name)) match { + case Success(_) => true + case Failure(_) => false + } + case _ => false + } + } + + def isIndexClosed(name: String): Boolean = { + doesIndexExists(name) && !isIndexOpened(name) + } + + def doesAliasExists(name: String): Boolean = { + elasticClient + .execute { + aliasExists(name) + } + .complete() match { + case Success(s) => s.result.isExists + case _ => false + } + } + + def deleteIndex(name: String): Unit = { + if (doesIndexExists(name)) { + elasticClient + .execute { + ElasticDsl.deleteIndex(name) + } + .complete() match { + case Success(_) => () + case Failure(f) => throw f + } + } + } + + def truncateIndex(index: String): Unit = { + deleteIndex(index) + ensureIndexExists(index) + blockUntilEmpty(index) + } + + def blockUntilDocumentExists(id: String, index: String, _type: String): Unit = { + blockUntil(s"Expected to find document $id") { () => + elasticClient + .execute { + get(id).from(index / _type) + } + .complete() match { + case Success(s) => s.result.exists + case _ => false + } + } + } + + def blockUntilCount(expected: Long, index: String): Unit = { + blockUntil(s"Expected count of $expected") { () => + elasticClient + .execute { + search(index).matchAllQuery().size(0) + } + .complete() match { + case Success(s) => expected <= s.result.totalHits + case Failure(f) => throw f + } + } + } + + def blockUntilCount(expected: Long, indexAndTypes: IndexAndTypes): Unit = { + blockUntil(s"Expected count of $expected") { () => + elasticClient + .execute { + searchWithType(indexAndTypes).matchAllQuery().size(0) + } + .complete() match { + case Success(s) => expected <= s.result.totalHits + case Failure(f) => throw f + } + } + } + + /** Will block until the given index and optional types have at least the given number of + * documents. + */ + def blockUntilCount(expected: Long, index: String, types: String*): Unit = { + blockUntil(s"Expected count of $expected") { () => + elasticClient + .execute { + searchWithType(index / types).matchAllQuery().size(0) + } + .complete() match { + case Success(s) => expected <= s.result.totalHits + case Failure(f) => throw f + } + } + } + + def blockUntilExactCount(expected: Long, index: String, types: String*): Unit = { + blockUntil(s"Expected count of $expected") { () => + elasticClient + .execute { + searchWithType(index / types).size(0) + } + .complete() match { + case Success(s) => expected == s.result.totalHits + case Failure(f) => throw f + } + } + } + + def blockUntilEmpty(index: String): Unit = { + blockUntil(s"Expected empty index $index") { () => + elasticClient + .execute { + search(Indexes(index)).size(0) + } + .complete() match { + case Success(s) => s.result.totalHits == 0 + case Failure(f) => throw f + } + } + } + + def blockUntilIndexExists(index: String): Unit = { + blockUntil(s"Expected exists index $index") { () => + doesIndexExists(index) + } + } + + def blockUntilIndexNotExists(index: String): Unit = { + blockUntil(s"Expected not exists index $index") { () => + !doesIndexExists(index) + } + } + + def blockUntilAliasExists(alias: String): Unit = { + blockUntil(s"Expected exists alias $alias") { () => + doesAliasExists(alias) + } + } + + def blockUntilDocumentHasVersion( + index: String, + _type: String, + id: String, + version: Long + ): Unit = { + blockUntil(s"Expected document $id to have version $version") { () => + elasticClient + .execute { + get(id).from(index / _type) + } + .complete() match { + case Success(s) => s.result.version == version + case Failure(f) => throw f + } + } + } +} diff --git a/es6/testkit/src/main/scala/app/softnetwork/elastic/scalatest/EmbeddedElasticTestKit.scala b/es6/testkit/src/main/scala/app/softnetwork/elastic/scalatest/EmbeddedElasticTestKit.scala new file mode 100644 index 00000000..3fe39859 --- /dev/null +++ b/es6/testkit/src/main/scala/app/softnetwork/elastic/scalatest/EmbeddedElasticTestKit.scala @@ -0,0 +1,37 @@ +package app.softnetwork.elastic.scalatest + +import org.scalatest.Suite +import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic +import pl.allegro.tech.embeddedelasticsearch.PopularProperties._ + +import java.net.ServerSocket +import java.util.UUID +import java.util.concurrent.TimeUnit +import scala.reflect.io.Path + +trait EmbeddedElasticTestKit extends ElasticTestKit { _: Suite => + + override lazy val elasticURL: String = s"http://127.0.0.1:${embeddedElastic.getHttpPort}" + + override def stop(): Unit = embeddedElastic.stop() + + private[this] def dynamicPort: Int = { + val socket = new ServerSocket(0) + val port = socket.getLocalPort + socket.close() + port + } + + private[this] val embeddedElastic: EmbeddedElastic = EmbeddedElastic + .builder() + .withElasticVersion(elasticVersion) + .withSetting(HTTP_PORT, dynamicPort) + .withSetting(CLUSTER_NAME, clusterName) + .withInstallationDirectory(Path(s"target/embedded-elastic-${UUID.randomUUID.toString}").jfile) + .withCleanInstallationDirectoryOnStop(true) + .withEsJavaOpts("-Xms128m -Xmx512m") + .withStartTimeout(2, TimeUnit.MINUTES) + .build() + .start() + +} diff --git a/es6/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala b/es6/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala new file mode 100644 index 00000000..4ad9dc53 --- /dev/null +++ b/es6/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala @@ -0,0 +1,15 @@ +package app.softnetwork.persistence.person + +import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.persistence.scalatest.InMemoryPersistenceTestKit + +trait ElasticPersonTestKit + extends PersonTestKit + with InMemoryPersistenceTestKit + with EmbeddedElasticTestKit { + + override def beforeAll(): Unit = { + super.beforeAll() + initAndJoinCluster() + } +} diff --git a/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala b/es6/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala similarity index 100% rename from testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala rename to es6/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala diff --git a/testkit/src/test/resources/application.conf b/es6/testkit/src/test/resources/application.conf similarity index 100% rename from testkit/src/test/resources/application.conf rename to es6/testkit/src/test/resources/application.conf diff --git a/testkit/src/test/resources/avatar.jpg b/es6/testkit/src/test/resources/avatar.jpg similarity index 100% rename from testkit/src/test/resources/avatar.jpg rename to es6/testkit/src/test/resources/avatar.jpg diff --git a/testkit/src/test/resources/avatar.pdf b/es6/testkit/src/test/resources/avatar.pdf similarity index 100% rename from testkit/src/test/resources/avatar.pdf rename to es6/testkit/src/test/resources/avatar.pdf diff --git a/testkit/src/test/resources/avatar.png b/es6/testkit/src/test/resources/avatar.png similarity index 100% rename from testkit/src/test/resources/avatar.png rename to es6/testkit/src/test/resources/avatar.png diff --git a/testkit/src/test/resources/mapping/person.mustache b/es6/testkit/src/test/resources/mapping/person.mustache similarity index 100% rename from testkit/src/test/resources/mapping/person.mustache rename to es6/testkit/src/test/resources/mapping/person.mustache diff --git a/es6/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/es6/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala new file mode 100644 index 00000000..648f99ff --- /dev/null +++ b/es6/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -0,0 +1,907 @@ +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.sql.SQLQuery +import com.fasterxml.jackson.core.JsonParseException +import com.sksamuel.elastic4s.searches.queries.matches.MatchAllQuery +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import app.softnetwork.persistence._ +import app.softnetwork.serialization._ +import app.softnetwork.elastic.model._ +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.elastic.scalatest.EmbeddedElasticTestKit +import app.softnetwork.persistence.person.model.Person +import com.google.gson.JsonParser +import com.sksamuel.exts.Logging +import org.json4s.Formats +import org.slf4j.{Logger, LoggerFactory} + +import _root_.java.io.ByteArrayInputStream +import _root_.java.nio.file.{Files, Paths} +import _root_.java.time.format.DateTimeFormatter +import _root_.java.util.concurrent.TimeUnit +import _root_.java.util.UUID +import java.time.{LocalDate, LocalDateTime, ZoneOffset} +import scala.concurrent.{Await, ExecutionContextExecutor} +import scala.concurrent.duration.Duration +import scala.util.{Failure, Success} + +/** Created by smanciot on 28/06/2018. + */ +trait ElasticClientSpec + extends AnyFlatSpecLike + with EmbeddedElasticTestKit + with Matchers + with Logging { + + lazy val log: Logger = LoggerFactory getLogger getClass.getName + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + + implicit val formats: Formats = commonFormats + + def pClient: ElasticProvider[Person] with ElasticClientApi + def sClient: ElasticProvider[Sample] with ElasticClientApi + def bClient: ElasticProvider[Binary] with ElasticClientApi + def parentClient: ElasticProvider[Parent] with ElasticClientApi + + import scala.language.implicitConversions + + implicit def toSQLQuery(sqlQuery: String): SQLQuery = SQLQuery(sqlQuery) + + override def beforeAll(): Unit = { + super.beforeAll() + pClient.createIndex("person") + } + + override def afterAll(): Unit = { + Await.result(system.terminate(), Duration(30, TimeUnit.SECONDS)) + super.afterAll() + } + + val persons: List[String] = List( + """ { "uuid": "A12", "name": "Homer Simpson", "birthDate": "1967-11-21", "childrenCount": 0} """, + """ { "uuid": "A14", "name": "Moe Szyslak", "birthDate": "1967-11-21", "childrenCount": 0} """, + """ { "uuid": "A16", "name": "Barney Gumble", "birthDate": "1969-05-09", "childrenCount": 0} """ + ) + + private val personsWithUpsert = + persons :+ """ { "uuid": "A16", "name": "Barney Gumble2", "birthDate": "1969-05-09", "children": [{ "parentId": "A16", "name": "Steve Gumble", "birthDate": "1999-05-09"}, { "parentId": "A16", "name": "Josh Gumble", "birthDate": "2002-05-09"}], "childrenCount": 2 } """ + + val children: List[String] = List( + """ { "parentId": "A16", "name": "Steve Gumble", "birthDate": "1999-05-09"} """, + """ { "parentId": "A16", "name": "Josh Gumble", "birthDate": "1999-05-09"} """ + ) + + "Creating an index and then delete it" should "work fine" in { + pClient.createIndex("create_delete") + blockUntilIndexExists("create_delete") + "create_delete" should beCreated + + pClient.deleteIndex("create_delete") + blockUntilIndexNotExists("create_delete") + "create_delete" should not(beCreated()) + } + + "Adding an alias and then removing it" should "work" in { + pClient.addAlias("person", "person_alias") + + doesAliasExists("person_alias") shouldBe true + + pClient.removeAlias("person", "person_alias") + + doesAliasExists("person_alias") shouldBe false + } + + private def settings: Map[String, String] = { + elasticClient + .execute { + getSettings("person") + } + .complete() match { + case Success(s) => s.result.settingsForIndex("person") + case Failure(f) => throw f + } + } + + "Toggle refresh" should "work" in { + pClient.toggleRefresh("person", enable = false) + new JsonParser() + .parse(pClient.loadSettings("person")) + .getAsJsonObject + .get("person") + .getAsJsonObject + .get("settings") + .getAsJsonObject + .get("index") + .getAsJsonObject + .get("refresh_interval") + .getAsString shouldBe "-1" + // settings.getOrElse("index.refresh_interval", "") shouldBe "-1" + + pClient.toggleRefresh("person", enable = true) + // settings.getOrElse("index.refresh_interval", "") shouldBe "1s" + new JsonParser() + .parse(pClient.loadSettings("person")) + .getAsJsonObject + .get("person") + .getAsJsonObject + .get("settings") + .getAsJsonObject + .get("index") + .getAsJsonObject + .get("refresh_interval") + .getAsString shouldBe "1s" + } + + "Opening an index and then closing it" should "work" in { + pClient.openIndex("person") + + isIndexOpened("person") shouldBe true + + pClient.closeIndex("person") + isIndexClosed("person") shouldBe true + } + + "Updating number of replicas" should "work" in { + pClient.setReplicas("person", 3) + settings.getOrElse("index.number_of_replicas", "") shouldBe "3" + + pClient.setReplicas("person", 0) + settings.getOrElse("index.number_of_replicas", "") shouldBe "0" + } + + "Setting a mapping" should "work" in { + pClient.createIndex("person_mapping") + blockUntilIndexExists("person_mapping") + "person_mapping" should beCreated + + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "text", + | "analyzer": "ngram_analyzer", + | "search_analyzer": "search_analyzer", + | "fields": { + | "raw": { + | "type": "keyword" + | }, + | "fr": { + | "type": "text", + | "analyzer": "french" + | } + | } + | } + | } + |}""".stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + pClient.setMapping("person_mapping", mapping) shouldBe true + + val properties = pClient.getMappingProperties("person_mapping") + logger.info(s"properties: $properties") + MappingComparator.isMappingDifferent( + properties, + mapping + ) shouldBe false + + implicit val bulkOptions: BulkOptions = BulkOptions("person_mapping", "_doc", 1000) + val indices = pClient.bulk[String](persons.iterator, identity, Some("uuid"), None, None) + refresh(indices) + pClient.flush("person_mapping") + + indices should contain only "person_mapping" + + blockUntilCount(3, "person_mapping") + + "person_mapping" should haveCount(3) + + pClient.search[Person]("select * from person_mapping") match { + case r if r.size == 3 => + r.map(_.uuid) should contain allOf ("A12", "A14", "A16") + case other => fail(other.toString) + } + + pClient.search[Person]("select * from person_mapping where uuid = 'A16'") match { + case r if r.size == 1 => + r.map(_.uuid) should contain only "A16" + case other => fail(other.toString) + } + + pClient.search[Person]("select * from person_mapping where match(name, 'gum')") match { + case r if r.size == 1 => + r.map(_.uuid) should contain only "A16" + case other => fail(other.toString) + } + + pClient.search[Person]( + "select * from person_mapping where uuid <> 'A16' and match(name, 'gum')" + ) match { + case r if r.isEmpty => + case other => fail(other.toString) + } + } + + "Updating a mapping" should "work" in { + val mapping = + """{ + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + pClient.updateMapping("person_migration", mapping) shouldBe true + blockUntilIndexExists("person_migration") + "person_migration" should beCreated + + implicit val bulkOptions: BulkOptions = BulkOptions("person_migration", "_doc", 1000) + val indices = pClient.bulk[String](persons.iterator, identity, Some("uuid"), None, None) + refresh(indices) + pClient.flush("person_migration") + + indices should contain only "person_migration" + + blockUntilCount(3, "person_migration") + + "person_migration" should haveCount(3) + + pClient.search[Person]("select * from person_migration where match(name, 'gum')") match { + case r if r.isEmpty => + case other => fail(other.toString) + } + + val newMapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "text", + | "analyzer": "ngram_analyzer", + | "search_analyzer": "search_analyzer", + | "fields": { + | "raw": { + | "type": "keyword" + | }, + | "fr": { + | "type": "text", + | "analyzer": "french" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + pClient.shouldUpdateMapping("person_migration", newMapping) shouldBe true + pClient.updateMapping("person_migration", newMapping) shouldBe true + + pClient.search[Person]("select * from person_migration where match(name, 'gum')") match { + case r if r.size == 1 => + r.map(_.uuid) should contain only "A16" + case other => fail(other.toString) + } + + } + + "Bulk index valid json without id key and suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person1", "person", 2) + val indices = pClient.bulk[String](persons.iterator, identity, None, None, None) + + indices should contain only "person1" + + blockUntilCount(3, "person1") + + "person1" should haveCount(3) + + val response = elasticClient + .execute { + search("person1").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id should not be h.sourceField("uuid") + } + + response.result.hits.hits + .map( + _.sourceField("name") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble") + } + + "Bulk index valid json with an id key but no suffix key" should "work" in { + elasticClient.execute { + createIndex("person2").mappings( + mapping("child").fields(textField("name").index(false)).parent("person") + ) + } + + implicit val bulkOptions: BulkOptions = BulkOptions("person2", "person", 1000) + val indices = pClient.bulk[String](persons.iterator, identity, Some("uuid"), None, None) + refresh(indices) + pClient.flush("person2") + + indices should contain only "person2" + + blockUntilCount(3, "person2") + + "person2" should haveCount(3) + + val response = elasticClient + .execute { + search("person2").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceField("uuid") + } + + response.result.hits.hits + .map( + _.sourceField("name") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble") + + } + + "Bulk index valid json with an id key and a suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person", "person", 1000) + val indices = + pClient.bulk[String](persons.iterator, identity, Some("uuid"), Some("birthDate"), None, None) + refresh(indices) + + indices should contain allOf ("person-1967-11-21", "person-1969-05-09") + + blockUntilCount(2, "person-1967-11-21") + blockUntilCount(1, "person-1969-05-09") + + "person-1967-11-21" should haveCount(2) + "person-1969-05-09" should haveCount(1) + + val response = elasticClient + .execute { + search("person-1967-11-21", "person-1969-05-09").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceField("uuid") + } + + response.result.hits.hits + .map( + _.sourceField("name") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble") + } + + "Bulk index invalid json with an id key and a suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person_error", "person", 1000) + intercept[JsonParseException] { + val invalidJson = persons :+ "fail" + pClient.bulk[String](invalidJson.iterator, identity, None, None, None) + } + } + + "Bulk upsert valid json with an id key but no suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person4", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person4" + + blockUntilCount(3, "person4") + + "person4" should haveCount(3) + + val response = elasticClient + .execute { + search("person4").query(MatchAllQuery()) + } + .complete() + + logger.info(s"response: ${response.result.hits.hits.mkString("\n")}") + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceAsMap.getOrElse("uuid", "") + } + + response.result.hits.hits + .map( + _.sourceAsMap.getOrElse("name", "") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble2") + } + + "Bulk upsert valid json with an id key and a suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person5", "person", 1000) + val indices = pClient.bulk[String]( + personsWithUpsert.iterator, + identity, + Some("uuid"), + Some("birthDate"), + None, + Some(true) + ) + refresh(indices) + + indices should contain allOf ("person5-1967-11-21", "person5-1969-05-09") + + blockUntilCount(2, "person5-1967-11-21") + blockUntilCount(1, "person5-1969-05-09") + + "person5-1967-11-21" should haveCount(2) + "person5-1969-05-09" should haveCount(1) + + val response = elasticClient + .execute { + search("person5-1967-11-21", "person5-1969-05-09").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceAsMap.getOrElse("uuid", "") + } + + response.result.hits.hits + .map( + _.sourceAsMap.getOrElse("name", "") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble2") + } + + "Count" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person6", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person6" + + blockUntilCount(3, "person6") + + "person6" should haveCount(3) + + import scala.collection.immutable.Seq + + pClient + .count(JSONQuery("{}", Seq[String]("person6"), Seq[String]())) + .getOrElse(0d) + .toInt should ===(3) + + pClient.countAsync(JSONQuery("{}", Seq[String]("person6"), Seq[String]())).complete() match { + case Success(s) => s.getOrElse(0d).toInt should ===(3) + case Failure(f) => fail(f.getMessage) + } + } + + "Search" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person7", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person7" + + blockUntilCount(3, "person7") + + "person7" should haveCount(3) + + val r1 = pClient.search[Person]("select * from person7") + r1.size should ===(3) + r1.map(_.uuid) should contain allOf ("A12", "A14", "A16") + + pClient.searchAsync[Person]("select * from person7") onComplete { + case Success(r) => + r.size should ===(3) + r.map(_.uuid) should contain allOf ("A12", "A14", "A16") + case Failure(f) => fail(f.getMessage) + } + + val r2 = pClient.search[Person]("select * from person7 where _id=\"A16\"") + r2.size should ===(1) + r2.map(_.uuid) should contain("A16") + + pClient.searchAsync[Person]("select * from person7 where _id=\"A16\"") onComplete { + case Success(r) => + r.size should ===(1) + r.map(_.uuid) should contain("A16") + case Failure(f) => fail(f.getMessage) + } + } + + "Get all" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person8", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person8" + + blockUntilCount(3, "person8") + + "person8" should haveCount(3) + + val response = pClient.search[Person]("select * from person8") + + response.size should ===(3) + + } + + "Get" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person9", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person9" + + blockUntilCount(3, "person9") + + "person9" should haveCount(3) + + val response = pClient.get[Person]("A16", Some("person9")) + + response.isDefined shouldBe true + response.get.uuid shouldBe "A16" + + pClient.getAsync[Person]("A16", Some("person9")).complete() match { + case Success(r) => + r.isDefined shouldBe true + r.get.uuid shouldBe "A16" + case Failure(f) => fail(f.getMessage) + } + } + + "Index" should "work" in { + val uuid = UUID.randomUUID().toString + val sample = Sample(uuid) + val result = sClient.index(sample) + result shouldBe true + + sClient.indexAsync(sample).complete() match { + case Success(r) => r shouldBe true + case Failure(f) => fail(f.getMessage) + } + + val result2 = sClient.get[Sample](uuid) + result2 match { + case Some(r) => + r.uuid shouldBe uuid + case _ => + fail("Sample not found") + } + } + + "Update" should "work" in { + val uuid = UUID.randomUUID().toString + val sample = Sample(uuid) + val result = sClient.update(sample) + result shouldBe true + + sClient.updateAsync(sample).complete() match { + case Success(r) => r shouldBe true + case Failure(f) => fail(f.getMessage) + } + + val result2 = sClient.get[Sample](uuid) + result2 match { + case Some(r) => + r.uuid shouldBe uuid + case _ => + fail("Sample not found") + } + } + + "Delete" should "work" in { + val uuid = UUID.randomUUID().toString + val sample = Sample(uuid) + val result = sClient.index(sample) + result shouldBe true + + val result2 = sClient.delete(sample.uuid, Some("sample")) + result2 shouldBe true + + /*FIXME sClient.deleteAsync(sample.uuid, Some("sample")).complete() match { + case Success(r) => r shouldBe true + case Failure(f) => fail(f.getMessage) + }*/ + + val result3 = sClient.get[Sample](uuid) + result3.isEmpty shouldBe true + } + + "Index binary data" should "work" in { + bClient.createIndex("binaries") shouldBe true + val mapping = + """{ + | "properties": { + | "uuid": { + | "type": "keyword", + | "index": true + | }, + | "createdDate": { + | "type": "date" + | }, + | "lastUpdated": { + | "type": "date" + | }, + | "content": { + | "type": "binary" + | }, + | "md5": { + | "type": "keyword" + | } + | } + |} + """.stripMargin + bClient.setMapping("binaries", mapping) shouldBe true + for (uuid <- Seq("png", "jpg", "pdf")) { + val path = + Paths.get(Thread.currentThread().getContextClassLoader.getResource(s"avatar.$uuid").getPath) + import app.softnetwork.utils.ImageTools._ + import app.softnetwork.utils.HashTools._ + import app.softnetwork.utils.Base64Tools._ + val encoded = encodeImageBase64(path).getOrElse("") + val binary = Binary( + uuid, + content = encoded, + md5 = hashStream(new ByteArrayInputStream(decodeBase64(encoded))).getOrElse("") + ) + bClient.index(binary) shouldBe true + bClient.get[Binary](uuid) match { + case Some(result) => + val decoded = decodeBase64(result.content) + val out = Paths.get(s"/tmp/${path.getFileName}") + val fos = Files.newOutputStream(out) + fos.write(decoded) + fos.close() + hashFile(out).getOrElse("") shouldBe binary.md5 + case _ => fail("no result found for \"" + uuid + "\"") + } + } + } + + "Aggregations" should "work" in { + pClient.createIndex("person10") shouldBe true + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "keyword" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + logger.info(s"mapping: $mapping") + pClient.setMapping("person10", mapping) shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person10", "_doc", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + pClient.flush("person10") + + indices should contain only "person10" + + blockUntilCount(3, "person10") + + "person10" should haveCount(3) + + pClient.get[Person]("A16", Some("person10")) match { + case Some(p) => + p.uuid shouldBe "A16" + p.birthDate shouldBe "1969-05-09" + case None => fail("Person A16 not found") + } + + // test distinct count aggregation + pClient + .aggregate( + "select count(distinct p.uuid) as c from person10 p" + ) + .complete() match { + case Success(s) => s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===(3d) + case Failure(f) => fail(f.getMessage) + } + + // test count aggregation + pClient.aggregate("select count(p.uuid) as c from person10 p").complete() match { + case Success(s) => s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===(3d) + case Failure(f) => fail(f.getMessage) + } + + // test max aggregation on date field + pClient.aggregate("select max(p.birthDate) as c from person10 p").complete() match { + case Success(s) => + s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===( + LocalDate.parse("1969-05-09").toEpochDay.toDouble * 3600 * 24 * 1000 + ) + case Failure(f) => fail(f.getMessage) + } + + // test min aggregation on date field + pClient.aggregate("select min(p.birthDate) as c from person10 p").complete() match { + case Success(s) => + s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===( + LocalDate.parse("1967-11-21").toEpochDay.toDouble * 3600 * 24 * 1000 + ) + case Failure(f) => fail(f.getMessage) + } + + // test avg aggregation on date field + pClient.aggregate("select avg(p.birthDate) as c from person10 p").complete() match { + case Success(s) => + s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===( + LocalDateTime + .parse("1968-05-17T08:00:00.000Z", DateTimeFormatter.ISO_OFFSET_DATE_TIME) + .toInstant(ZoneOffset.UTC) + .toEpochMilli + ) + case Failure(f) => fail(f.getMessage) + } + + // test sum aggregation on integer field + pClient + .aggregate( + "select sum(p.childrenCount) as c from person10 p" + ) + .complete() match { + case Success(s) => + s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===(2d) + case Failure(f) => fail(f.getMessage) + } + + } + + "Nested queries" should "work" in { + parentClient.createIndex("parent") shouldBe true + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "keyword" + | }, + | "createdDate": { + | "type": "date", + | "null_value": "1970-01-01" + | }, + | "lastUpdated": { + | "type": "date", + | "null_value": "1970-01-01" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + logger.info(s"mapping: $mapping") + parentClient.setMapping("parent", mapping) shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("parent", "_doc", 1000) + val indices = + parentClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + parentClient.flush("parent") + parentClient.refresh("parent") + + indices should contain only "parent" + + blockUntilCount(3, "parent") + + "parent" should haveCount(3) + + val parents = parentClient.search[Parent]("select * from parent") + assert(parents.size == 3) + + val results = parentClient.searchWithInnerHits[Parent, Child]( + """SELECT + | p.uuid, + | p.name, + | p.birthDate, + | p.children, + | inner_children.name, + | inner_children.birthDate + |FROM + | parent as p, + | UNNEST(p.children) as inner_children + |WHERE + | inner_children.name is not null AND p.uuid = 'A16' + |""".stripMargin, + "inner_children" + ) + results.size shouldBe 1 + val result = results.head + result._1.uuid shouldBe "A16" + result._1.children.size shouldBe 2 + result._2.size shouldBe 2 + result._2.map(_.name) should contain allOf ("Steve Gumble", "Josh Gumble") + result._2.map( + _.birthDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) + ) should contain allOf ("1999-05-09", "2002-05-09") + result._2.map(_.parentId) should contain only "A16" + } +} diff --git a/es6/testkit/src/test/scala/app/softnetwork/elastic/client/JestClientSpec.scala b/es6/testkit/src/test/scala/app/softnetwork/elastic/client/JestClientSpec.scala new file mode 100644 index 00000000..46ce98c9 --- /dev/null +++ b/es6/testkit/src/test/scala/app/softnetwork/elastic/client/JestClientSpec.scala @@ -0,0 +1,28 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.JestProviders.{ + BinaryProvider, + ParentProvider, + PersonProvider, + SampleProvider +} +import app.softnetwork.elastic.model.{Binary, Parent, Sample} +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.persistence.person.model.Person + +class JestClientSpec extends ElasticClientSpec { + + lazy val pClient: ElasticProvider[Person] with ElasticClientApi = new PersonProvider( + elasticConfig + ) + lazy val sClient: ElasticProvider[Sample] with ElasticClientApi = new SampleProvider( + elasticConfig + ) + lazy val bClient: ElasticProvider[Binary] with ElasticClientApi = new BinaryProvider( + elasticConfig + ) + + override def parentClient: ElasticProvider[Parent] with ElasticClientApi = new ParentProvider( + elasticConfig + ) +} diff --git a/es6/testkit/src/test/scala/app/softnetwork/elastic/client/JestProviders.scala b/es6/testkit/src/test/scala/app/softnetwork/elastic/client/JestProviders.scala new file mode 100644 index 00000000..b366eba5 --- /dev/null +++ b/es6/testkit/src/test/scala/app/softnetwork/elastic/client/JestProviders.scala @@ -0,0 +1,47 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.model.{Binary, Parent, Sample} +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.person.model.Person +import app.softnetwork.persistence.query.JestProvider +import com.typesafe.config.Config +import io.searchbox.client.JestClient + +object JestProviders { + + class PersonProvider(es: Config) extends JestProvider[Person] with ManifestWrapper[Person] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val jestClient: JestClient = + apply(elasticConfig.credentials, elasticConfig.multithreaded) + } + + class SampleProvider(es: Config) extends JestProvider[Sample] with ManifestWrapper[Sample] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val jestClient: JestClient = + apply(elasticConfig.credentials, elasticConfig.multithreaded) + } + + class BinaryProvider(es: Config) extends JestProvider[Binary] with ManifestWrapper[Binary] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val jestClient: JestClient = + apply(elasticConfig.credentials, elasticConfig.multithreaded) + } + + class ParentProvider(es: Config) extends JestProvider[Parent] with ManifestWrapper[Parent] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val jestClient: JestClient = + apply(elasticConfig.credentials, elasticConfig.multithreaded) + } +} diff --git a/es6/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSpec.scala b/es6/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSpec.scala new file mode 100644 index 00000000..b8aeba67 --- /dev/null +++ b/es6/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSpec.scala @@ -0,0 +1,28 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.RestHighLevelProviders.{ + BinaryProvider, + ParentProvider, + PersonProvider, + SampleProvider +} +import app.softnetwork.elastic.model.{Binary, Parent, Sample} +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.persistence.person.model.Person + +class RestHighLevelClientSpec extends ElasticClientSpec { + + lazy val pClient: ElasticProvider[Person] with ElasticClientApi = new PersonProvider( + elasticConfig + ) + lazy val sClient: ElasticProvider[Sample] with ElasticClientApi = new SampleProvider( + elasticConfig + ) + lazy val bClient: ElasticProvider[Binary] with ElasticClientApi = new BinaryProvider( + elasticConfig + ) + + override def parentClient: ElasticProvider[Parent] with ElasticClientApi = new ParentProvider( + elasticConfig + ) +} diff --git a/es6/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelProviders.scala b/es6/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelProviders.scala new file mode 100644 index 00000000..5efe18be --- /dev/null +++ b/es6/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelProviders.scala @@ -0,0 +1,51 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.model.{Binary, Parent, Sample} +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.person.model.Person +import app.softnetwork.persistence.query.RestHighLevelClientProvider +import com.typesafe.config.Config +import org.elasticsearch.client.RestHighLevelClient + +object RestHighLevelProviders { + + class PersonProvider(es: Config) + extends RestHighLevelClientProvider[Person] + with ManifestWrapper[Person] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val restHighLevelClient: RestHighLevelClient = apply() + } + + class SampleProvider(es: Config) + extends RestHighLevelClientProvider[Sample] + with ManifestWrapper[Sample] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val restHighLevelClient: RestHighLevelClient = apply() + } + + class BinaryProvider(es: Config) + extends RestHighLevelClientProvider[Binary] + with ManifestWrapper[Binary] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val restHighLevelClient: RestHighLevelClient = apply() + } + + class ParentProvider(es: Config) + extends RestHighLevelClientProvider[Parent] + with ManifestWrapper[Parent] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val restHighLevelClient: RestHighLevelClient = apply() + } +} diff --git a/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala b/es6/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala similarity index 100% rename from testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala rename to es6/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala diff --git a/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala b/es6/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala similarity index 100% rename from testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala rename to es6/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala diff --git a/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala b/es6/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala similarity index 100% rename from testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala rename to es6/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala diff --git a/es6/testkit/src/test/scala/app/softnetwork/persistence/person/JestClientPersonHandlerSpec.scala b/es6/testkit/src/test/scala/app/softnetwork/persistence/person/JestClientPersonHandlerSpec.scala new file mode 100644 index 00000000..c768a2b5 --- /dev/null +++ b/es6/testkit/src/test/scala/app/softnetwork/persistence/person/JestClientPersonHandlerSpec.scala @@ -0,0 +1,35 @@ +package app.softnetwork.persistence.person + +import akka.actor.typed.ActorSystem +import app.softnetwork.elastic.client.jest.JestClientApi +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.person.model.Person +import app.softnetwork.persistence.person.query.PersonToExternalProcessorStream +import app.softnetwork.persistence.query.{ + ExternalPersistenceProvider, + PersonToElasticProcessorStream +} +import com.typesafe.config.Config +import org.slf4j.{Logger, LoggerFactory} + +class JestClientPersonHandlerSpec extends ElasticPersonTestKit { + + override def externalPersistenceProvider: ExternalPersistenceProvider[Person] = + new ElasticProvider[Person] with JestClientApi with ManifestWrapper[Person] { + override protected val manifestWrapper: ManifestW = ManifestW() + override lazy val config: Config = JestClientPersonHandlerSpec.this.elasticConfig + } + + override def person2ExternalProcessorStream: ActorSystem[_] => PersonToExternalProcessorStream = + sys => + new PersonToElasticProcessorStream with JestClientApi { + override val forTests: Boolean = true + override protected val manifestWrapper: ManifestW = ManifestW() + override implicit def system: ActorSystem[_] = sys + override def log: Logger = LoggerFactory getLogger getClass.getName + override lazy val config: Config = JestClientPersonHandlerSpec.this.elasticConfig + } + + override def log: Logger = LoggerFactory getLogger getClass.getName +} diff --git a/es6/testkit/src/test/scala/app/softnetwork/persistence/person/RestHighLevelClientPersonHandlerSpec.scala b/es6/testkit/src/test/scala/app/softnetwork/persistence/person/RestHighLevelClientPersonHandlerSpec.scala new file mode 100644 index 00000000..fbce2cf8 --- /dev/null +++ b/es6/testkit/src/test/scala/app/softnetwork/persistence/person/RestHighLevelClientPersonHandlerSpec.scala @@ -0,0 +1,35 @@ +package app.softnetwork.persistence.person + +import akka.actor.typed.ActorSystem +import app.softnetwork.elastic.client.rest.RestHighLevelClientApi +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.person.model.Person +import app.softnetwork.persistence.person.query.PersonToExternalProcessorStream +import app.softnetwork.persistence.query.{ + ExternalPersistenceProvider, + PersonToElasticProcessorStream +} +import com.typesafe.config.Config +import org.slf4j.{Logger, LoggerFactory} + +class RestHighLevelClientPersonHandlerSpec extends ElasticPersonTestKit { + + override def externalPersistenceProvider: ExternalPersistenceProvider[Person] = + new ElasticProvider[Person] with RestHighLevelClientApi with ManifestWrapper[Person] { + override protected val manifestWrapper: ManifestW = ManifestW() + override lazy val config: Config = RestHighLevelClientPersonHandlerSpec.this.elasticConfig + } + + override def person2ExternalProcessorStream: ActorSystem[_] => PersonToExternalProcessorStream = + sys => + new PersonToElasticProcessorStream with RestHighLevelClientApi { + override val forTests: Boolean = true + override protected val manifestWrapper: ManifestW = ManifestW() + override implicit def system: ActorSystem[_] = sys + override def log: Logger = LoggerFactory getLogger getClass.getName + override lazy val config: Config = RestHighLevelClientPersonHandlerSpec.this.elasticConfig + } + + override def log: Logger = LoggerFactory getLogger getClass.getName +} diff --git a/es7/build.sbt b/es7/build.sbt new file mode 100644 index 00000000..28252611 --- /dev/null +++ b/es7/build.sbt @@ -0,0 +1,7 @@ +import SoftClient4es.* +organization := "app.softnetwork.elastic" +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}" +publish / skip := true +Compile / sources := Nil +Test / sources := Nil + diff --git a/es7/rest/build.sbt b/es7/rest/build.sbt new file mode 100644 index 00000000..1c65c2ce --- /dev/null +++ b/es7/rest/build.sbt @@ -0,0 +1,8 @@ +import SoftClient4es.* + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-rest-client" + +libraryDependencies ++= restClientDependencies(elasticSearchVersion.value) ++ + elastic4sDependencies(elasticSearchVersion.value) diff --git a/es7/rest/persistence/build.sbt b/es7/rest/persistence/build.sbt new file mode 100644 index 00000000..e1dc23b4 --- /dev/null +++ b/es7/rest/persistence/build.sbt @@ -0,0 +1,6 @@ +import SoftClient4es.{elasticSearchMajorVersion, elasticSearchVersion} + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-rest-persistence" + diff --git a/es7/rest/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/RestHighLevelClientProvider.scala b/es7/rest/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/RestHighLevelClientProvider.scala new file mode 100644 index 00000000..b76c13b4 --- /dev/null +++ b/es7/rest/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/RestHighLevelClientProvider.scala @@ -0,0 +1,12 @@ +package app.softnetwork.elastic.persistence.query + +import app.softnetwork.elastic.client.rest.RestHighLevelClientApi +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.model.Timestamped + +trait RestHighLevelClientProvider[T <: Timestamped] + extends ElasticProvider[T] + with RestHighLevelClientApi { + _: ManifestWrapper[T] => + +} diff --git a/es7/rest/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/State2ElasticProcessorStreamWithRestProvider.scala b/es7/rest/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/State2ElasticProcessorStreamWithRestProvider.scala new file mode 100644 index 00000000..3f79bb07 --- /dev/null +++ b/es7/rest/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/State2ElasticProcessorStreamWithRestProvider.scala @@ -0,0 +1,9 @@ +package app.softnetwork.elastic.persistence.query + +import app.softnetwork.persistence.message.CrudEvent +import app.softnetwork.persistence.model.Timestamped +import app.softnetwork.persistence.query.{JournalProvider, OffsetProvider} + +trait State2ElasticProcessorStreamWithRestProvider[T <: Timestamped, E <: CrudEvent] + extends State2ElasticProcessorStream[T, E] + with RestHighLevelClientProvider[T] { _: JournalProvider with OffsetProvider => } diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala new file mode 100644 index 00000000..92410fa3 --- /dev/null +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -0,0 +1,910 @@ +package app.softnetwork.elastic.client.rest + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Flow +import app.softnetwork.elastic.client._ +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.{client, sql} +import app.softnetwork.persistence.model.Timestamped +import app.softnetwork.serialization.serialization +import com.google.gson.JsonParser +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest +import org.elasticsearch.action.admin.indices.flush.FlushRequest +import org.elasticsearch.action.admin.indices.open.OpenIndexRequest +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest +import org.elasticsearch.action.bulk.{BulkItemResponse, BulkRequest, BulkResponse} +import org.elasticsearch.action.delete.{DeleteRequest, DeleteResponse} +import org.elasticsearch.action.get.{GetRequest, GetResponse} +import org.elasticsearch.action.index.{IndexRequest, IndexResponse} +import org.elasticsearch.action.search.{MultiSearchRequest, SearchRequest, SearchResponse} +import org.elasticsearch.action.update.{UpdateRequest, UpdateResponse} +import org.elasticsearch.action.{ActionListener, DocWriteRequest} +import org.elasticsearch.client.{Request, RequestOptions} +import org.elasticsearch.client.core.{CountRequest, CountResponse} +import org.elasticsearch.client.indices.{ + CloseIndexRequest, + CreateIndexRequest, + GetIndexRequest, + GetMappingsRequest, + PutMappingRequest +} +import org.elasticsearch.common.io.stream.InputStreamStreamInput +import org.elasticsearch.xcontent.{DeprecationHandler, XContentType} +import org.elasticsearch.rest.RestStatus +import org.elasticsearch.search.aggregations.bucket.filter.Filter +import org.elasticsearch.search.aggregations.bucket.nested.Nested +import org.elasticsearch.search.aggregations.metrics.{Avg, Cardinality, Max, Min, Sum, ValueCount} +import org.elasticsearch.search.builder.SearchSourceBuilder +import org.json4s.Formats + +import java.io.ByteArrayInputStream +import scala.collection.JavaConverters.mapAsScalaMapConverter +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.language.implicitConversions +import scala.util.{Failure, Success, Try} + +trait RestHighLevelClientApi + extends ElasticClientApi + with RestHighLevelClientIndicesApi + with RestHighLevelClientAliasApi + with RestHighLevelClientSettingsApi + with RestHighLevelClientMappingApi + with RestHighLevelClientRefreshApi + with RestHighLevelClientFlushApi + with RestHighLevelClientCountApi + with RestHighLevelClientSingleValueAggregateApi + with RestHighLevelClientIndexApi + with RestHighLevelClientUpdateApi + with RestHighLevelClientDeleteApi + with RestHighLevelClientGetApi + with RestHighLevelClientSearchApi + with RestHighLevelClientBulkApi + +trait RestHighLevelClientIndicesApi extends IndicesApi with RestHighLevelClientCompanion { + override def createIndex(index: String, settings: String): Boolean = { + tryOrElse( + apply() + .indices() + .create( + new CreateIndexRequest(index) + .settings(settings, XContentType.JSON), + RequestOptions.DEFAULT + ) + .isAcknowledged, + false + )(logger) + } + + override def deleteIndex(index: String): Boolean = { + tryOrElse( + apply() + .indices() + .delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT) + .isAcknowledged, + false + )(logger) + } + + override def openIndex(index: String): Boolean = { + tryOrElse( + apply().indices().open(new OpenIndexRequest(index), RequestOptions.DEFAULT).isAcknowledged, + false + )(logger) + } + + override def closeIndex(index: String): Boolean = { + tryOrElse( + apply().indices().close(new CloseIndexRequest(index), RequestOptions.DEFAULT).isAcknowledged, + false + )(logger) + } + + /** Reindex from source index to target index. + * + * @param sourceIndex + * - the name of the source index + * @param targetIndex + * - the name of the target index + * @param refresh + * - true to refresh the target index after reindexing, false otherwise + * @return + * true if the reindexing was successful, false otherwise + */ + override def reindex(sourceIndex: String, targetIndex: String, refresh: Boolean): Boolean = { + val request = new Request("POST", "/_reindex?refresh=true") + request.setJsonEntity( + s""" + |{ + | "source": { + | "index": "$sourceIndex" + | }, + | "dest": { + | "index": "$targetIndex" + | } + |} + """.stripMargin + ) + tryOrElse( + apply().getLowLevelClient.performRequest(request).getStatusLine.getStatusCode < 400, + false + )(logger) + } + + /** Check if an index exists. + * + * @param index + * - the name of the index to check + * @return + * true if the index exists, false otherwise + */ + override def indexExists(index: String): Boolean = { + tryOrElse( + apply().indices().exists(new GetIndexRequest(index), RequestOptions.DEFAULT), + false + )(logger) + } + +} + +trait RestHighLevelClientAliasApi extends AliasApi with RestHighLevelClientCompanion { + override def addAlias(index: String, alias: String): Boolean = { + tryOrElse( + apply() + .indices() + .updateAliases( + new IndicesAliasesRequest() + .addAliasAction( + new AliasActions(AliasActions.Type.ADD) + .index(index) + .alias(alias) + ), + RequestOptions.DEFAULT + ) + .isAcknowledged, + false + )(logger) + } + + override def removeAlias(index: String, alias: String): Boolean = { + tryOrElse( + apply() + .indices() + .updateAliases( + new IndicesAliasesRequest() + .addAliasAction( + new AliasActions(AliasActions.Type.REMOVE) + .index(index) + .alias(alias) + ), + RequestOptions.DEFAULT + ) + .isAcknowledged, + false + )(logger) + } +} + +trait RestHighLevelClientSettingsApi extends SettingsApi with RestHighLevelClientCompanion { + _: RestHighLevelClientIndicesApi => + + override def updateSettings(index: String, settings: String): Boolean = { + tryOrElse( + apply() + .indices() + .putSettings( + new UpdateSettingsRequest(index) + .settings(settings, XContentType.JSON), + RequestOptions.DEFAULT + ) + .isAcknowledged, + false + )(logger) + } + + override def loadSettings(index: String): String = { + tryOrElse( + apply() + .indices() + .getSettings( + new GetSettingsRequest().indices(index), + RequestOptions.DEFAULT + ) + .toString, + s"""{"$index": {"settings": {"index": {}}}}""" + )(logger) + } +} + +trait RestHighLevelClientMappingApi extends MappingApi with RestHighLevelClientCompanion { + override def setMapping(index: String, mapping: String): Boolean = { + tryOrElse( + apply() + .indices() + .putMapping( + new PutMappingRequest(index) + .source(mapping, XContentType.JSON), + RequestOptions.DEFAULT + ) + .isAcknowledged, + false + )(logger) + } + + override def getMapping(index: String): String = { + tryOrElse( + apply() + .indices() + .getMapping( + new GetMappingsRequest().indices(index), + RequestOptions.DEFAULT + ) + .mappings() + .asScala + .get(index) + .map(metadata => metadata.source().string()), + None + )(logger).getOrElse(s""""{$index: {"mappings": {}}}""") + } + + override def getMappingProperties(index: String): String = { + tryOrElse( + getMapping(index), + "{\"properties\": {}}" + )(logger) + } + +} + +trait RestHighLevelClientRefreshApi extends RefreshApi with RestHighLevelClientCompanion { + override def refresh(index: String): Boolean = { + tryOrElse( + apply() + .indices() + .refresh( + new RefreshRequest(index), + RequestOptions.DEFAULT + ) + .getStatus + .getStatus < 400, + false + )(logger) + } +} + +trait RestHighLevelClientFlushApi extends FlushApi with RestHighLevelClientCompanion { + override def flush(index: String, force: Boolean = true, wait: Boolean = true): Boolean = { + tryOrElse( + apply() + .indices() + .flush( + new FlushRequest(index).force(force).waitIfOngoing(wait), + RequestOptions.DEFAULT + ) + .getStatus == RestStatus.OK, + false + )(logger) + } +} + +trait RestHighLevelClientCountApi extends CountApi with RestHighLevelClientCompanion { + override def countAsync( + query: client.JSONQuery + )(implicit ec: ExecutionContext): Future[Option[Double]] = { + val promise = Promise[Option[Double]]() + apply().countAsync( + new CountRequest().indices(query.indices: _*).types(query.types: _*), + RequestOptions.DEFAULT, + new ActionListener[CountResponse] { + override def onResponse(response: CountResponse): Unit = + promise.success(Option(response.getCount.toDouble)) + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } + + override def count(query: client.JSONQuery): Option[Double] = { + tryOrElse( + Option( + apply() + .count( + new CountRequest().indices(query.indices: _*).types(query.types: _*), + RequestOptions.DEFAULT + ) + .getCount + .toDouble + ), + None + )(logger) + } +} + +trait RestHighLevelClientSingleValueAggregateApi + extends SingleValueAggregateApi + with RestHighLevelClientCountApi { + override def aggregate( + sqlQuery: SQLQuery + )(implicit ec: ExecutionContext): Future[Seq[SingleValueAggregateResult]] = { + val aggregations: Seq[ElasticAggregation] = sqlQuery + val futures = for (aggregation <- aggregations) yield { + val promise: Promise[SingleValueAggregateResult] = Promise() + val field = aggregation.field + val sourceField = aggregation.sourceField + val aggType = aggregation.aggType + val aggName = aggregation.aggName + val query = aggregation.query.getOrElse("") + val sources = aggregation.sources + sourceField match { + case "_id" if aggType.sql == "count" => + countAsync( + JSONQuery( + query, + collection.immutable.Seq(sources: _*), + collection.immutable.Seq.empty[String] + ) + ).onComplete { + case Success(result) => + promise.success( + SingleValueAggregateResult( + field, + aggType, + result.map(r => NumericValue(r.doubleValue())).getOrElse(EmptyValue), + None + ) + ) + case Failure(f) => + logger.error(f.getMessage, f.fillInStackTrace()) + promise.success( + SingleValueAggregateResult(field, aggType, EmptyValue, Some(f.getMessage)) + ) + } + promise.future + case _ => + val jsonQuery = JSONQuery( + query, + collection.immutable.Seq(sources: _*), + collection.immutable.Seq.empty[String] + ) + import jsonQuery._ + // Create a parser for the query + val xContentParser = XContentType.JSON + .xContent() + .createParser( + namedXContentRegistry, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + jsonQuery.query + ) + apply().searchAsync( + new SearchRequest(indices: _*) + .types(types: _*) + .source( + SearchSourceBuilder.fromXContent(xContentParser) + ), + RequestOptions.DEFAULT, + new ActionListener[SearchResponse] { + override def onResponse(response: SearchResponse): Unit = { + val agg = aggName.split("\\.").last + + val itAgg = aggName.split("\\.").iterator + + var root = + if (aggregation.nested) { + response.getAggregations.get(itAgg.next()).asInstanceOf[Nested].getAggregations + } else { + response.getAggregations + } + + if (aggregation.filtered) { + root = root.get(itAgg.next()).asInstanceOf[Filter].getAggregations + } + + promise.success( + SingleValueAggregateResult( + field, + aggType, + aggType match { + case sql.Count => + if (aggregation.distinct) { + NumericValue(root.get(agg).asInstanceOf[Cardinality].value()) + } else { + NumericValue(root.get(agg).asInstanceOf[ValueCount].value()) + } + case sql.Sum => + NumericValue(root.get(agg).asInstanceOf[Sum].value()) + case sql.Avg => + NumericValue(root.get(agg).asInstanceOf[Avg].value()) + case sql.Min => + NumericValue(root.get(agg).asInstanceOf[Min].value()) + case sql.Max => + NumericValue(root.get(agg).asInstanceOf[Max].value()) + case _ => EmptyValue + }, + None + ) + ) + } + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } + } + Future.sequence(futures) + } +} + +trait RestHighLevelClientIndexApi extends IndexApi with RestHighLevelClientCompanion { + _: RestHighLevelClientRefreshApi => + override def index(index: String, id: String, source: String): Boolean = { + tryOrElse( + apply() + .index( + new IndexRequest(index) + .id(id) + .source(source, XContentType.JSON), + RequestOptions.DEFAULT + ) + .status() + .getStatus < 400, + false + )(logger) + } + + override def indexAsync(index: String, id: String, source: String)(implicit + ec: ExecutionContext + ): Future[Boolean] = { + val promise: Promise[Boolean] = Promise() + apply().indexAsync( + new IndexRequest(index) + .id(id) + .source(source, XContentType.JSON), + RequestOptions.DEFAULT, + new ActionListener[IndexResponse] { + override def onResponse(response: IndexResponse): Unit = + promise.success(response.status().getStatus < 400) + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } +} + +trait RestHighLevelClientUpdateApi extends UpdateApi with RestHighLevelClientCompanion { + _: RestHighLevelClientRefreshApi => + override def update( + index: String, + id: String, + source: String, + upsert: Boolean + ): Boolean = { + tryOrElse( + apply() + .update( + new UpdateRequest(index, id) + .doc(source, XContentType.JSON) + .docAsUpsert(upsert), + RequestOptions.DEFAULT + ) + .status() + .getStatus < 400, + false + )(logger) + } + + override def updateAsync( + index: String, + id: String, + source: String, + upsert: Boolean + )(implicit ec: ExecutionContext): Future[Boolean] = { + val promise: Promise[Boolean] = Promise() + apply().updateAsync( + new UpdateRequest(index, id) + .doc(source, XContentType.JSON) + .docAsUpsert(upsert), + RequestOptions.DEFAULT, + new ActionListener[UpdateResponse] { + override def onResponse(response: UpdateResponse): Unit = + promise.success(response.status().getStatus < 400) + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } +} + +trait RestHighLevelClientDeleteApi extends DeleteApi with RestHighLevelClientCompanion { + _: RestHighLevelClientRefreshApi => + + override def delete(uuid: String, index: String): Boolean = { + tryOrElse( + apply() + .delete( + new DeleteRequest(index, uuid), + RequestOptions.DEFAULT + ) + .status() + .getStatus < 400, + false + )(logger) + } + + override def deleteAsync(uuid: String, index: String)(implicit + ec: ExecutionContext + ): Future[Boolean] = { + val promise: Promise[Boolean] = Promise() + apply().deleteAsync( + new DeleteRequest(index, uuid), + RequestOptions.DEFAULT, + new ActionListener[DeleteResponse] { + override def onResponse(response: DeleteResponse): Unit = + promise.success(response.status().getStatus < 400) + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } +} + +trait RestHighLevelClientGetApi extends GetApi with RestHighLevelClientCompanion { + def get[U <: Timestamped]( + id: String, + index: Option[String] = None, + maybeType: Option[String] = None + )(implicit m: Manifest[U], formats: Formats): Option[U] = { + Try( + apply().get( + new GetRequest( + index.getOrElse( + maybeType.getOrElse( + m.runtimeClass.getSimpleName.toLowerCase + ) + ), + id + ), + RequestOptions.DEFAULT + ) + ) match { + case Success(response) => + if (response.isExists) { + val source = response.getSourceAsString + logger.info(s"Deserializing response $source for id: $id, index: ${index + .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}") + // Deserialize the source string to the expected type + // Note: This assumes that the source is a valid JSON representation of U + // and that the serialization library is capable of handling it. + Try(serialization.read[U](source)) match { + case Success(value) => Some(value) + case Failure(f) => + logger.error( + s"Failed to deserialize response $source for id: $id, index: ${index + .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}", + f + ) + None + } + } else { + None + } + case Failure(f) => + logger.error( + s"Failed to get document with id: $id, index: ${index + .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}", + f + ) + None + } + } + + override def getAsync[U <: Timestamped]( + id: String, + index: Option[String] = None, + maybeType: Option[String] = None + )(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats): Future[Option[U]] = { + val promise = Promise[Option[U]]() + apply().getAsync( + new GetRequest( + index.getOrElse( + maybeType.getOrElse( + m.runtimeClass.getSimpleName.toLowerCase + ) + ), + id + ), + RequestOptions.DEFAULT, + new ActionListener[GetResponse] { + override def onResponse(response: GetResponse): Unit = { + if (response.isExists) { + promise.success(Some(serialization.read[U](response.getSourceAsString))) + } else { + promise.success(None) + } + } + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } +} + +trait RestHighLevelClientSearchApi extends SearchApi with RestHighLevelClientCompanion { + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + implicitly[ElasticSearchRequest](sqlSearch).query + + override def search[U]( + jsonQuery: JSONQuery + )(implicit m: Manifest[U], formats: Formats): List[U] = { + import jsonQuery._ + logger.info(s"Searching with query: $query on indices: ${indices.mkString(", ")}") + // Create a parser for the query + val xContentParser = XContentType.JSON + .xContent() + .createParser( + namedXContentRegistry, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + query + ) + val response = apply().search( + new SearchRequest(indices: _*) + .types(types: _*) + .source( + SearchSourceBuilder.fromXContent(xContentParser) + ), + RequestOptions.DEFAULT + ) + if (response.getHits.getTotalHits.value > 0) { + response.getHits.getHits.toList.map { hit => + logger.info(s"Deserializing hit: ${hit.getSourceAsString}") + serialization.read[U](hit.getSourceAsString) + } + } else { + List.empty[U] + } + } + + override def searchAsync[U]( + sqlQuery: SQLQuery + )(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats): Future[List[U]] = { + val jsonQuery: JSONQuery = sqlQuery + import jsonQuery._ + val promise = Promise[List[U]]() + logger.info(s"Searching with query: $query on indices: ${indices.mkString(", ")}") + // Create a parser for the query + val xContentParser = XContentType.JSON + .xContent() + .createParser( + namedXContentRegistry, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + query + ) + // Execute the search asynchronously + apply().searchAsync( + new SearchRequest(indices: _*) + .types(types: _*) + .source( + SearchSourceBuilder.fromXContent(xContentParser) + ), + RequestOptions.DEFAULT, + new ActionListener[SearchResponse] { + override def onResponse(response: SearchResponse): Unit = { + if (response.getHits.getTotalHits.value > 0) { + promise.success(response.getHits.getHits.toList.map { hit => + serialization.read[U](hit.getSourceAsString) + }) + } else { + promise.success(List.empty[U]) + } + } + + override def onFailure(e: Exception): Unit = promise.failure(e) + } + ) + promise.future + } + + override def searchWithInnerHits[U, I](jsonQuery: JSONQuery, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[(U, List[I])] = { + import jsonQuery._ + // Create a parser for the query + val xContentParser = XContentType.JSON + .xContent() + .createParser( + namedXContentRegistry, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + jsonQuery.query + ) + val response = apply().search( + new SearchRequest(indices: _*) + .types(types: _*) + .source( + SearchSourceBuilder.fromXContent(xContentParser) + ), + RequestOptions.DEFAULT + ) + Try(new JsonParser().parse(response.toString).getAsJsonObject ~> [U, I] innerField) match { + case Success(s) => s + case Failure(f) => + logger.error(f.getMessage, f) + List.empty + } + } + + override def multiSearch[U]( + jsonQueries: JSONQueries + )(implicit m: Manifest[U], formats: Formats): List[List[U]] = { + import jsonQueries._ + val request = new MultiSearchRequest() + for (query <- queries) { + request.add( + new SearchRequest(query.indices: _*) + .types(query.types: _*) + .source( + new SearchSourceBuilder( + new InputStreamStreamInput( + new ByteArrayInputStream( + query.query.getBytes() + ) + ) + ) + ) + ) + } + val responses = apply().msearch(request, RequestOptions.DEFAULT) + responses.getResponses.toList.map { response => + if (response.isFailure) { + logger.error(s"Error in multi search: ${response.getFailureMessage}") + List.empty[U] + } else { + response.getResponse.getHits.getHits.toList.map { hit => + serialization.read[U](hit.getSourceAsString) + } + } + } + } + + override def multiSearchWithInnerHits[U, I](jsonQueries: JSONQueries, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[List[(U, List[I])]] = { + import jsonQueries._ + val request = new MultiSearchRequest() + for (query <- queries) { + request.add( + new SearchRequest(query.indices: _*) + .types(query.types: _*) + .source( + new SearchSourceBuilder( + new InputStreamStreamInput( + new ByteArrayInputStream( + query.query.getBytes() + ) + ) + ) + ) + ) + } + val responses = apply().msearch(request, RequestOptions.DEFAULT) + responses.getResponses.toList.map { response => + if (response.isFailure) { + logger.error(s"Error in multi search: ${response.getFailureMessage}") + List.empty[(U, List[I])] + } else { + Try( + new JsonParser().parse(response.getResponse.toString).getAsJsonObject ~> [U, I] innerField + ) match { + case Success(s) => s + case Failure(f) => + logger.error(f.getMessage, f) + List.empty + } + } + } + } + +} + +trait RestHighLevelClientBulkApi + extends RestHighLevelClientRefreshApi + with RestHighLevelClientSettingsApi + with RestHighLevelClientIndicesApi + with BulkApi { + override type A = DocWriteRequest[_] + override type R = BulkResponse + + override def toBulkAction(bulkItem: BulkItem): A = { + import bulkItem._ + val request = action match { + case BulkAction.UPDATE => + new UpdateRequest(index, id.orNull) + .doc(body, XContentType.JSON) + .docAsUpsert(true) + case BulkAction.DELETE => + new DeleteRequest(index).id(id.getOrElse("_all")) + case _ => + new IndexRequest(index).source(body, XContentType.JSON).id(id.orNull) + } + request + } + + override def bulkResult: Flow[R, Set[String], NotUsed] = + Flow[BulkResponse] + .named("result") + .map(result => { + val items = result.getItems + val grouped = items.groupBy(_.getIndex) + val indices = grouped.keys.toSet + for (index <- indices) { + logger + .info(s"Bulk operation succeeded for index $index with ${grouped(index).length} items.") + } + indices + }) + + override def bulk(implicit + bulkOptions: BulkOptions, + system: ActorSystem + ): Flow[Seq[A], R, NotUsed] = { + val parallelism = Math.max(1, bulkOptions.balance) + Flow[Seq[A]] + .named("bulk") + .mapAsyncUnordered[R](parallelism) { items => + val request = new BulkRequest(bulkOptions.index) + items.foreach(request.add) + val promise: Promise[R] = Promise[R]() + apply().bulkAsync( + request, + RequestOptions.DEFAULT, + new ActionListener[BulkResponse] { + override def onResponse(response: BulkResponse): Unit = { + if (response.hasFailures) { + logger.error(s"Bulk operation failed: ${response.buildFailureMessage()}") + } else { + logger.info(s"Bulk operation succeeded with ${response.getItems.length} items.") + } + promise.success(response) + } + + override def onFailure(e: Exception): Unit = { + logger.error("Bulk operation failed", e) + promise.failure(e) + } + } + ) + promise.future + } + } + + private[this] def toBulkElasticResultItem(i: BulkItemResponse): BulkElasticResultItem = + new BulkElasticResultItem { + override def index: String = i.getIndex + } + + override implicit def toBulkElasticAction(a: DocWriteRequest[_]): BulkElasticAction = { + new BulkElasticAction { + override def index: String = a.index + } + } + + override implicit def toBulkElasticResult(r: BulkResponse): BulkElasticResult = { + new BulkElasticResult { + override def items: List[BulkElasticResultItem] = + r.getItems.toList.map(toBulkElasticResultItem) + } + } +} diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala new file mode 100644 index 00000000..cf6c40bd --- /dev/null +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala @@ -0,0 +1,55 @@ +package app.softnetwork.elastic.client.rest + +import app.softnetwork.elastic.client.ElasticConfig +import org.apache.http.HttpHost +import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.client.{RestClient, RestClientBuilder, RestHighLevelClient} +import org.elasticsearch.common.settings.Settings +import org.elasticsearch.xcontent.NamedXContentRegistry +import org.elasticsearch.plugins.SearchPlugin +import org.elasticsearch.search.SearchModule +import org.slf4j.{Logger, LoggerFactory} + +trait RestHighLevelClientCompanion { + + val logger: Logger = LoggerFactory getLogger getClass.getName + + def elasticConfig: ElasticConfig + + private var client: Option[RestHighLevelClient] = None + + lazy val namedXContentRegistry: NamedXContentRegistry = { + import scala.collection.JavaConverters._ + val searchModule = new SearchModule(Settings.EMPTY, false, List.empty[SearchPlugin].asJava) + new NamedXContentRegistry(searchModule.getNamedXContents) + } + + def apply(): RestHighLevelClient = { + client match { + case Some(c) => c + case _ => + val credentialsProvider = new BasicCredentialsProvider() + if (elasticConfig.credentials.username.nonEmpty) { + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials( + elasticConfig.credentials.username, + elasticConfig.credentials.password + ) + ) + } + val restClientBuilder: RestClientBuilder = RestClient + .builder( + HttpHost.create(elasticConfig.credentials.url) + ) + .setHttpClientConfigCallback((httpAsyncClientBuilder: HttpAsyncClientBuilder) => + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + ) + val c = new RestHighLevelClient(restClientBuilder) + client = Some(c) + c + } + } +} diff --git a/es7/testkit/build.sbt b/es7/testkit/build.sbt new file mode 100644 index 00000000..94b22a86 --- /dev/null +++ b/es7/testkit/build.sbt @@ -0,0 +1,42 @@ +import SoftClient4es.* + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-testkit" + +libraryDependencies ++= elastic4sTestkitDependencies(elasticSearchVersion.value) ++ Seq( + "org.apache.logging.log4j" % "log4j-api" % Versions.log4j, + // "org.apache.logging.log4j" % "log4j-slf4j-impl" % Versions.log4j, + "org.apache.logging.log4j" % "log4j-core" % Versions.log4j, + "app.softnetwork.persistence" %% "persistence-core-testkit" % Versions.genericPersistence, + "org.testcontainers" % "elasticsearch" % Versions.testContainers excludeAll (jacksonExclusions: _*) +) + +val testJavaOptions = { + val heapSize = sys.env.getOrElse("HEAP_SIZE", "1g") + val extraTestJavaArgs = Seq( + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED" + ).mkString(" ") + s"-Xmx$heapSize -Xss4m -XX:ReservedCodeCacheSize=128m -Dfile.encoding=UTF-8 $extraTestJavaArgs" + .split(" ") + .toSeq +} + +Test / javaOptions ++= testJavaOptions + +// Required by the Test container framework +Test / fork := true diff --git a/es7/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/es7/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala new file mode 100644 index 00000000..cdb90e30 --- /dev/null +++ b/es7/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -0,0 +1,194 @@ +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Flow +import app.softnetwork.elastic.sql.SQLQuery +import org.json4s.Formats +import app.softnetwork.persistence.model.Timestamped +import org.slf4j.{Logger, LoggerFactory} + +import scala.concurrent.{ExecutionContext, Future} +import scala.language.implicitConversions +import scala.reflect.ClassTag + +/** Created by smanciot on 12/04/2020. + */ +trait MockElasticClientApi extends ElasticClientApi { + + protected lazy val log: Logger = LoggerFactory getLogger getClass.getName + + protected val elasticDocuments: ElasticDocuments = new ElasticDocuments() {} + + override def toggleRefresh(index: String, enable: Boolean): Boolean = true + + override def setReplicas(index: String, replicas: Int): Boolean = true + + override def updateSettings(index: String, settings: String) = true + + override def addAlias(index: String, alias: String): Boolean = true + + /** Remove an alias from the given index. + * + * @param index + * - the name of the index + * @param alias + * - the name of the alias + * @return + * true if the alias was removed successfully, false otherwise + */ + override def removeAlias(index: String, alias: String): Boolean = true + + override def createIndex(index: String, settings: String): Boolean = true + + override def setMapping(index: String, mapping: String): Boolean = true + + override def deleteIndex(index: String): Boolean = true + + override def closeIndex(index: String): Boolean = true + + override def openIndex(index: String): Boolean = true + + /** Reindex from source index to target index. + * + * @param sourceIndex + * - the name of the source index + * @param targetIndex + * - the name of the target index + * @param refresh + * - true to refresh the target index after reindexing, false otherwise + * @return + * true if the reindexing was successful, false otherwise + */ + override def reindex(sourceIndex: String, targetIndex: String, refresh: Boolean = true): Boolean = + true + + /** Check if an index exists. + * + * @param index + * - the name of the index to check + * @return + * true if the index exists, false otherwise + */ + override def indexExists(index: String): Boolean = false + + override def count(jsonQuery: JSONQuery): Option[Double] = + throw new UnsupportedOperationException + + override def get[U <: Timestamped]( + id: String, + index: Option[String] = None, + maybeType: Option[String] = None + )(implicit m: Manifest[U], formats: Formats): Option[U] = + elasticDocuments.get(id).asInstanceOf[Option[U]] + + override def search[U](sqlQuery: SQLQuery)(implicit m: Manifest[U], formats: Formats): List[U] = + elasticDocuments.getAll.toList.asInstanceOf[List[U]] + + override def multiSearch[U]( + jsonQueries: JSONQueries + )(implicit m: Manifest[U], formats: Formats): List[List[U]] = + throw new UnsupportedOperationException + + override def index(index: String, id: String, source: String): Boolean = + throw new UnsupportedOperationException + + override def update[U <: Timestamped]( + entity: U, + index: Option[String] = None, + maybeType: Option[String] = None, + upsert: Boolean = true + )(implicit u: ClassTag[U], formats: Formats): Boolean = { + elasticDocuments.createOrUpdate(entity) + true + } + + override def update( + index: String, + id: String, + source: String, + upsert: Boolean + ): Boolean = { + log.warn(s"MockElasticClient - $id not updated for $source") + false + } + + override def delete(uuid: String, index: String): Boolean = { + if (elasticDocuments.get(uuid).isDefined) { + elasticDocuments.delete(uuid) + true + } else { + false + } + } + + override def refresh(index: String): Boolean = true + + override def flush(index: String, force: Boolean, wait: Boolean): Boolean = true + + override type A = this.type + + override def bulk(implicit + bulkOptions: BulkOptions, + system: ActorSystem + ): Flow[Seq[A], R, NotUsed] = + throw new UnsupportedOperationException + + override def bulkResult: Flow[R, Set[String], NotUsed] = + throw new UnsupportedOperationException + + override type R = this.type + + override def toBulkAction(bulkItem: BulkItem): A = + throw new UnsupportedOperationException + + override implicit def toBulkElasticAction(a: A): BulkElasticAction = + throw new UnsupportedOperationException + + override implicit def toBulkElasticResult(r: R): BulkElasticResult = + throw new UnsupportedOperationException + + override def multiSearchWithInnerHits[U, I](jsonQueries: JSONQueries, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[List[(U, List[I])]] = List.empty + + override def search[U](jsonQuery: JSONQuery)(implicit m: Manifest[U], formats: Formats): List[U] = + List.empty + + override def searchWithInnerHits[U, I](jsonQuery: JSONQuery, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[(U, List[I])] = List.empty + + override def getMapping(index: String): String = + throw new UnsupportedOperationException + + override def aggregate(sqlQuery: SQLQuery)(implicit + ec: ExecutionContext + ): Future[Seq[SingleValueAggregateResult]] = + throw new UnsupportedOperationException + + override def loadSettings(index: String): String = + throw new UnsupportedOperationException +} + +trait ElasticDocuments { + + private[this] var documents: Map[String, Timestamped] = Map() + + def createOrUpdate(entity: Timestamped): Unit = { + documents = documents.updated(entity.uuid, entity) + } + + def delete(uuid: String): Unit = { + documents = documents - uuid + } + + def getAll: Iterable[Timestamped] = documents.values + + def get(uuid: String): Option[Timestamped] = documents.get(uuid) + +} diff --git a/es7/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala b/es7/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala new file mode 100644 index 00000000..b2e88a55 --- /dev/null +++ b/es7/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala @@ -0,0 +1,58 @@ +package app.softnetwork.elastic.scalatest + +import org.scalatest.Suite +import org.testcontainers.containers.BindMode +//import org.testcontainers.containers.wait.strategy.Wait +import org.testcontainers.elasticsearch.ElasticsearchContainer +import org.testcontainers.utility.DockerImageName + +import java.nio.file.Files +import java.time.Duration + +/** Created by smanciot on 28/06/2018. + */ +trait ElasticDockerTestKit extends ElasticTestKit { _: Suite => + + override lazy val elasticURL: String = s"http://${elasticContainer.getHttpHostAddress}" + + lazy val localExecution: Boolean = sys.props.get("LOCAL_EXECUTION") match { + case Some("true") => true + case _ => false + } + + lazy val elasticContainer: ElasticsearchContainer = { + val tmpDir = + if (localExecution) { + val tmp = Files.createTempDirectory("es-tmp") + tmp.toFile.setWritable(true, false) + tmp.toAbsolutePath.toString + } else { + "/tmp" + } + Console.println(s"Using temporary directory for Elasticsearch: $tmpDir") + val container = new ElasticsearchContainer( + DockerImageName + .parse("docker.elastic.co/elasticsearch/elasticsearch") + .withTag(elasticVersion) + ) + container.addEnv("ES_TMPDIR", "/usr/share/elasticsearch/tmp") + container.addEnv("discovery.type", "single-node") + container.addEnv("xpack.security.enabled", "false") + container.addEnv("xpack.ml.enabled", "false") + container.addEnv("xpack.watcher.enabled", "false") + container.addEnv("xpack.graph.enabled", "false") + container.addFileSystemBind( + tmpDir, + "/usr/share/elasticsearch/tmp", + BindMode.READ_WRITE + ) + // container.addEnv("ES_JAVA_OPTS", "-Xms1024m -Xmx1024m") + // container.setWaitStrategy(Wait.forHttp("/").forStatusCode(200)) + container.withStartupTimeout(Duration.ofMinutes(2)) + } + + override def start(): Unit = elasticContainer.start() + + override def stop(): Unit = elasticContainer.stop() + +} diff --git a/es7/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala b/es7/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala new file mode 100644 index 00000000..178f1a65 --- /dev/null +++ b/es7/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala @@ -0,0 +1,344 @@ +package app.softnetwork.elastic.scalatest + +import app.softnetwork.concurrent.scalatest.CompletionTestKit +import app.softnetwork.elastic.Softclient4es7TestkitBuildInfo +import com.sksamuel.elastic4s.http.JavaClient +import com.sksamuel.elastic4s.requests.indexes.admin.RefreshIndexResponse +import com.sksamuel.elastic4s.{ElasticClient, ElasticDsl, Indexes} +import com.typesafe.config.{Config, ConfigFactory} +import org.apache.http.HttpHost +import org.elasticsearch.ResourceAlreadyExistsException +import org.elasticsearch.client.RestClient +import org.elasticsearch.transport.RemoteTransportException +import org.scalatest.{BeforeAndAfterAll, Suite} +import org.scalatest.matchers.{MatchResult, Matcher} +import org.slf4j.Logger + +import scala.util.{Failure, Success, Try} + +/** Created by smanciot on 18/05/2021. + */ +trait ElasticTestKit extends ElasticDsl with CompletionTestKit with BeforeAndAfterAll { _: Suite => + + def log: Logger + + def elasticVersion: String = Softclient4es7TestkitBuildInfo.elasticVersion + + def elasticURL: String + + lazy val elasticConfig: Config = ConfigFactory + .parseString(elasticConfigAsString) + .withFallback(ConfigFactory.load("softnetwork-elastic.conf")) + + lazy val elasticConfigAsString: String = + s""" + |elastic { + | credentials { + | url = "$elasticURL" + | } + | multithreaded = false + | discovery-enabled = false + |} + |""".stripMargin + + lazy val elasticClient: ElasticClient = ElasticClient( + new JavaClient( + RestClient + .builder( + HttpHost.create(elasticURL) + ) + .build() + ) + ) + + def start(): Unit = () + + def stop(): Unit = () + + override def beforeAll(): Unit = { + start() + elasticClient + .execute { + createIndexTemplate("all_templates", "*").settings( + Map("number_of_shards" -> 1, "number_of_replicas" -> 0) + ) + } + .complete() match { + case Success(_) => () + case Failure(f) => throw f + } + } + + override def afterAll(): Unit = { + elasticClient.close() + stop() + } + + // Rewriting methods from IndexMatchers in elastic4s with the ElasticClient + def haveCount(expectedCount: Int): Matcher[String] = + (index: String) => { + elasticClient.execute(search(index).size(0)).complete() match { + case Success(s) => + val count = s.result.totalHits + MatchResult( + count == expectedCount, + s"Index $index had count $count but expected $expectedCount", + s"Index $index had document count $expectedCount" + ) + case Failure(f) => throw f + } + } + + def containDoc(expectedId: String): Matcher[String] = + (index: String) => { + elasticClient.execute(get(index, expectedId)).complete() match { + case Success(s) => + val exists = s.result.exists + MatchResult( + exists, + s"Index $index did not contain expected document $expectedId", + s"Index $index contained document $expectedId" + ) + case Failure(f) => throw f + } + } + + def beCreated(): Matcher[String] = + (index: String) => { + elasticClient.execute(indexExists(index)).complete() match { + case Success(s) => + val exists = s.result.isExists + MatchResult( + exists, + s"Index $index did not exist", + s"Index $index exists" + ) + case Failure(f) => throw f + } + } + + def beEmpty(): Matcher[String] = + (index: String) => { + elasticClient.execute(search(index).size(0)).complete() match { + case Success(s) => + val count = s.result.totalHits + MatchResult( + count == 0, + s"Index $index was not empty", + s"Index $index was empty" + ) + case Failure(f) => throw f + } + } + + // Copy/paste methos HttpElasticSugar as it is not available yet + + // refresh all indexes + def refreshAll(): RefreshIndexResponse = refresh(Indexes.All) + + // refreshes all specified indexes + def refresh(indexes: Indexes): RefreshIndexResponse = { + elasticClient + .execute { + refreshIndex(indexes) + } + .complete() match { + case Success(s) => s.result + case Failure(f) => throw f + } + } + + def blockUntilGreen(): Unit = { + blockUntil("Expected cluster to have green status") { () => + elasticClient + .execute { + clusterHealth() + } + .complete() match { + case Success(s) => s.result.status.toUpperCase == "GREEN" + case Failure(f) => throw f + } + } + } + + def blockUntil(explain: String)(predicate: () => Boolean): Unit = { + blockUntil(explain, 16, 200)(predicate) + } + + def ensureIndexExists(index: String): Unit = { + elasticClient + .execute { + createIndex(index) + } + .complete() match { + case Success(_) => () + case Failure(f) => + f match { + case _: ResourceAlreadyExistsException => // Ok, ignore. + case _: RemoteTransportException => // Ok, ignore. + case other => throw other + } + } + } + + def doesIndexExists(name: String): Boolean = { + elasticClient + .execute { + indexExists(name) + } + .complete() match { + case Success(s) => s.result.isExists + case _ => false + } + } + + def isIndexOpened(name: String): Boolean = { + elasticClient + .execute { + indexStats(name) + } + .complete() match { + case Success(s) => + Try(s.result.indices.contains(name)).toOption.getOrElse(false) + case _ => false + } + } + + def isIndexClosed(name: String): Boolean = { + doesIndexExists(name) && !isIndexOpened(name) + } + + def doesAliasExists(name: String): Boolean = { + elasticClient + .execute { + aliasExists(name) + } + .complete() match { + case Success(s) => s.result.isExists + case _ => false + } + } + + def deleteIndex(name: String): Unit = { + if (doesIndexExists(name)) { + elasticClient + .execute { + ElasticDsl.deleteIndex(name) + } + .complete() match { + case Success(_) => () + case Failure(f) => throw f + } + } + } + + def truncateIndex(index: String): Unit = { + deleteIndex(index) + ensureIndexExists(index) + blockUntilEmpty(index) + } + + def blockUntilDocumentExists(id: String, index: String, _type: String): Unit = { + blockUntil(s"Expected to find document $id") { () => + elasticClient + .execute { + get(index, id) + } + .complete() match { + case Success(s) => s.result.exists + case _ => false + } + } + } + + def blockUntilCount(expected: Long, index: String): Unit = { + blockUntil(s"Expected count of $expected") { () => + elasticClient + .execute { + search(index).matchAllQuery().size(0) + } + .complete() match { + case Success(s) => expected <= s.result.totalHits + case Failure(f) => throw f + } + } + } + + /** Will block until the given index and optional types have at least the given number of + * documents. + */ + def blockUntilCount(expected: Long, index: String, types: String*): Unit = { + blockUntil(s"Expected count of $expected") { () => + elasticClient + .execute { + search(index).matchAllQuery().size(0) + } + .complete() match { + case Success(s) => expected <= s.result.totalHits + case Failure(f) => throw f + } + } + } + + def blockUntilExactCount(expected: Long, index: String, types: String*): Unit = { + blockUntil(s"Expected count of $expected") { () => + elasticClient + .execute { + search(index).size(0) + } + .complete() match { + case Success(s) => expected == s.result.totalHits + case Failure(f) => throw f + } + } + } + + def blockUntilEmpty(index: String): Unit = { + blockUntil(s"Expected empty index $index") { () => + elasticClient + .execute { + search(Indexes(index)).size(0) + } + .complete() match { + case Success(s) => s.result.totalHits == 0 + case Failure(f) => throw f + } + } + } + + def blockUntilIndexExists(index: String): Unit = { + blockUntil(s"Expected exists index $index") { () => + doesIndexExists(index) + } + } + + def blockUntilIndexNotExists(index: String): Unit = { + blockUntil(s"Expected not exists index $index") { () => + !doesIndexExists(index) + } + } + + def blockUntilAliasExists(alias: String): Unit = { + blockUntil(s"Expected exists alias $alias") { () => + doesAliasExists(alias) + } + } + + def blockUntilDocumentHasVersion( + index: String, + _type: String, + id: String, + version: Long + ): Unit = { + blockUntil(s"Expected document $id to have version $version") { () => + elasticClient + .execute { + get(index, id) + } + .complete() match { + case Success(s) => s.result.version == version + case Failure(f) => throw f + } + } + } +} diff --git a/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala b/es7/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala similarity index 100% rename from testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala rename to es7/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala diff --git a/es7/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala b/es7/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala new file mode 100644 index 00000000..d3b6c3f9 --- /dev/null +++ b/es7/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala @@ -0,0 +1,14 @@ +package app.softnetwork.persistence.query + +import app.softnetwork.elastic.client.ElasticClientApi +import app.softnetwork.elastic.persistence.query.{ElasticProvider, State2ElasticProcessorStream} +import app.softnetwork.persistence.person.message.PersonEvent +import app.softnetwork.persistence.person.model.Person +import app.softnetwork.persistence.person.query.PersonToExternalProcessorStream + +trait PersonToElasticProcessorStream + extends State2ElasticProcessorStream[Person, PersonEvent] + with PersonToExternalProcessorStream + with InMemoryJournalProvider + with InMemoryOffsetProvider + with ElasticProvider[Person] { _: ElasticClientApi => } diff --git a/es7/testkit/src/test/resources/application.conf b/es7/testkit/src/test/resources/application.conf new file mode 100644 index 00000000..ba8abfad --- /dev/null +++ b/es7/testkit/src/test/resources/application.conf @@ -0,0 +1,3 @@ +akka.coordinated-shutdown.exit-jvm = off +elastic.multithreaded = false +clustering.port = 0 diff --git a/es7/testkit/src/test/resources/avatar.jpg b/es7/testkit/src/test/resources/avatar.jpg new file mode 100644 index 00000000..7a214ba8 Binary files /dev/null and b/es7/testkit/src/test/resources/avatar.jpg differ diff --git a/es7/testkit/src/test/resources/avatar.pdf b/es7/testkit/src/test/resources/avatar.pdf new file mode 100644 index 00000000..cf44452f Binary files /dev/null and b/es7/testkit/src/test/resources/avatar.pdf differ diff --git a/es7/testkit/src/test/resources/avatar.png b/es7/testkit/src/test/resources/avatar.png new file mode 100644 index 00000000..a11b4dcd Binary files /dev/null and b/es7/testkit/src/test/resources/avatar.png differ diff --git a/es7/testkit/src/test/resources/mapping/person.mustache b/es7/testkit/src/test/resources/mapping/person.mustache new file mode 100644 index 00000000..21829e1c --- /dev/null +++ b/es7/testkit/src/test/resources/mapping/person.mustache @@ -0,0 +1,21 @@ +{ + "properties": { + "uuid": { + "type": "keyword", + "index": true + }, + "name": { + "type": "text", + "analyzer": "search_analyzer" + }, + "birthDate": { + "type": "keyword" + }, + "createdDate": { + "type": "date" + }, + "lastUpdated": { + "type": "date" + } + } +} \ No newline at end of file diff --git a/es7/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/es7/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala new file mode 100644 index 00000000..92ec2a74 --- /dev/null +++ b/es7/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -0,0 +1,914 @@ +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.sql.SQLQuery +import com.fasterxml.jackson.core.JsonParseException +import com.sksamuel.elastic4s.requests.searches.queries.matches.MatchAllQuery +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import app.softnetwork.persistence._ +import app.softnetwork.serialization._ +import app.softnetwork.elastic.model._ +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.persistence.person.model.Person +import com.google.gson.JsonParser +import com.typesafe.scalalogging.StrictLogging +import org.json4s.Formats +import org.slf4j.{Logger, LoggerFactory} + +import _root_.java.io.ByteArrayInputStream +import _root_.java.nio.file.{Files, Paths} +import _root_.java.time.format.DateTimeFormatter +import _root_.java.util.concurrent.TimeUnit +import _root_.java.util.UUID +import java.time.{LocalDate, LocalDateTime, ZoneOffset} +import scala.concurrent.{Await, ExecutionContextExecutor} +import scala.concurrent.duration.Duration +import scala.util.{Failure, Success} + +/** Created by smanciot on 28/06/2018. + */ +trait ElasticClientSpec + extends AnyFlatSpecLike + with ElasticDockerTestKit + with Matchers + with StrictLogging { + + lazy val log: Logger = LoggerFactory getLogger getClass.getName + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + + implicit val formats: Formats = commonFormats + + def pClient: ElasticProvider[Person] with ElasticClientApi + def sClient: ElasticProvider[Sample] with ElasticClientApi + def bClient: ElasticProvider[Binary] with ElasticClientApi + def parentClient: ElasticProvider[Parent] with ElasticClientApi + + import scala.language.implicitConversions + + implicit def toSQLQuery(sqlQuery: String): SQLQuery = SQLQuery(sqlQuery) + + override def beforeAll(): Unit = { + super.beforeAll() + pClient.createIndex("person") + } + + override def afterAll(): Unit = { + Await.result(system.terminate(), Duration(30, TimeUnit.SECONDS)) + super.afterAll() + } + + val persons: List[String] = List( + """ { "uuid": "A12", "name": "Homer Simpson", "birthDate": "1967-11-21", "childrenCount": 0} """, + """ { "uuid": "A14", "name": "Moe Szyslak", "birthDate": "1967-11-21", "childrenCount": 0} """, + """ { "uuid": "A16", "name": "Barney Gumble", "birthDate": "1969-05-09", "childrenCount": 0} """ + ) + + private val personsWithUpsert = + persons :+ """ { "uuid": "A16", "name": "Barney Gumble2", "birthDate": "1969-05-09", "children": [{ "parentId": "A16", "name": "Steve Gumble", "birthDate": "1999-05-09"}, { "parentId": "A16", "name": "Josh Gumble", "birthDate": "2002-05-09"}], "childrenCount": 2 } """ + + val children: List[String] = List( + """ { "parentId": "A16", "name": "Steve Gumble", "birthDate": "1999-05-09"} """, + """ { "parentId": "A16", "name": "Josh Gumble", "birthDate": "1999-05-09"} """ + ) + + "Creating an index and then delete it" should "work fine" in { + pClient.createIndex("create_delete") + blockUntilIndexExists("create_delete") + "create_delete" should beCreated() + + pClient.deleteIndex("create_delete") + blockUntilIndexNotExists("create_delete") + "create_delete" should not(beCreated()) + } + + "Adding an alias and then removing it" should "work" in { + pClient.addAlias("person", "person_alias") + + doesAliasExists("person_alias") shouldBe true + + pClient.removeAlias("person", "person_alias") + + doesAliasExists("person_alias") shouldBe false + } + + private def settings: Map[String, String] = { + elasticClient + .execute { + getSettings("person") + } + .complete() match { + case Success(s) => s.result.settingsForIndex("person") + case Failure(f) => throw f + } + } + + "Toggle refresh" should "work" in { + pClient.toggleRefresh("person", enable = false) + new JsonParser() + .parse(pClient.loadSettings("person")) + .getAsJsonObject + .get("person") + .getAsJsonObject + .get("settings") + .getAsJsonObject + .get("index") + .getAsJsonObject + .get("refresh_interval") + .getAsString shouldBe "-1" + // settings.getOrElse("index.refresh_interval", "") shouldBe "-1" + + pClient.toggleRefresh("person", enable = true) + // settings.getOrElse("index.refresh_interval", "") shouldBe "1s" + new JsonParser() + .parse(pClient.loadSettings("person")) + .getAsJsonObject + .get("person") + .getAsJsonObject + .get("settings") + .getAsJsonObject + .get("index") + .getAsJsonObject + .get("refresh_interval") + .getAsString shouldBe "1s" + } + + "Opening an index and then closing it" should "work" in { + pClient.openIndex("person") + + isIndexOpened("person") shouldBe true + + pClient.closeIndex("person") + isIndexClosed("person") shouldBe true + } + + "Updating number of replicas" should "work" in { + pClient.setReplicas("person", 3) + settings.getOrElse("index.number_of_replicas", "") shouldBe "3" + + pClient.setReplicas("person", 0) + settings.getOrElse("index.number_of_replicas", "") shouldBe "0" + } + + "Setting a mapping" should "work" in { + pClient.createIndex("person_mapping") + blockUntilIndexExists("person_mapping") + "person_mapping" should beCreated() + + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "text", + | "analyzer": "ngram_analyzer", + | "search_analyzer": "search_analyzer", + | "fields": { + | "raw": { + | "type": "keyword" + | }, + | "fr": { + | "type": "text", + | "analyzer": "french" + | } + | } + | } + | } + |}""".stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + pClient.setMapping("person_mapping", mapping) shouldBe true + + val properties = pClient.getMappingProperties("person_mapping") + logger.info(s"properties: $properties") + MappingComparator.isMappingDifferent( + properties, + mapping + ) shouldBe false + + implicit val bulkOptions: BulkOptions = BulkOptions("person_mapping", "person", 1000) + val indices = pClient.bulk[String](persons.iterator, identity, Some("uuid"), None, None) + refresh(indices) + pClient.flush("person_mapping") + + indices should contain only "person_mapping" + + blockUntilCount(3, "person_mapping") + + "person_mapping" should haveCount(3) + + pClient.search[Person]("select * from person_mapping") match { + case r if r.size == 3 => + r.map(_.uuid) should contain allOf ("A12", "A14", "A16") + case other => fail(other.toString) + } + + pClient.search[Person]("select * from person_mapping where uuid = 'A16'") match { + case r if r.size == 1 => + r.map(_.uuid) should contain only "A16" + case other => fail(other.toString) + } + + pClient.search[Person]("select * from person_mapping where match(name, 'gum')") match { + case r if r.size == 1 => + r.map(_.uuid) should contain only "A16" + case other => fail(other.toString) + } + + pClient.search[Person]( + "select * from person_mapping where uuid <> 'A16' and match(name, 'gum')" + ) match { + case r if r.isEmpty => + case other => fail(other.toString) + } + } + + "Updating a mapping" should "work" in { + val mapping = + """{ + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + pClient.updateMapping("person_migration", mapping) shouldBe true + blockUntilIndexExists("person_migration") + "person_migration" should beCreated() + + implicit val bulkOptions: BulkOptions = BulkOptions("person_migration", "person", 1000) + val indices = pClient.bulk[String](persons.iterator, identity, Some("uuid"), None, None) + refresh(indices) + pClient.flush("person_migration") + + indices should contain only "person_migration" + + blockUntilCount(3, "person_migration") + + "person_migration" should haveCount(3) + + pClient.search[Person]("select * from person_migration where match(name, 'gum')") match { + case r if r.isEmpty => + case other => fail(other.toString) + } + + val newMapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "text", + | "analyzer": "ngram_analyzer", + | "search_analyzer": "search_analyzer", + | "fields": { + | "raw": { + | "type": "keyword" + | }, + | "fr": { + | "type": "text", + | "analyzer": "french" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + pClient.shouldUpdateMapping("person_migration", newMapping) shouldBe true + pClient.updateMapping("person_migration", newMapping) shouldBe true + + pClient.search[Person]("select * from person_migration where match(name, 'gum')") match { + case r if r.size == 1 => + r.map(_.uuid) should contain only "A16" + case other => fail(other.toString) + } + + } + + "Bulk index valid json without id key and suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person1", "person", 2) + val indices = pClient.bulk[String](persons.iterator, identity, None, None, None) + + indices should contain only "person1" + + blockUntilCount(3, "person1") + + "person1" should haveCount(3) + + val response = elasticClient + .execute { + search("person1").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id should not be h.sourceField("uuid") + } + + response.result.hits.hits + .map( + _.sourceField("name") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble") + } + + "Bulk index valid json with an id key but no suffix key" should "work" in { + elasticClient + .execute( + createIndex("person2").mapping( + properties( + objectField("child").copy(properties = Seq(textField("name").copy(index = Some(false)))) + ) + ) + ) + .complete() match { + case Success(_) => () + case Failure(f) => throw f + } + + implicit val bulkOptions: BulkOptions = BulkOptions("person2", "person", 1000) + val indices = pClient.bulk[String](persons.iterator, identity, Some("uuid"), None, None) + refresh(indices) + pClient.flush("person2") + + indices should contain only "person2" + + blockUntilCount(3, "person2") + + "person2" should haveCount(3) + + val response = elasticClient + .execute { + search("person2").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceField("uuid") + } + + response.result.hits.hits + .map( + _.sourceField("name") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble") + + } + + "Bulk index valid json with an id key and a suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person", "person", 1000) + val indices = + pClient.bulk[String](persons.iterator, identity, Some("uuid"), Some("birthDate"), None, None) + refresh(indices) + + indices should contain allOf ("person-1967-11-21", "person-1969-05-09") + + blockUntilCount(2, "person-1967-11-21") + blockUntilCount(1, "person-1969-05-09") + + "person-1967-11-21" should haveCount(2) + "person-1969-05-09" should haveCount(1) + + val response = elasticClient + .execute { + search("person-1967-11-21", "person-1969-05-09").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceField("uuid") + } + + response.result.hits.hits + .map( + _.sourceField("name") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble") + } + + "Bulk index invalid json with an id key and a suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person_error", "person", 1000) + intercept[JsonParseException] { + val invalidJson = persons :+ "fail" + pClient.bulk[String](invalidJson.iterator, identity, None, None, None) + } + } + + "Bulk upsert valid json with an id key but no suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person4", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person4" + + blockUntilCount(3, "person4") + + "person4" should haveCount(3) + + val response = elasticClient + .execute { + search("person4").query(MatchAllQuery()) + } + .complete() + + logger.info(s"response: ${response.result.hits.hits.mkString("\n")}") + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceAsMap.getOrElse("uuid", "") + } + + response.result.hits.hits + .map( + _.sourceAsMap.getOrElse("name", "") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble2") + } + + "Bulk upsert valid json with an id key and a suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person5", "person", 1000) + val indices = pClient.bulk[String]( + personsWithUpsert.iterator, + identity, + Some("uuid"), + Some("birthDate"), + None, + Some(true) + ) + refresh(indices) + + indices should contain allOf ("person5-1967-11-21", "person5-1969-05-09") + + blockUntilCount(2, "person5-1967-11-21") + blockUntilCount(1, "person5-1969-05-09") + + "person5-1967-11-21" should haveCount(2) + "person5-1969-05-09" should haveCount(1) + + val response = elasticClient + .execute { + search("person5-1967-11-21", "person5-1969-05-09").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceAsMap.getOrElse("uuid", "") + } + + response.result.hits.hits + .map( + _.sourceAsMap.getOrElse("name", "") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble2") + } + + "Count" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person6", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person6" + + blockUntilCount(3, "person6") + + "person6" should haveCount(3) + + import scala.collection.immutable.Seq + + pClient + .count(JSONQuery("{}", Seq[String]("person6"), Seq[String]())) + .getOrElse(0d) + .toInt should ===(3) + + pClient.countAsync(JSONQuery("{}", Seq[String]("person6"), Seq[String]())).complete() match { + case Success(s) => s.getOrElse(0d).toInt should ===(3) + case Failure(f) => fail(f.getMessage) + } + } + + "Search" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person7", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person7" + + blockUntilCount(3, "person7") + + "person7" should haveCount(3) + + val r1 = pClient.search[Person]("select * from person7") + r1.size should ===(3) + r1.map(_.uuid) should contain allOf ("A12", "A14", "A16") + + pClient.searchAsync[Person]("select * from person7") onComplete { + case Success(r) => + r.size should ===(3) + r.map(_.uuid) should contain allOf ("A12", "A14", "A16") + case Failure(f) => fail(f.getMessage) + } + + val r2 = pClient.search[Person]("select * from person7 where _id=\"A16\"") + r2.size should ===(1) + r2.map(_.uuid) should contain("A16") + + pClient.searchAsync[Person]("select * from person7 where _id=\"A16\"") onComplete { + case Success(r) => + r.size should ===(1) + r.map(_.uuid) should contain("A16") + case Failure(f) => fail(f.getMessage) + } + } + + "Get all" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person8", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person8" + + blockUntilCount(3, "person8") + + "person8" should haveCount(3) + + val response = pClient.search[Person]("select * from person8") + + response.size should ===(3) + + } + + "Get" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person9", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person9" + + blockUntilCount(3, "person9") + + "person9" should haveCount(3) + + val response = pClient.get[Person]("A16", Some("person9")) + + response.isDefined shouldBe true + response.get.uuid shouldBe "A16" + + pClient.getAsync[Person]("A16", Some("person9")).complete() match { + case Success(r) => + r.isDefined shouldBe true + r.get.uuid shouldBe "A16" + case Failure(f) => fail(f.getMessage) + } + } + + "Index" should "work" in { + val uuid = UUID.randomUUID().toString + val sample = Sample(uuid) + val result = sClient.index(sample) + result shouldBe true + + sClient.indexAsync(sample).complete() match { + case Success(r) => r shouldBe true + case Failure(f) => fail(f.getMessage) + } + + val result2 = sClient.get[Sample](uuid) + result2 match { + case Some(r) => + r.uuid shouldBe uuid + case _ => + fail("Sample not found") + } + } + + "Update" should "work" in { + val uuid = UUID.randomUUID().toString + val sample = Sample(uuid) + val result = sClient.update(sample) + result shouldBe true + + sClient.updateAsync(sample).complete() match { + case Success(r) => r shouldBe true + case Failure(f) => fail(f.getMessage) + } + + val result2 = sClient.get[Sample](uuid) + result2 match { + case Some(r) => + r.uuid shouldBe uuid + case _ => + fail("Sample not found") + } + } + + "Delete" should "work" in { + val uuid = UUID.randomUUID().toString + val sample = Sample(uuid) + val result = sClient.index(sample) + result shouldBe true + + val result2 = sClient.delete(sample.uuid, Some("sample")) + result2 shouldBe true + + /*FIXME sClient.deleteAsync(sample.uuid, Some("sample")).complete() match { + case Success(r) => r shouldBe true + case Failure(f) => fail(f.getMessage) + }*/ + + val result3 = sClient.get[Sample](uuid) + result3.isEmpty shouldBe true + } + + "Index binary data" should "work" in { + bClient.createIndex("binaries") shouldBe true + val mapping = + """{ + | "properties": { + | "uuid": { + | "type": "keyword", + | "index": true + | }, + | "createdDate": { + | "type": "date" + | }, + | "lastUpdated": { + | "type": "date" + | }, + | "content": { + | "type": "binary" + | }, + | "md5": { + | "type": "keyword" + | } + | } + |} + """.stripMargin + bClient.setMapping("binaries", mapping) shouldBe true + for (uuid <- Seq("png", "jpg", "pdf")) { + val path = + Paths.get(Thread.currentThread().getContextClassLoader.getResource(s"avatar.$uuid").getPath) + import app.softnetwork.utils.ImageTools._ + import app.softnetwork.utils.HashTools._ + import app.softnetwork.utils.Base64Tools._ + val encoded = encodeImageBase64(path).getOrElse("") + val binary = Binary( + uuid, + content = encoded, + md5 = hashStream(new ByteArrayInputStream(decodeBase64(encoded))).getOrElse("") + ) + bClient.index(binary) shouldBe true + bClient.get[Binary](uuid) match { + case Some(result) => + val decoded = decodeBase64(result.content) + val out = Paths.get(s"/tmp/${path.getFileName}") + val fos = Files.newOutputStream(out) + fos.write(decoded) + fos.close() + hashFile(out).getOrElse("") shouldBe binary.md5 + case _ => fail("no result found for \"" + uuid + "\"") + } + } + } + + "Aggregations" should "work" in { + pClient.createIndex("person10") shouldBe true + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "keyword" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + logger.info(s"mapping: $mapping") + pClient.setMapping("person10", mapping) shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person10", "_doc", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + pClient.flush("person10") + + indices should contain only "person10" + + blockUntilCount(3, "person10") + + "person10" should haveCount(3) + + pClient.get[Person]("A16", Some("person10")) match { + case Some(p) => + p.uuid shouldBe "A16" + p.birthDate shouldBe "1969-05-09" + case None => fail("Person A16 not found") + } + + // test distinct count aggregation + pClient + .aggregate( + "select count(distinct p.uuid) as c from person10 p" + ) + .complete() match { + case Success(s) => s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===(3d) + case Failure(f) => fail(f.getMessage) + } + + // test count aggregation + pClient.aggregate("select count(p.uuid) as c from person10 p").complete() match { + case Success(s) => s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===(3d) + case Failure(f) => fail(f.getMessage) + } + + // test max aggregation on date field + pClient.aggregate("select max(p.birthDate) as c from person10 p").complete() match { + case Success(s) => + s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===( + LocalDate.parse("1969-05-09").toEpochDay.toDouble * 3600 * 24 * 1000 + ) + case Failure(f) => fail(f.getMessage) + } + + // test min aggregation on date field + pClient.aggregate("select min(p.birthDate) as c from person10 p").complete() match { + case Success(s) => + s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===( + LocalDate.parse("1967-11-21").toEpochDay.toDouble * 3600 * 24 * 1000 + ) + case Failure(f) => fail(f.getMessage) + } + + // test avg aggregation on date field + pClient.aggregate("select avg(p.birthDate) as c from person10 p").complete() match { + case Success(s) => + s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===( + LocalDateTime + .parse("1968-05-17T08:00:00.000Z", DateTimeFormatter.ISO_OFFSET_DATE_TIME) + .toInstant(ZoneOffset.UTC) + .toEpochMilli + ) + case Failure(f) => fail(f.getMessage) + } + + // test sum aggregation on integer field + pClient + .aggregate( + "select sum(p.childrenCount) as c from person10 p" + ) + .complete() match { + case Success(s) => + s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===(2d) + case Failure(f) => fail(f.getMessage) + } + + } + + "Nested queries" should "work" in { + parentClient.createIndex("parent") shouldBe true + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "keyword" + | }, + | "createdDate": { + | "type": "date", + | "null_value": "1970-01-01" + | }, + | "lastUpdated": { + | "type": "date", + | "null_value": "1970-01-01" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + logger.info(s"mapping: $mapping") + parentClient.setMapping("parent", mapping) shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("parent", "_doc", 1000) + val indices = + parentClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + parentClient.flush("parent") + parentClient.refresh("parent") + + indices should contain only "parent" + + blockUntilCount(3, "parent") + + "parent" should haveCount(3) + + val parents = parentClient.search[Parent]("select * from parent") + assert(parents.size == 3) + + val results = parentClient.searchWithInnerHits[Parent, Child]( + """SELECT + | p.uuid, + | p.name, + | p.birthDate, + | p.children, + | inner_children.name, + | inner_children.birthDate + |FROM + | parent as p, + | UNNEST(p.children) as inner_children + |WHERE + | inner_children.name is not null AND p.uuid = 'A16' + |""".stripMargin, + "inner_children" + ) + results.size shouldBe 1 + val result = results.head + result._1.uuid shouldBe "A16" + result._1.children.size shouldBe 2 + result._2.size shouldBe 2 + result._2.map(_.name) should contain allOf ("Steve Gumble", "Josh Gumble") + result._2.map( + _.birthDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) + ) should contain allOf ("1999-05-09", "2002-05-09") + result._2.map(_.parentId) should contain only "A16" + } +} diff --git a/es7/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSpec.scala b/es7/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSpec.scala new file mode 100644 index 00000000..b8aeba67 --- /dev/null +++ b/es7/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSpec.scala @@ -0,0 +1,28 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.RestHighLevelProviders.{ + BinaryProvider, + ParentProvider, + PersonProvider, + SampleProvider +} +import app.softnetwork.elastic.model.{Binary, Parent, Sample} +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.persistence.person.model.Person + +class RestHighLevelClientSpec extends ElasticClientSpec { + + lazy val pClient: ElasticProvider[Person] with ElasticClientApi = new PersonProvider( + elasticConfig + ) + lazy val sClient: ElasticProvider[Sample] with ElasticClientApi = new SampleProvider( + elasticConfig + ) + lazy val bClient: ElasticProvider[Binary] with ElasticClientApi = new BinaryProvider( + elasticConfig + ) + + override def parentClient: ElasticProvider[Parent] with ElasticClientApi = new ParentProvider( + elasticConfig + ) +} diff --git a/es7/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelProviders.scala b/es7/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelProviders.scala new file mode 100644 index 00000000..4356c541 --- /dev/null +++ b/es7/testkit/src/test/scala/app/softnetwork/elastic/client/RestHighLevelProviders.scala @@ -0,0 +1,51 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.model.{Binary, Parent, Sample} +import app.softnetwork.elastic.persistence.query.RestHighLevelClientProvider +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.person.model.Person +import com.typesafe.config.Config +import org.elasticsearch.client.RestHighLevelClient + +object RestHighLevelProviders { + + class PersonProvider(es: Config) + extends RestHighLevelClientProvider[Person] + with ManifestWrapper[Person] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val restHighLevelClient: RestHighLevelClient = apply() + } + + class SampleProvider(es: Config) + extends RestHighLevelClientProvider[Sample] + with ManifestWrapper[Sample] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val restHighLevelClient: RestHighLevelClient = apply() + } + + class BinaryProvider(es: Config) + extends RestHighLevelClientProvider[Binary] + with ManifestWrapper[Binary] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val restHighLevelClient: RestHighLevelClient = apply() + } + + class ParentProvider(es: Config) + extends RestHighLevelClientProvider[Parent] + with ManifestWrapper[Parent] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val restHighLevelClient: RestHighLevelClient = apply() + } +} diff --git a/es7/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala b/es7/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala new file mode 100644 index 00000000..55e82460 --- /dev/null +++ b/es7/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala @@ -0,0 +1,13 @@ +package app.softnetwork.elastic.model + +import app.softnetwork.persistence.model.Timestamped + +import java.time.Instant + +case class Binary( + uuid: String, + var createdDate: Instant = Instant.now(), + var lastUpdated: Instant = Instant.now(), + content: String, + md5: String +) extends Timestamped diff --git a/es7/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala b/es7/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala new file mode 100644 index 00000000..ace5d9b8 --- /dev/null +++ b/es7/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala @@ -0,0 +1,47 @@ +package app.softnetwork.elastic.model + +import app.softnetwork.persistence.{generateUUID, now} +import app.softnetwork.persistence.model.Timestamped +import app.softnetwork.time._ + +import java.time.{Instant, LocalDate} + +case class Parent( + uuid: String, + name: String, + birthDate: LocalDate, + children: Seq[Child] = Seq.empty[Child] +) extends Timestamped { + def addChild(child: Child): Parent = copy(children = children :+ child) + lazy val createdDate: Instant = Instant.now() + lazy val lastUpdated: Instant = Instant.now() +} + +case class Child(name: String, birthDate: LocalDate, parentId: String) + +object Parent { + def apply(name: String, birthDate: LocalDate): Parent = + apply( + generateUUID(), + name, + birthDate + ) + + def apply(uuid: String, name: String, birthDate: LocalDate): Parent = + apply( + uuid, + name, + birthDate, + Seq.empty[Child] + ) + + def apply(uuid: String, name: String, birthDate: LocalDate, children: Seq[Child]): Parent = { + Parent( + uuid = uuid, + name = name, + birthDate = birthDate, + children = children + ) + } + +} diff --git a/es7/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala b/es7/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala new file mode 100644 index 00000000..bf9cf5b3 --- /dev/null +++ b/es7/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala @@ -0,0 +1,13 @@ +package app.softnetwork.elastic.model + +import app.softnetwork.persistence.model.Timestamped + +import java.time.Instant + +/** Created by smanciot on 12/04/2020. + */ +case class Sample( + uuid: String, + var createdDate: Instant = Instant.now(), + var lastUpdated: Instant = Instant.now() +) extends Timestamped diff --git a/es7/testkit/src/test/scala/app/softnetwork/persistence/person/RestHighLevelClientPersonHandlerSpec.scala b/es7/testkit/src/test/scala/app/softnetwork/persistence/person/RestHighLevelClientPersonHandlerSpec.scala new file mode 100644 index 00000000..fbce2cf8 --- /dev/null +++ b/es7/testkit/src/test/scala/app/softnetwork/persistence/person/RestHighLevelClientPersonHandlerSpec.scala @@ -0,0 +1,35 @@ +package app.softnetwork.persistence.person + +import akka.actor.typed.ActorSystem +import app.softnetwork.elastic.client.rest.RestHighLevelClientApi +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.person.model.Person +import app.softnetwork.persistence.person.query.PersonToExternalProcessorStream +import app.softnetwork.persistence.query.{ + ExternalPersistenceProvider, + PersonToElasticProcessorStream +} +import com.typesafe.config.Config +import org.slf4j.{Logger, LoggerFactory} + +class RestHighLevelClientPersonHandlerSpec extends ElasticPersonTestKit { + + override def externalPersistenceProvider: ExternalPersistenceProvider[Person] = + new ElasticProvider[Person] with RestHighLevelClientApi with ManifestWrapper[Person] { + override protected val manifestWrapper: ManifestW = ManifestW() + override lazy val config: Config = RestHighLevelClientPersonHandlerSpec.this.elasticConfig + } + + override def person2ExternalProcessorStream: ActorSystem[_] => PersonToExternalProcessorStream = + sys => + new PersonToElasticProcessorStream with RestHighLevelClientApi { + override val forTests: Boolean = true + override protected val manifestWrapper: ManifestW = ManifestW() + override implicit def system: ActorSystem[_] = sys + override def log: Logger = LoggerFactory getLogger getClass.getName + override lazy val config: Config = RestHighLevelClientPersonHandlerSpec.this.elasticConfig + } + + override def log: Logger = LoggerFactory getLogger getClass.getName +} diff --git a/es8/build.sbt b/es8/build.sbt new file mode 100644 index 00000000..28252611 --- /dev/null +++ b/es8/build.sbt @@ -0,0 +1,7 @@ +import SoftClient4es.* +organization := "app.softnetwork.elastic" +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}" +publish / skip := true +Compile / sources := Nil +Test / sources := Nil + diff --git a/es8/java/build.sbt b/es8/java/build.sbt new file mode 100644 index 00000000..53f53322 --- /dev/null +++ b/es8/java/build.sbt @@ -0,0 +1,8 @@ +import SoftClient4es.* + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-java-client" + +libraryDependencies ++= javaClientDependencies(elasticSearchVersion.value) ++ + elastic4sDependencies(elasticSearchVersion.value) diff --git a/es8/java/persistence/build.sbt b/es8/java/persistence/build.sbt new file mode 100644 index 00000000..4c7dc991 --- /dev/null +++ b/es8/java/persistence/build.sbt @@ -0,0 +1,6 @@ +import SoftClient4es.{elasticSearchMajorVersion, elasticSearchVersion} + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-java-persistence" + diff --git a/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticsearchClientProvider.scala b/es8/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticsearchClientProvider.scala similarity index 100% rename from java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticsearchClientProvider.scala rename to es8/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticsearchClientProvider.scala diff --git a/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/State2ElasticProcessorStreamWithJavaProvider.scala b/es8/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/State2ElasticProcessorStreamWithJavaProvider.scala similarity index 100% rename from java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/State2ElasticProcessorStreamWithJavaProvider.scala rename to es8/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/State2ElasticProcessorStreamWithJavaProvider.scala diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala new file mode 100644 index 00000000..7938b1a8 --- /dev/null +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala @@ -0,0 +1,1000 @@ +package app.softnetwork.elastic.client.java + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Flow +import app.softnetwork.elastic.client._ +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} +import app.softnetwork.elastic.{client, sql} +import app.softnetwork.persistence.model.Timestamped +import app.softnetwork.serialization.serialization +import co.elastic.clients.elasticsearch.core.bulk.{ + BulkOperation, + BulkResponseItem, + DeleteOperation, + IndexOperation, + UpdateAction, + UpdateOperation +} +import co.elastic.clients.elasticsearch.core.msearch.{ + MultisearchBody, + MultisearchHeader, + RequestItem +} +import co.elastic.clients.elasticsearch.core._ +import co.elastic.clients.elasticsearch.core.reindex.{Destination, Source} +import co.elastic.clients.elasticsearch.indices.update_aliases.{Action, AddAction, RemoveAction} +import co.elastic.clients.elasticsearch.indices.{ExistsRequest => IndexExistsRequest, _} +import co.elastic.clients.json.jackson.JacksonJsonpMapper +import com.google.gson.{Gson, JsonParser} + +import _root_.java.io.{StringReader, StringWriter} +import _root_.java.util.{Map => JMap} +import scala.collection.JavaConverters._ +import org.json4s.Formats + +import scala.concurrent.{ExecutionContext, Future, Promise} +import scala.language.implicitConversions +import scala.util.{Failure, Success, Try} + +trait ElasticsearchClientApi + extends ElasticClientApi + with ElasticsearchClientIndicesApi + with ElasticsearchClientAliasApi + with ElasticsearchClientSettingsApi + with ElasticsearchClientMappingApi + with ElasticsearchClientRefreshApi + with ElasticsearchClientFlushApi + with ElasticsearchClientCountApi + with ElasticsearchClientSingleValueAggregateApi + with ElasticsearchClientIndexApi + with ElasticsearchClientUpdateApi + with ElasticsearchClientDeleteApi + with ElasticsearchClientGetApi + with ElasticsearchClientSearchApi + with ElasticsearchClientBulkApi + +trait ElasticsearchClientIndicesApi extends IndicesApi with ElasticsearchClientCompanion { + override def createIndex(index: String, settings: String): Boolean = { + tryOrElse( + apply() + .indices() + .create( + new CreateIndexRequest.Builder() + .index(index) + .settings(new IndexSettings.Builder().withJson(new StringReader(settings)).build()) + .build() + ) + .acknowledged(), + false + )(logger) + } + + override def deleteIndex(index: String): Boolean = { + tryOrElse( + apply() + .indices() + .delete(new DeleteIndexRequest.Builder().index(index).build()) + .acknowledged(), + false + )(logger) + } + + override def openIndex(index: String): Boolean = { + tryOrElse( + apply().indices().open(new OpenRequest.Builder().index(index).build()).acknowledged(), + false + )(logger) + } + + override def closeIndex(index: String): Boolean = { + tryOrElse( + apply().indices().close(new CloseIndexRequest.Builder().index(index).build()).acknowledged(), + false + )(logger) + } + + override def reindex( + sourceIndex: String, + targetIndex: String, + refresh: Boolean = true + ): Boolean = { + val failures = apply() + .reindex( + new ReindexRequest.Builder() + .source(new Source.Builder().index(sourceIndex).build()) + .dest(new Destination.Builder().index(targetIndex).build()) + .refresh(refresh) + .build() + ) + .failures() + .asScala + .map(_.cause().reason()) + if (failures.nonEmpty) { + logger.error( + s"Reindexing from $sourceIndex to $targetIndex failed with errors: ${failures.take(100).mkString(", ")}" + ) + } + failures.isEmpty + } + + override def indexExists(index: String): Boolean = { + tryOrElse( + apply() + .indices() + .exists( + new IndexExistsRequest.Builder().index(index).build() + ) + .value(), + false + )(logger) + } +} + +trait ElasticsearchClientAliasApi extends AliasApi with ElasticsearchClientCompanion { + override def addAlias(index: String, alias: String): Boolean = { + tryOrElse( + apply() + .indices() + .updateAliases( + new UpdateAliasesRequest.Builder() + .actions( + new Action.Builder() + .add(new AddAction.Builder().index(index).alias(alias).build()) + .build() + ) + .build() + ) + .acknowledged(), + false + )(logger) + } + + override def removeAlias(index: String, alias: String): Boolean = { + tryOrElse( + apply() + .indices() + .updateAliases( + new UpdateAliasesRequest.Builder() + .actions( + new Action.Builder() + .remove(new RemoveAction.Builder().index(index).alias(alias).build()) + .build() + ) + .build() + ) + .acknowledged(), + false + )(logger) + } +} + +trait ElasticsearchClientSettingsApi extends SettingsApi with ElasticsearchClientCompanion { + _: ElasticsearchClientIndicesApi => + + override def updateSettings(index: String, settings: String): Boolean = { + tryOrElse( + apply() + .indices() + .putSettings( + new PutIndicesSettingsRequest.Builder() + .index(index) + .settings(new IndexSettings.Builder().withJson(new StringReader(settings)).build()) + .build() + ) + .acknowledged(), + false + )(logger) + } + + override def loadSettings(index: String): String = { + tryOrElse( + Option( + apply() + .indices() + .getSettings( + new GetIndicesSettingsRequest.Builder().index(index).build() + ) + .result() + .get(index) + ).map { value => + val mapper = new JacksonJsonpMapper() + val writer = new StringWriter() + val generator = mapper.jsonProvider().createGenerator(writer) + mapper.serialize(value.settings().index(), generator) + generator.close() + writer.toString + }, + None + )(logger).getOrElse("{}") + } +} + +trait ElasticsearchClientMappingApi + extends MappingApi + with ElasticsearchClientIndicesApi + with ElasticsearchClientRefreshApi + with ElasticsearchClientCompanion { + override def setMapping(index: String, mapping: String): Boolean = { + tryOrElse( + apply() + .indices() + .putMapping( + new PutMappingRequest.Builder().index(index).withJson(new StringReader(mapping)).build() + ) + .acknowledged(), + false + )(logger) + } + + override def getMapping(index: String): String = { + tryOrElse( + { + Option( + apply() + .indices() + .getMapping( + new GetMappingRequest.Builder().index(index).build() + ) + .result() + .get(index) + ).map { value => + val mapper = new JacksonJsonpMapper() + val writer = new StringWriter() + val generator = mapper.jsonProvider().createGenerator(writer) + mapper.serialize(value, generator) + generator.close() + writer.toString + } + }, + None + )(logger).getOrElse(s""""{$index: {"mappings": {}}}""") + } +} + +trait ElasticsearchClientRefreshApi extends RefreshApi with ElasticsearchClientCompanion { + override def refresh(index: String): Boolean = { + tryOrElse( + apply() + .indices() + .refresh( + new RefreshRequest.Builder().index(index).build() + ) + .shards() + .failed() + .intValue() == 0, + false + )(logger) + } +} + +trait ElasticsearchClientFlushApi extends FlushApi with ElasticsearchClientCompanion { + override def flush(index: String, force: Boolean = true, wait: Boolean = true): Boolean = { + tryOrElse( + apply() + .indices() + .flush( + new FlushRequest.Builder().index(index).force(force).waitIfOngoing(wait).build() + ) + .shards() + .failed() + .intValue() == 0, + false + )(logger) + } +} + +trait ElasticsearchClientCountApi extends CountApi with ElasticsearchClientCompanion { + override def count(query: client.JSONQuery): Option[Double] = { + tryOrElse( + Option( + apply() + .count( + new CountRequest.Builder().index(query.indices.asJava).build() + ) + .count() + .toDouble + ), + None + )(logger) + } + + override def countAsync(query: client.JSONQuery)(implicit + ec: ExecutionContext + ): Future[Option[Double]] = { + fromCompletableFuture( + async() + .count( + new CountRequest.Builder().index(query.indices.asJava).build() + ) + ).map(response => Option(response.count().toDouble)) + } +} + +trait ElasticsearchClientSingleValueAggregateApi + extends SingleValueAggregateApi + with ElasticsearchClientCountApi { + private[this] def aggregateValue(value: Double, valueAsString: String): AggregateValue = + if (valueAsString.nonEmpty) StringValue(valueAsString) + else NumericValue(value) + + override def aggregate( + sqlQuery: SQLQuery + )(implicit ec: ExecutionContext): Future[Seq[SingleValueAggregateResult]] = { + val aggregations: Seq[ElasticAggregation] = sqlQuery + val futures = for (aggregation <- aggregations) yield { + val promise: Promise[SingleValueAggregateResult] = Promise() + val field = aggregation.field + val sourceField = aggregation.sourceField + val aggType = aggregation.aggType + val aggName = aggregation.aggName + val query = aggregation.query.getOrElse("") + val sources = aggregation.sources + sourceField match { + case "_id" if aggType.sql == "count" => + countAsync( + JSONQuery( + query, + collection.immutable.Seq(sources: _*), + collection.immutable.Seq.empty[String] + ) + ).onComplete { + case Success(result) => + promise.success( + SingleValueAggregateResult( + field, + aggType, + NumericValue(result.getOrElse(0d)), + None + ) + ) + case Failure(f) => + logger.error(f.getMessage, f.fillInStackTrace()) + promise.success( + SingleValueAggregateResult(field, aggType, EmptyValue, Some(f.getMessage)) + ) + } + promise.future + case _ => + val jsonQuery = JSONQuery( + query, + collection.immutable.Seq(sources: _*), + collection.immutable.Seq.empty[String] + ) + import jsonQuery._ + logger.info( + s"Aggregating with query: ${jsonQuery.query} on indices: ${indices.mkString(", ")}" + ) + // Create a parser for the query + Try( + apply().search( + new SearchRequest.Builder() + .index(indices.asJava) + .withJson( + new StringReader(jsonQuery.query) + ) + .build() + ) + ) match { + case Success(response) => + logger.debug( + s"Aggregation response: ${response.toString}" + ) + val agg = aggName.split("\\.").last + + val itAgg = aggName.split("\\.").iterator + + var root = + if (aggregation.nested) { + response.aggregations().get(itAgg.next()).nested().aggregations() + } else { + response.aggregations() + } + + if (aggregation.filtered) { + root = root.get(itAgg.next()).filter().aggregations() + } + + promise.success( + SingleValueAggregateResult( + field, + aggType, + aggType match { + case sql.Count => + NumericValue( + if (aggregation.distinct) { + root.get(agg).cardinality().value().toDouble + } else { + root.get(agg).valueCount().value() + } + ) + case sql.Sum => + NumericValue(root.get(agg).sum().value()) + case sql.Avg => + val avgAgg = root.get(agg).avg() + aggregateValue(avgAgg.value(), avgAgg.valueAsString()) + case sql.Min => + val minAgg = root.get(agg).min() + aggregateValue(minAgg.value(), minAgg.valueAsString()) + case sql.Max => + val maxAgg = root.get(agg).max() + aggregateValue(maxAgg.value(), maxAgg.valueAsString()) + case _ => EmptyValue + }, + None + ) + ) + case Failure(exception) => + logger.error(s"Failed to execute search for aggregation: $aggName", exception) + promise.success( + SingleValueAggregateResult( + field, + aggType, + EmptyValue, + Some(exception.getMessage) + ) + ) + } + promise.future + } + } + Future.sequence(futures) + } +} + +trait ElasticsearchClientIndexApi extends IndexApi with ElasticsearchClientCompanion { + _: ElasticsearchClientRefreshApi => + override def index(index: String, id: String, source: String): Boolean = { + tryOrElse( + apply() + .index( + new IndexRequest.Builder() + .index(index) + .id(id) + .withJson(new StringReader(source)) + .build() + ) + .shards() + .failed() + .intValue() == 0, + false + )(logger) + } + + override def indexAsync(index: String, id: String, source: String)(implicit + ec: ExecutionContext + ): Future[Boolean] = { + fromCompletableFuture( + async() + .index( + new IndexRequest.Builder() + .index(index) + .id(id) + .withJson(new StringReader(source)) + .build() + ) + ).flatMap { response => + if (response.shards().failed().intValue() == 0) { + Future.successful(true) + } else { + Future.failed(new Exception(s"Failed to index document with id: $id in index: $index")) + } + } + } +} + +trait ElasticsearchClientUpdateApi extends UpdateApi with ElasticsearchClientCompanion { + _: ElasticsearchClientRefreshApi => + override def update( + index: String, + id: String, + source: String, + upsert: Boolean + ): Boolean = { + tryOrElse( + apply() + .update( + new UpdateRequest.Builder[JMap[String, Object], JMap[String, Object]]() + .index(index) + .id(id) + .doc(mapper.readValue(source, classOf[JMap[String, Object]])) + .docAsUpsert(upsert) + .build(), + classOf[JMap[String, Object]] + ) + .shards() + .failed() + .intValue() == 0, + false + )(logger) + } + + override def updateAsync(index: String, id: String, source: String, upsert: Boolean)(implicit + ec: ExecutionContext + ): Future[Boolean] = { + fromCompletableFuture( + async() + .update( + new UpdateRequest.Builder[JMap[String, Object], JMap[String, Object]]() + .index(index) + .id(id) + .doc(mapper.readValue(source, classOf[JMap[String, Object]])) + .docAsUpsert(upsert) + .build(), + classOf[JMap[String, Object]] + ) + ).flatMap { response => + if (response.shards().failed().intValue() == 0) { + Future.successful(true) + } else { + Future.failed(new Exception(s"Failed to update document with id: $id in index: $index")) + } + } + } +} + +trait ElasticsearchClientDeleteApi extends DeleteApi with ElasticsearchClientCompanion { + _: ElasticsearchClientRefreshApi => + + override def delete(uuid: String, index: String): Boolean = { + tryOrElse( + apply() + .delete( + new DeleteRequest.Builder().index(index).id(uuid).build() + ) + .shards() + .failed() + .intValue() == 0, + false + )(logger) + } + + override def deleteAsync(uuid: String, index: String)(implicit + ec: ExecutionContext + ): Future[Boolean] = { + fromCompletableFuture( + async() + .delete( + new DeleteRequest.Builder().index(index).id(uuid).build() + ) + ).flatMap { response => + if (response.shards().failed().intValue() == 0) { + Future.successful(true) + } else { + Future.failed(new Exception(s"Failed to delete document with id: $uuid in index: $index")) + } + } + } + +} + +trait ElasticsearchClientGetApi extends GetApi with ElasticsearchClientCompanion { + + def get[U <: Timestamped]( + id: String, + index: Option[String] = None, + maybeType: Option[String] = None + )(implicit m: Manifest[U], formats: Formats): Option[U] = { + Try( + apply().get( + new GetRequest.Builder() + .index( + index.getOrElse( + maybeType.getOrElse( + m.runtimeClass.getSimpleName.toLowerCase + ) + ) + ) + .id(id) + .build(), + classOf[JMap[String, Object]] + ) + ) match { + case Success(response) => + if (response.found()) { + val source = mapper.writeValueAsString(response.source()) + logger.debug(s"Deserializing response $source for id: $id, index: ${index + .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}") + // Deserialize the source string to the expected type + // Note: This assumes that the source is a valid JSON representation of U + // and that the serialization library is capable of handling it. + Try(serialization.read[U](source)) match { + case Success(value) => Some(value) + case Failure(f) => + logger.error( + s"Failed to deserialize response $source for id: $id, index: ${index + .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}", + f + ) + None + } + } else { + None + } + case Failure(f) => + logger.error( + s"Failed to get document with id: $id, index: ${index + .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}", + f + ) + None + } + } + + override def getAsync[U <: Timestamped]( + id: String, + index: Option[String] = None, + maybeType: Option[String] = None + )(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats): Future[Option[U]] = { + fromCompletableFuture( + async() + .get( + new GetRequest.Builder() + .index( + index.getOrElse( + maybeType.getOrElse( + m.runtimeClass.getSimpleName.toLowerCase + ) + ) + ) + .id(id) + .build(), + classOf[JMap[String, Object]] + ) + ).flatMap { + case response if response.found() => + val source = mapper.writeValueAsString(response.source()) + logger.debug(s"Deserializing response $source for id: $id, index: ${index + .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}") + // Deserialize the source string to the expected type + // Note: This assumes that the source is a valid JSON representation of U + // and that the serialization library is capable of handling it. + Try(serialization.read[U](source)) match { + case Success(value) => Future.successful(Some(value)) + case Failure(f) => + logger.error( + s"Failed to deserialize response $source for id: $id, index: ${index + .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}", + f + ) + Future.successful(None) + } + case _ => Future.successful(None) + } + Future { + this.get[U](id, index, maybeType) + } + } +} + +trait ElasticsearchClientSearchApi extends SearchApi with ElasticsearchClientCompanion { + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + implicitly[ElasticSearchRequest](sqlSearch).query + + override def search[U]( + jsonQuery: JSONQuery + )(implicit m: Manifest[U], formats: Formats): List[U] = { + import jsonQuery._ + logger.info(s"Searching with query: $query on indices: ${indices.mkString(", ")}") + val response = apply().search( + new SearchRequest.Builder() + .index(indices.asJava) + .withJson( + new StringReader(query) + ) + .build(), + classOf[JMap[String, Object]] + ) + if (response.hits().total().value() > 0) { + response + .hits() + .hits() + .asScala + .flatMap { hit => + val source = mapper.writeValueAsString(hit.source()) + logger.debug(s"Deserializing hit: $source") + Try(serialization.read[U](source)).toOption.orElse { + logger.error( + s"Failed to deserialize hit: $source" + ) + None + } + } + .toList + } else { + List.empty[U] + } + } + + override def searchAsync[U]( + sqlQuery: SQLQuery + )(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats): Future[List[U]] = { + val jsonQuery: JSONQuery = sqlQuery + import jsonQuery._ + fromCompletableFuture( + async() + .search( + new SearchRequest.Builder() + .index(indices.asJava) + .withJson(new StringReader(query)) + .build(), + classOf[JMap[String, Object]] + ) + ).flatMap { + case response if response.hits().total().value() > 0 => + Future.successful( + response + .hits() + .hits() + .asScala + .map { hit => + val source = mapper.writeValueAsString(hit.source()) + logger.debug(s"Deserializing hit: $source") + serialization.read[U](source) + } + .toList + ) + case _ => + logger.warn( + s"No hits found for query: ${sqlQuery.query} on indices: ${indices.mkString(", ")}" + ) + Future.successful(List.empty[U]) + } + } + + override def searchWithInnerHits[U, I](jsonQuery: JSONQuery, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[(U, List[I])] = { + import jsonQuery._ + logger.info(s"Searching with query: $query on indices: ${indices.mkString(", ")}") + val response = apply() + .search( + new SearchRequest.Builder() + .index(indices.asJava) + .withJson( + new StringReader(query) + ) + .build(), + classOf[JMap[String, Object]] + ) + val results = response + .hits() + .hits() + .asScala + .toList + if (results.nonEmpty) { + results.flatMap { hit => + val hitSource = hit.source() + Option(hitSource) + .map(mapper.writeValueAsString) + .flatMap { source => + logger.debug(s"Deserializing hit: $source") + Try(serialization.read[U](source)) match { + case Success(mainObject) => + Some(mainObject) + case Failure(f) => + logger.error( + s"Failed to deserialize hit: $source for query: $query on indices: ${indices.mkString(", ")}", + f + ) + None + } + } + .map { mainObject => + val innerHits = hit + .innerHits() + .asScala + .get(innerField) + .map(_.hits().hits().asScala.toList) + .getOrElse(Nil) + val innerObjects = innerHits.flatMap { innerHit => + val mapper = new JacksonJsonpMapper() + val writer = new StringWriter() + val generator = mapper.jsonProvider().createGenerator(writer) + mapper.serialize(innerHit, generator) + generator.close() + val innerSource = writer.toString + logger.debug(s"Processing inner hit: $innerSource") + val json = new JsonParser().parse(innerSource).getAsJsonObject + val gson = new Gson() + Try(serialization.read[I](gson.toJson(json.get("_source")))) match { + case Success(innerObject) => Some(innerObject) + case Failure(f) => + logger.error(s"Failed to deserialize inner hit: $innerSource", f) + None + } + } + (mainObject, innerObjects) + } + } + } else { + logger.warn(s"No hits found for query: $query on indices: ${indices.mkString(", ")}") + List.empty[(U, List[I])] + } + } + + override def multiSearch[U]( + jsonQueries: JSONQueries + )(implicit m: Manifest[U], formats: Formats): List[List[U]] = { + import jsonQueries._ + + val items = queries.map { query => + new RequestItem.Builder() + .header(new MultisearchHeader.Builder().index(query.indices.asJava).build()) + .body(new MultisearchBody.Builder().withJson(new StringReader(query.query)).build()) + .build() + } + + val request = new MsearchRequest.Builder().searches(items.asJava).build() + val responses = apply().msearch(request, classOf[JMap[String, Object]]) + + responses.responses().asScala.toList.map { + case response if response.isFailure => + logger.error(s"Error in multi search: ${response.failure().error().reason()}") + List.empty[U] + + case response => + response + .result() + .hits() + .hits() + .asScala + .toList + .map(hit => serialization.read[U](mapper.writeValueAsString(hit.source()))) + } + } + + override def multiSearchWithInnerHits[U, I](jsonQueries: JSONQueries, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[List[(U, List[I])]] = { + import jsonQueries._ + val items = queries.map { query => + new RequestItem.Builder() + .header(new MultisearchHeader.Builder().index(query.indices.asJava).build()) + .body(new MultisearchBody.Builder().withJson(new StringReader(query.query)).build()) + .build() + } + + val request = new MsearchRequest.Builder().searches(items.asJava).build() + val responses = apply().msearch(request, classOf[JMap[String, Object]]) + + responses.responses().asScala.toList.map { + case response if response.isFailure => + logger.error(s"Error in multi search: ${response.failure().error().reason()}") + List.empty[(U, List[I])] + + case response => + Try( + new JsonParser().parse(response.result().toString).getAsJsonObject ~> [U, I] innerField + ) match { + case Success(s) => s + case Failure(f) => + logger.error(f.getMessage, f) + List.empty + } + } + } + +} + +trait ElasticsearchClientBulkApi + extends ElasticsearchClientRefreshApi + with ElasticsearchClientSettingsApi + with ElasticsearchClientIndicesApi + with BulkApi { + override type A = BulkOperation + override type R = BulkResponse + + override def toBulkAction(bulkItem: BulkItem): A = { + import bulkItem._ + + action match { + case BulkAction.UPDATE => + new BulkOperation.Builder() + .update( + new UpdateOperation.Builder() + .index(index) + .id(id.orNull) + .action( + new UpdateAction.Builder[JMap[String, Object], JMap[String, Object]]() + .doc(mapper.readValue(body, classOf[JMap[String, Object]])) + .docAsUpsert(true) + .build() + ) + .build() + ) + .build() + + case BulkAction.DELETE => + val deleteId = id.getOrElse { + throw new IllegalArgumentException(s"Missing id for delete on index $index") + } + new BulkOperation.Builder() + .delete(new DeleteOperation.Builder().index(index).id(deleteId).build()) + .build() + + case _ => + new BulkOperation.Builder() + .index( + new IndexOperation.Builder[JMap[String, Object]]() + .index(index) + .id(id.orNull) + .document(mapper.readValue(body, classOf[JMap[String, Object]])) + .build() + ) + .build() + } + } + override def bulkResult: Flow[R, Set[String], NotUsed] = + Flow[BulkResponse] + .named("result") + .map(result => { + val items = result.items().asScala.toList + val grouped = items.groupBy(_.index()) + val indices = grouped.keys.toSet + for (index <- indices) { + logger + .info(s"Bulk operation succeeded for index $index with ${grouped(index).length} items.") + } + indices + }) + + override def bulk(implicit + bulkOptions: BulkOptions, + system: ActorSystem + ): Flow[Seq[A], R, NotUsed] = { + val parallelism = Math.max(1, bulkOptions.balance) + Flow[Seq[A]] + .named("bulk") + .mapAsyncUnordered[R](parallelism) { items => + val request = + new BulkRequest.Builder().index(bulkOptions.index).operations(items.asJava).build() + Try(apply().bulk(request)) match { + case Success(response) if response.errors() => + val failedItems = response.items().asScala.filter(_.status() >= 400) + if (failedItems.nonEmpty) { + val errorMessages = + failedItems.map(i => s"${i.id()} - ${i.error().reason()}").mkString(", ") + Future.failed(new Exception(s"Bulk operation failed for items: $errorMessages")) + } else { + Future.successful(response) + } + case Success(response) => + Future.successful(response) + case Failure(exception) => + logger.error("Bulk operation failed", exception) + Future.failed(exception) + } + } + } + + private[this] def toBulkElasticResultItem(i: BulkResponseItem): BulkElasticResultItem = + new BulkElasticResultItem { + override def index: String = i.index() + } + + override implicit def toBulkElasticAction(a: BulkOperation): BulkElasticAction = + new BulkElasticAction { + override def index: String = { + a match { + case op if op.isIndex => op.index().index() + case op if op.isDelete => op.delete().index() + case op if op.isUpdate => op.update().index() + case _ => + throw new IllegalArgumentException(s"Unsupported bulk operation type: ${a.getClass}") + } + } + } + + override implicit def toBulkElasticResult(r: BulkResponse): BulkElasticResult = { + new BulkElasticResult { + override def items: List[BulkElasticResultItem] = + r.items().asScala.toList.map(toBulkElasticResultItem) + } + } +} diff --git a/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientCompanion.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientCompanion.scala similarity index 94% rename from java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientCompanion.scala rename to es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientCompanion.scala index 956ff0fa..ead36f44 100644 --- a/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientCompanion.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientCompanion.scala @@ -6,17 +6,19 @@ import co.elastic.clients.json.jackson.JacksonJsonpMapper import co.elastic.clients.transport.rest_client.RestClientTransport import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.ClassTagExtensions -import com.typesafe.scalalogging.StrictLogging import org.apache.http.HttpHost import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} import org.apache.http.impl.client.BasicCredentialsProvider import org.apache.http.impl.nio.client.HttpAsyncClientBuilder import org.elasticsearch.client.{RestClient, RestClientBuilder} +import org.slf4j.{Logger, LoggerFactory} import java.util.concurrent.CompletableFuture import scala.concurrent.{Future, Promise} -trait ElasticsearchClientCompanion extends StrictLogging { +trait ElasticsearchClientCompanion { + + val logger: Logger = LoggerFactory getLogger getClass.getName def elasticConfig: ElasticConfig diff --git a/es8/testkit/build.sbt b/es8/testkit/build.sbt new file mode 100644 index 00000000..94b22a86 --- /dev/null +++ b/es8/testkit/build.sbt @@ -0,0 +1,42 @@ +import SoftClient4es.* + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-testkit" + +libraryDependencies ++= elastic4sTestkitDependencies(elasticSearchVersion.value) ++ Seq( + "org.apache.logging.log4j" % "log4j-api" % Versions.log4j, + // "org.apache.logging.log4j" % "log4j-slf4j-impl" % Versions.log4j, + "org.apache.logging.log4j" % "log4j-core" % Versions.log4j, + "app.softnetwork.persistence" %% "persistence-core-testkit" % Versions.genericPersistence, + "org.testcontainers" % "elasticsearch" % Versions.testContainers excludeAll (jacksonExclusions: _*) +) + +val testJavaOptions = { + val heapSize = sys.env.getOrElse("HEAP_SIZE", "1g") + val extraTestJavaArgs = Seq( + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED" + ).mkString(" ") + s"-Xmx$heapSize -Xss4m -XX:ReservedCodeCacheSize=128m -Dfile.encoding=UTF-8 $extraTestJavaArgs" + .split(" ") + .toSeq +} + +Test / javaOptions ++= testJavaOptions + +// Required by the Test container framework +Test / fork := true diff --git a/es8/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/es8/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala new file mode 100644 index 00000000..cdb90e30 --- /dev/null +++ b/es8/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -0,0 +1,194 @@ +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Flow +import app.softnetwork.elastic.sql.SQLQuery +import org.json4s.Formats +import app.softnetwork.persistence.model.Timestamped +import org.slf4j.{Logger, LoggerFactory} + +import scala.concurrent.{ExecutionContext, Future} +import scala.language.implicitConversions +import scala.reflect.ClassTag + +/** Created by smanciot on 12/04/2020. + */ +trait MockElasticClientApi extends ElasticClientApi { + + protected lazy val log: Logger = LoggerFactory getLogger getClass.getName + + protected val elasticDocuments: ElasticDocuments = new ElasticDocuments() {} + + override def toggleRefresh(index: String, enable: Boolean): Boolean = true + + override def setReplicas(index: String, replicas: Int): Boolean = true + + override def updateSettings(index: String, settings: String) = true + + override def addAlias(index: String, alias: String): Boolean = true + + /** Remove an alias from the given index. + * + * @param index + * - the name of the index + * @param alias + * - the name of the alias + * @return + * true if the alias was removed successfully, false otherwise + */ + override def removeAlias(index: String, alias: String): Boolean = true + + override def createIndex(index: String, settings: String): Boolean = true + + override def setMapping(index: String, mapping: String): Boolean = true + + override def deleteIndex(index: String): Boolean = true + + override def closeIndex(index: String): Boolean = true + + override def openIndex(index: String): Boolean = true + + /** Reindex from source index to target index. + * + * @param sourceIndex + * - the name of the source index + * @param targetIndex + * - the name of the target index + * @param refresh + * - true to refresh the target index after reindexing, false otherwise + * @return + * true if the reindexing was successful, false otherwise + */ + override def reindex(sourceIndex: String, targetIndex: String, refresh: Boolean = true): Boolean = + true + + /** Check if an index exists. + * + * @param index + * - the name of the index to check + * @return + * true if the index exists, false otherwise + */ + override def indexExists(index: String): Boolean = false + + override def count(jsonQuery: JSONQuery): Option[Double] = + throw new UnsupportedOperationException + + override def get[U <: Timestamped]( + id: String, + index: Option[String] = None, + maybeType: Option[String] = None + )(implicit m: Manifest[U], formats: Formats): Option[U] = + elasticDocuments.get(id).asInstanceOf[Option[U]] + + override def search[U](sqlQuery: SQLQuery)(implicit m: Manifest[U], formats: Formats): List[U] = + elasticDocuments.getAll.toList.asInstanceOf[List[U]] + + override def multiSearch[U]( + jsonQueries: JSONQueries + )(implicit m: Manifest[U], formats: Formats): List[List[U]] = + throw new UnsupportedOperationException + + override def index(index: String, id: String, source: String): Boolean = + throw new UnsupportedOperationException + + override def update[U <: Timestamped]( + entity: U, + index: Option[String] = None, + maybeType: Option[String] = None, + upsert: Boolean = true + )(implicit u: ClassTag[U], formats: Formats): Boolean = { + elasticDocuments.createOrUpdate(entity) + true + } + + override def update( + index: String, + id: String, + source: String, + upsert: Boolean + ): Boolean = { + log.warn(s"MockElasticClient - $id not updated for $source") + false + } + + override def delete(uuid: String, index: String): Boolean = { + if (elasticDocuments.get(uuid).isDefined) { + elasticDocuments.delete(uuid) + true + } else { + false + } + } + + override def refresh(index: String): Boolean = true + + override def flush(index: String, force: Boolean, wait: Boolean): Boolean = true + + override type A = this.type + + override def bulk(implicit + bulkOptions: BulkOptions, + system: ActorSystem + ): Flow[Seq[A], R, NotUsed] = + throw new UnsupportedOperationException + + override def bulkResult: Flow[R, Set[String], NotUsed] = + throw new UnsupportedOperationException + + override type R = this.type + + override def toBulkAction(bulkItem: BulkItem): A = + throw new UnsupportedOperationException + + override implicit def toBulkElasticAction(a: A): BulkElasticAction = + throw new UnsupportedOperationException + + override implicit def toBulkElasticResult(r: R): BulkElasticResult = + throw new UnsupportedOperationException + + override def multiSearchWithInnerHits[U, I](jsonQueries: JSONQueries, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[List[(U, List[I])]] = List.empty + + override def search[U](jsonQuery: JSONQuery)(implicit m: Manifest[U], formats: Formats): List[U] = + List.empty + + override def searchWithInnerHits[U, I](jsonQuery: JSONQuery, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[(U, List[I])] = List.empty + + override def getMapping(index: String): String = + throw new UnsupportedOperationException + + override def aggregate(sqlQuery: SQLQuery)(implicit + ec: ExecutionContext + ): Future[Seq[SingleValueAggregateResult]] = + throw new UnsupportedOperationException + + override def loadSettings(index: String): String = + throw new UnsupportedOperationException +} + +trait ElasticDocuments { + + private[this] var documents: Map[String, Timestamped] = Map() + + def createOrUpdate(entity: Timestamped): Unit = { + documents = documents.updated(entity.uuid, entity) + } + + def delete(uuid: String): Unit = { + documents = documents - uuid + } + + def getAll: Iterable[Timestamped] = documents.values + + def get(uuid: String): Option[Timestamped] = documents.get(uuid) + +} diff --git a/es8/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala b/es8/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala new file mode 100644 index 00000000..b2e88a55 --- /dev/null +++ b/es8/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala @@ -0,0 +1,58 @@ +package app.softnetwork.elastic.scalatest + +import org.scalatest.Suite +import org.testcontainers.containers.BindMode +//import org.testcontainers.containers.wait.strategy.Wait +import org.testcontainers.elasticsearch.ElasticsearchContainer +import org.testcontainers.utility.DockerImageName + +import java.nio.file.Files +import java.time.Duration + +/** Created by smanciot on 28/06/2018. + */ +trait ElasticDockerTestKit extends ElasticTestKit { _: Suite => + + override lazy val elasticURL: String = s"http://${elasticContainer.getHttpHostAddress}" + + lazy val localExecution: Boolean = sys.props.get("LOCAL_EXECUTION") match { + case Some("true") => true + case _ => false + } + + lazy val elasticContainer: ElasticsearchContainer = { + val tmpDir = + if (localExecution) { + val tmp = Files.createTempDirectory("es-tmp") + tmp.toFile.setWritable(true, false) + tmp.toAbsolutePath.toString + } else { + "/tmp" + } + Console.println(s"Using temporary directory for Elasticsearch: $tmpDir") + val container = new ElasticsearchContainer( + DockerImageName + .parse("docker.elastic.co/elasticsearch/elasticsearch") + .withTag(elasticVersion) + ) + container.addEnv("ES_TMPDIR", "/usr/share/elasticsearch/tmp") + container.addEnv("discovery.type", "single-node") + container.addEnv("xpack.security.enabled", "false") + container.addEnv("xpack.ml.enabled", "false") + container.addEnv("xpack.watcher.enabled", "false") + container.addEnv("xpack.graph.enabled", "false") + container.addFileSystemBind( + tmpDir, + "/usr/share/elasticsearch/tmp", + BindMode.READ_WRITE + ) + // container.addEnv("ES_JAVA_OPTS", "-Xms1024m -Xmx1024m") + // container.setWaitStrategy(Wait.forHttp("/").forStatusCode(200)) + container.withStartupTimeout(Duration.ofMinutes(2)) + } + + override def start(): Unit = elasticContainer.start() + + override def stop(): Unit = elasticContainer.stop() + +} diff --git a/es8/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala b/es8/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala new file mode 100644 index 00000000..26ef9e90 --- /dev/null +++ b/es8/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala @@ -0,0 +1,347 @@ +package app.softnetwork.elastic.scalatest + +import app.softnetwork.concurrent.scalatest.CompletionTestKit +import app.softnetwork.elastic.Softclient4es8TestkitBuildInfo +import com.sksamuel.elastic4s.http.JavaClient +import com.sksamuel.elastic4s.requests.indexes.admin.RefreshIndexResponse +import com.sksamuel.elastic4s.{ElasticClient, ElasticDsl, Indexes} +import com.typesafe.config.{Config, ConfigFactory} +import org.apache.http.HttpHost +import org.elasticsearch.ResourceAlreadyExistsException +import org.elasticsearch.client.RestClient +import org.elasticsearch.transport.RemoteTransportException +import org.scalatest.{BeforeAndAfterAll, Suite} +import org.scalatest.matchers.{MatchResult, Matcher} +import org.slf4j.Logger + +import scala.util.{Failure, Success, Try} + +/** Created by smanciot on 18/05/2021. + */ +trait ElasticTestKit extends ElasticDsl with CompletionTestKit with BeforeAndAfterAll { _: Suite => + + def log: Logger + + def elasticVersion: String = Softclient4es8TestkitBuildInfo.elasticVersion + + def elasticURL: String + + lazy val elasticConfig: Config = ConfigFactory + .parseString(elasticConfigAsString) + .withFallback(ConfigFactory.load("softnetwork-elastic.conf")) + + lazy val elasticConfigAsString: String = + s""" + |elastic { + | credentials { + | url = "$elasticURL" + | } + | multithreaded = false + | discovery-enabled = false + |} + |""".stripMargin + + lazy val elasticClient: ElasticClient = ElasticClient( + new JavaClient( + RestClient + .builder( + HttpHost.create(elasticURL) + ) + .build() + ) + ) + + def start(): Unit = () + + def stop(): Unit = () + + override def beforeAll(): Unit = { + start() + elasticClient + .execute { + createIndexTemplate("all_templates", "*").settings( + Map("number_of_shards" -> 1, "number_of_replicas" -> 0) + ) + } + .complete() match { + case Success(_) => () + case Failure(f) => throw f + } + } + + override def afterAll(): Unit = { + elasticClient.close() + stop() + } + + // Rewriting methods from IndexMatchers in elastic4s with the ElasticClient + def haveCount(expectedCount: Int): Matcher[String] = + (index: String) => { + elasticClient.execute(search(index).size(0)).complete() match { + case Success(s) => + val count = s.result.totalHits + MatchResult( + count == expectedCount, + s"Index $index had count $count but expected $expectedCount", + s"Index $index had document count $expectedCount" + ) + case Failure(f) => throw f + } + } + + def containDoc(expectedId: String): Matcher[String] = + (index: String) => { + elasticClient.execute(get(index, expectedId)).complete() match { + case Success(s) => + val exists = s.result.exists + MatchResult( + exists, + s"Index $index did not contain expected document $expectedId", + s"Index $index contained document $expectedId" + ) + case Failure(f) => throw f + } + } + + def beCreated(): Matcher[String] = + (index: String) => { + elasticClient.execute(indexExists(index)).complete() match { + case Success(s) => + val exists = s.result.isExists + MatchResult( + exists, + s"Index $index did not exist", + s"Index $index exists" + ) + case Failure(f) => throw f + } + } + + def beEmpty(): Matcher[String] = + (index: String) => { + elasticClient.execute(search(index).size(0)).complete() match { + case Success(s) => + val count = s.result.totalHits + MatchResult( + count == 0, + s"Index $index was not empty", + s"Index $index was empty" + ) + case Failure(f) => throw f + } + } + + // Copy/paste methos HttpElasticSugar as it is not available yet + + // refresh all indexes + def refreshAll(): RefreshIndexResponse = refresh(Indexes.All) + + // refreshes all specified indexes + def refresh(indexes: Indexes): RefreshIndexResponse = { + elasticClient + .execute { + refreshIndex(indexes) + } + .complete() match { + case Success(s) => s.result + case Failure(f) => throw f + } + } + + def blockUntilGreen(): Unit = { + blockUntil("Expected cluster to have green status") { () => + elasticClient + .execute { + clusterHealth() + } + .complete() match { + case Success(s) => s.result.status.toUpperCase == "GREEN" + case Failure(f) => throw f + } + } + } + + def blockUntil(explain: String)(predicate: () => Boolean): Unit = { + blockUntil(explain, 16, 200)(predicate) + } + + def ensureIndexExists(index: String): Unit = { + elasticClient + .execute { + createIndex(index) + } + .complete() match { + case Success(_) => () + case Failure(f) => + f match { + case _: ResourceAlreadyExistsException => // Ok, ignore. + case _: RemoteTransportException => // Ok, ignore. + case other => throw other + } + } + } + + def doesIndexExists(name: String): Boolean = { + elasticClient + .execute { + indexExists(name) + } + .complete() match { + case Success(s) => s.result.isExists + case _ => false + } + } + + def isIndexOpened(name: String): Boolean = { + elasticClient + .execute { + indexStats(name) + } + .complete() match { + case Success(s) => + Try(s.result.indices.contains(name)) match { + case Success(_) => true + case Failure(_) => false + } + case _ => false + } + } + + def isIndexClosed(name: String): Boolean = { + doesIndexExists(name) && !isIndexOpened(name) + } + + def doesAliasExists(name: String): Boolean = { + elasticClient + .execute { + aliasExists(name) + } + .complete() match { + case Success(s) => s.result.isExists + case _ => false + } + } + + def deleteIndex(name: String): Unit = { + if (doesIndexExists(name)) { + elasticClient + .execute { + ElasticDsl.deleteIndex(name) + } + .complete() match { + case Success(_) => () + case Failure(f) => throw f + } + } + } + + def truncateIndex(index: String): Unit = { + deleteIndex(index) + ensureIndexExists(index) + blockUntilEmpty(index) + } + + def blockUntilDocumentExists(id: String, index: String, _type: String): Unit = { + blockUntil(s"Expected to find document $id") { () => + elasticClient + .execute { + get(index, id) + } + .complete() match { + case Success(s) => s.result.exists + case _ => false + } + } + } + + def blockUntilCount(expected: Long, index: String): Unit = { + blockUntil(s"Expected count of $expected") { () => + elasticClient + .execute { + search(index).matchAllQuery().size(0) + } + .complete() match { + case Success(s) => expected <= s.result.totalHits + case Failure(f) => throw f + } + } + } + + /** Will block until the given index and optional types have at least the given number of + * documents. + */ + def blockUntilCount(expected: Long, index: String, types: String*): Unit = { + blockUntil(s"Expected count of $expected") { () => + elasticClient + .execute { + search(index).matchAllQuery().size(0) + } + .complete() match { + case Success(s) => expected <= s.result.totalHits + case Failure(f) => throw f + } + } + } + + def blockUntilExactCount(expected: Long, index: String, types: String*): Unit = { + blockUntil(s"Expected count of $expected") { () => + elasticClient + .execute { + search(index).size(0) + } + .complete() match { + case Success(s) => expected == s.result.totalHits + case Failure(f) => throw f + } + } + } + + def blockUntilEmpty(index: String): Unit = { + blockUntil(s"Expected empty index $index") { () => + elasticClient + .execute { + search(Indexes(index)).size(0) + } + .complete() match { + case Success(s) => s.result.totalHits == 0 + case Failure(f) => throw f + } + } + } + + def blockUntilIndexExists(index: String): Unit = { + blockUntil(s"Expected exists index $index") { () => + doesIndexExists(index) + } + } + + def blockUntilIndexNotExists(index: String): Unit = { + blockUntil(s"Expected not exists index $index") { () => + !doesIndexExists(index) + } + } + + def blockUntilAliasExists(alias: String): Unit = { + blockUntil(s"Expected exists alias $alias") { () => + doesAliasExists(alias) + } + } + + def blockUntilDocumentHasVersion( + index: String, + _type: String, + id: String, + version: Long + ): Unit = { + blockUntil(s"Expected document $id to have version $version") { () => + elasticClient + .execute { + get(index, id) + } + .complete() match { + case Success(s) => s.result.version == version + case Failure(f) => throw f + } + } + } +} diff --git a/es8/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala b/es8/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala new file mode 100644 index 00000000..cc0f00ed --- /dev/null +++ b/es8/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala @@ -0,0 +1,15 @@ +package app.softnetwork.persistence.person + +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.persistence.scalatest.InMemoryPersistenceTestKit + +trait ElasticPersonTestKit + extends PersonTestKit + with InMemoryPersistenceTestKit + with ElasticDockerTestKit { + + override def beforeAll(): Unit = { + super.beforeAll() + initAndJoinCluster() + } +} diff --git a/es8/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala b/es8/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala new file mode 100644 index 00000000..d3b6c3f9 --- /dev/null +++ b/es8/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala @@ -0,0 +1,14 @@ +package app.softnetwork.persistence.query + +import app.softnetwork.elastic.client.ElasticClientApi +import app.softnetwork.elastic.persistence.query.{ElasticProvider, State2ElasticProcessorStream} +import app.softnetwork.persistence.person.message.PersonEvent +import app.softnetwork.persistence.person.model.Person +import app.softnetwork.persistence.person.query.PersonToExternalProcessorStream + +trait PersonToElasticProcessorStream + extends State2ElasticProcessorStream[Person, PersonEvent] + with PersonToExternalProcessorStream + with InMemoryJournalProvider + with InMemoryOffsetProvider + with ElasticProvider[Person] { _: ElasticClientApi => } diff --git a/es8/testkit/src/test/resources/application.conf b/es8/testkit/src/test/resources/application.conf new file mode 100644 index 00000000..ba8abfad --- /dev/null +++ b/es8/testkit/src/test/resources/application.conf @@ -0,0 +1,3 @@ +akka.coordinated-shutdown.exit-jvm = off +elastic.multithreaded = false +clustering.port = 0 diff --git a/es8/testkit/src/test/resources/avatar.jpg b/es8/testkit/src/test/resources/avatar.jpg new file mode 100644 index 00000000..7a214ba8 Binary files /dev/null and b/es8/testkit/src/test/resources/avatar.jpg differ diff --git a/es8/testkit/src/test/resources/avatar.pdf b/es8/testkit/src/test/resources/avatar.pdf new file mode 100644 index 00000000..cf44452f Binary files /dev/null and b/es8/testkit/src/test/resources/avatar.pdf differ diff --git a/es8/testkit/src/test/resources/avatar.png b/es8/testkit/src/test/resources/avatar.png new file mode 100644 index 00000000..a11b4dcd Binary files /dev/null and b/es8/testkit/src/test/resources/avatar.png differ diff --git a/es8/testkit/src/test/resources/mapping/person.mustache b/es8/testkit/src/test/resources/mapping/person.mustache new file mode 100644 index 00000000..21829e1c --- /dev/null +++ b/es8/testkit/src/test/resources/mapping/person.mustache @@ -0,0 +1,21 @@ +{ + "properties": { + "uuid": { + "type": "keyword", + "index": true + }, + "name": { + "type": "text", + "analyzer": "search_analyzer" + }, + "birthDate": { + "type": "keyword" + }, + "createdDate": { + "type": "date" + }, + "lastUpdated": { + "type": "date" + } + } +} \ No newline at end of file diff --git a/es8/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/es8/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala new file mode 100644 index 00000000..99debbf5 --- /dev/null +++ b/es8/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala @@ -0,0 +1,895 @@ +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.sql.SQLQuery +import com.fasterxml.jackson.core.JsonParseException +import com.sksamuel.elastic4s.requests.searches.queries.matches.MatchAllQuery +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import app.softnetwork.persistence._ +import app.softnetwork.serialization._ +import app.softnetwork.elastic.model._ +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.persistence.person.model.Person +import com.google.gson.JsonParser +import com.typesafe.scalalogging.StrictLogging +import org.json4s.Formats +import org.slf4j.{Logger, LoggerFactory} + +import _root_.java.io.ByteArrayInputStream +import _root_.java.nio.file.{Files, Paths} +import _root_.java.time.format.DateTimeFormatter +import _root_.java.util.concurrent.TimeUnit +import _root_.java.util.UUID +import scala.concurrent.{Await, ExecutionContextExecutor} +import scala.concurrent.duration.Duration +import scala.util.{Failure, Success} + +/** Created by smanciot on 28/06/2018. + */ +trait ElasticClientSpec + extends AnyFlatSpecLike + with ElasticDockerTestKit + with Matchers + with StrictLogging { + + lazy val log: Logger = LoggerFactory getLogger getClass.getName + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + implicit val executionContext: ExecutionContextExecutor = system.dispatcher + + implicit val formats: Formats = commonFormats + + def pClient: ElasticProvider[Person] with ElasticClientApi + def sClient: ElasticProvider[Sample] with ElasticClientApi + def bClient: ElasticProvider[Binary] with ElasticClientApi + def parentClient: ElasticProvider[Parent] with ElasticClientApi + + import scala.language.implicitConversions + + implicit def toSQLQuery(sqlQuery: String): SQLQuery = SQLQuery(sqlQuery) + + override def beforeAll(): Unit = { + super.beforeAll() + pClient.createIndex("person") + } + + override def afterAll(): Unit = { + Await.result(system.terminate(), Duration(30, TimeUnit.SECONDS)) + super.afterAll() + } + + val persons: List[String] = List( + """ { "uuid": "A12", "name": "Homer Simpson", "birthDate": "1967-11-21", "childrenCount": 0} """, + """ { "uuid": "A14", "name": "Moe Szyslak", "birthDate": "1967-11-21", "childrenCount": 0} """, + """ { "uuid": "A16", "name": "Barney Gumble", "birthDate": "1969-05-09", "childrenCount": 0} """ + ) + + private val personsWithUpsert = + persons :+ """ { "uuid": "A16", "name": "Barney Gumble2", "birthDate": "1969-05-09", "children": [{ "parentId": "A16", "name": "Steve Gumble", "birthDate": "1999-05-09"}, { "parentId": "A16", "name": "Josh Gumble", "birthDate": "2002-05-09"}], "childrenCount": 2 } """ + + val children: List[String] = List( + """ { "parentId": "A16", "name": "Steve Gumble", "birthDate": "1999-05-09"} """, + """ { "parentId": "A16", "name": "Josh Gumble", "birthDate": "1999-05-09"} """ + ) + + "Creating an index and then delete it" should "work fine" in { + pClient.createIndex("create_delete") + blockUntilIndexExists("create_delete") + "create_delete" should beCreated() + + pClient.deleteIndex("create_delete") + blockUntilIndexNotExists("create_delete") + "create_delete" should not(beCreated()) + } + + "Adding an alias and then removing it" should "work" in { + pClient.addAlias("person", "person_alias") + + doesAliasExists("person_alias") shouldBe true + + pClient.removeAlias("person", "person_alias") + + doesAliasExists("person_alias") shouldBe false + } + + private def settings: Map[String, String] = { + elasticClient + .execute { + getSettings("person") + } + .complete() match { + case Success(s) => s.result.settingsForIndex("person") + case Failure(f) => throw f + } + } + + "Toggle refresh" should "work" in { + pClient.toggleRefresh("person", enable = false) + new JsonParser() + .parse(pClient.loadSettings("person")) + .getAsJsonObject + .get("refresh_interval") + .getAsString shouldBe "-1" + // settings.getOrElse("index.refresh_interval", "") shouldBe "-1" + + pClient.toggleRefresh("person", enable = true) +// settings.getOrElse("index.refresh_interval", "") shouldBe "1s" + new JsonParser() + .parse(pClient.loadSettings("person")) + .getAsJsonObject + .get("refresh_interval") + .getAsString shouldBe "1s" + } + + "Opening an index and then closing it" should "work" in { + pClient.openIndex("person") + + isIndexOpened("person") shouldBe true + + pClient.closeIndex("person") + isIndexClosed("person") shouldBe true + } + + "Updating number of replicas" should "work" in { + pClient.setReplicas("person", 3) + settings.getOrElse("index.number_of_replicas", "") shouldBe "3" + + pClient.setReplicas("person", 0) + settings.getOrElse("index.number_of_replicas", "") shouldBe "0" + } + + "Setting a mapping" should "work" in { + pClient.createIndex("person_mapping") + blockUntilIndexExists("person_mapping") + "person_mapping" should beCreated() + + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "text", + | "analyzer": "ngram_analyzer", + | "search_analyzer": "search_analyzer", + | "fields": { + | "raw": { + | "type": "keyword" + | }, + | "fr": { + | "type": "text", + | "analyzer": "french" + | } + | } + | } + | } + |}""".stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + pClient.setMapping("person_mapping", mapping) shouldBe true + + val properties = pClient.getMappingProperties("person_mapping") + logger.info(s"properties: $properties") + MappingComparator.isMappingDifferent( + properties, + mapping + ) shouldBe false + + implicit val bulkOptions: BulkOptions = BulkOptions("person_mapping", "person", 1000) + val indices = pClient.bulk[String](persons.iterator, identity, Some("uuid"), None, None) + refresh(indices) + pClient.flush("person_mapping") + + indices should contain only "person_mapping" + + blockUntilCount(3, "person_mapping") + + "person_mapping" should haveCount(3) + + pClient.search[Person]("select * from person_mapping") match { + case r if r.size == 3 => + r.map(_.uuid) should contain allOf ("A12", "A14", "A16") + case other => fail(other.toString) + } + + pClient.search[Person]("select * from person_mapping where uuid = 'A16'") match { + case r if r.size == 1 => + r.map(_.uuid) should contain only "A16" + case other => fail(other.toString) + } + + pClient.search[Person]("select * from person_mapping where match(name, 'gum')") match { + case r if r.size == 1 => + r.map(_.uuid) should contain only "A16" + case other => fail(other.toString) + } + + pClient.search[Person]( + "select * from person_mapping where uuid <> 'A16' and match(name, 'gum')" + ) match { + case r if r.isEmpty => + case other => fail(other.toString) + } + } + + "Updating a mapping" should "work" in { + val mapping = + """{ + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + pClient.updateMapping("person_migration", mapping) shouldBe true + blockUntilIndexExists("person_migration") + "person_migration" should beCreated() + + implicit val bulkOptions: BulkOptions = BulkOptions("person_migration", "person", 1000) + val indices = pClient.bulk[String](persons.iterator, identity, Some("uuid"), None, None) + refresh(indices) + pClient.flush("person_migration") + + indices should contain only "person_migration" + + blockUntilCount(3, "person_migration") + + "person_migration" should haveCount(3) + + pClient.search[Person]("select * from person_migration where name like '%gum%'") match { + case r if r.isEmpty => + case other => fail(other.toString) + } + + val newMapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "text", + | "analyzer": "ngram_analyzer", + | "search_analyzer": "search_analyzer", + | "fields": { + | "raw": { + | "type": "keyword" + | }, + | "fr": { + | "type": "text", + | "analyzer": "french" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + pClient.shouldUpdateMapping("person_migration", newMapping) shouldBe true + pClient.updateMapping("person_migration", newMapping) shouldBe true + + pClient.search[Person]("select * from person_migration where name like '%gum%'") match { + case r if r.size == 1 => + r.map(_.uuid) should contain only "A16" + case other => fail(other.toString) + } + + } + + "Bulk index valid json without id key and suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person1", "person", 2) + val indices = pClient.bulk[String](persons.iterator, identity, None, None, None) + + indices should contain only "person1" + + blockUntilCount(3, "person1") + + "person1" should haveCount(3) + + val response = elasticClient + .execute { + search("person1").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id should not be h.sourceField("uuid") + } + + response.result.hits.hits + .map( + _.sourceField("name") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble") + } + + "Bulk index valid json with an id key but no suffix key" should "work" in { + elasticClient + .execute( + createIndex("person2").mapping( + properties( + objectField("child").copy(properties = Seq(textField("name").copy(index = Some(false)))) + ) + ) + ) + .complete() match { + case Success(_) => () + case Failure(f) => throw f + } + + implicit val bulkOptions: BulkOptions = BulkOptions("person2", "person", 1000) + val indices = pClient.bulk[String](persons.iterator, identity, Some("uuid"), None, None) + refresh(indices) + pClient.flush("person2") + + indices should contain only "person2" + + blockUntilCount(3, "person2") + + "person2" should haveCount(3) + + val response = elasticClient + .execute { + search("person2").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceField("uuid") + } + + response.result.hits.hits + .map( + _.sourceField("name") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble") + + } + + "Bulk index valid json with an id key and a suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person", "person", 1000) + val indices = + pClient.bulk[String](persons.iterator, identity, Some("uuid"), Some("birthDate"), None, None) + refresh(indices) + + indices should contain allOf ("person-1967-11-21", "person-1969-05-09") + + blockUntilCount(2, "person-1967-11-21") + blockUntilCount(1, "person-1969-05-09") + + "person-1967-11-21" should haveCount(2) + "person-1969-05-09" should haveCount(1) + + val response = elasticClient + .execute { + search("person-1967-11-21", "person-1969-05-09").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceField("uuid") + } + + response.result.hits.hits + .map( + _.sourceField("name") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble") + } + + "Bulk index invalid json with an id key and a suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person_error", "person", 1000) + intercept[JsonParseException] { + val invalidJson = persons :+ "fail" + pClient.bulk[String](invalidJson.iterator, identity, None, None, None) + } + } + + "Bulk upsert valid json with an id key but no suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person4", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person4" + + blockUntilCount(3, "person4") + + "person4" should haveCount(3) + + val response = elasticClient + .execute { + search("person4").query(MatchAllQuery()) + } + .complete() + + logger.info(s"response: ${response.result.hits.hits.mkString("\n")}") + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceAsMap.getOrElse("uuid", "") + } + + response.result.hits.hits + .map( + _.sourceAsMap.getOrElse("name", "") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble2") + } + + "Bulk upsert valid json with an id key and a suffix key" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person5", "person", 1000) + val indices = pClient.bulk[String]( + personsWithUpsert.iterator, + identity, + Some("uuid"), + Some("birthDate"), + None, + Some(true) + ) + refresh(indices) + + indices should contain allOf ("person5-1967-11-21", "person5-1969-05-09") + + blockUntilCount(2, "person5-1967-11-21") + blockUntilCount(1, "person5-1969-05-09") + + "person5-1967-11-21" should haveCount(2) + "person5-1969-05-09" should haveCount(1) + + val response = elasticClient + .execute { + search("person5-1967-11-21", "person5-1969-05-09").query(MatchAllQuery()) + } + .complete() + + response.result.hits.hits.foreach { h => + h.id shouldBe h.sourceAsMap.getOrElse("uuid", "") + } + + response.result.hits.hits + .map( + _.sourceAsMap.getOrElse("name", "") + ) should contain allOf ("Homer Simpson", "Moe Szyslak", "Barney Gumble2") + } + + "Count" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person6", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person6" + + blockUntilCount(3, "person6") + + "person6" should haveCount(3) + + import scala.collection.immutable.Seq + + pClient + .count(JSONQuery("{}", Seq[String]("person6"), Seq[String]())) + .getOrElse(0d) + .toInt should ===(3) + + pClient.countAsync(JSONQuery("{}", Seq[String]("person6"), Seq[String]())).complete() match { + case Success(s) => s.getOrElse(0d).toInt should ===(3) + case Failure(f) => fail(f.getMessage) + } + } + + "Search" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person7", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person7" + + blockUntilCount(3, "person7") + + "person7" should haveCount(3) + + val r1 = pClient.search[Person]("select * from person7") + r1.size should ===(3) + r1.map(_.uuid) should contain allOf ("A12", "A14", "A16") + + pClient.searchAsync[Person]("select * from person7") onComplete { + case Success(r) => + r.size should ===(3) + r.map(_.uuid) should contain allOf ("A12", "A14", "A16") + case Failure(f) => fail(f.getMessage) + } + + val r2 = pClient.search[Person]("select * from person7 where _id=\"A16\"") + r2.size should ===(1) + r2.map(_.uuid) should contain("A16") + + pClient.searchAsync[Person]("select * from person7 where _id=\"A16\"") onComplete { + case Success(r) => + r.size should ===(1) + r.map(_.uuid) should contain("A16") + case Failure(f) => fail(f.getMessage) + } + } + + "Get all" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person8", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person8" + + blockUntilCount(3, "person8") + + "person8" should haveCount(3) + + val response = pClient.search[Person]("select * from person8") + + response.size should ===(3) + + } + + "Get" should "work" in { + implicit val bulkOptions: BulkOptions = BulkOptions("person9", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + + indices should contain only "person9" + + blockUntilCount(3, "person9") + + "person9" should haveCount(3) + + val response = pClient.get[Person]("A16", Some("person9")) + + response.isDefined shouldBe true + response.get.uuid shouldBe "A16" + + pClient.getAsync[Person]("A16", Some("person9")).complete() match { + case Success(r) => + r.isDefined shouldBe true + r.get.uuid shouldBe "A16" + case Failure(f) => fail(f.getMessage) + } + } + + "Index" should "work" in { + val uuid = UUID.randomUUID().toString + val sample = Sample(uuid) + val result = sClient.index(sample) + result shouldBe true + + sClient.indexAsync(sample).complete() match { + case Success(r) => r shouldBe true + case Failure(f) => fail(f.getMessage) + } + + val result2 = sClient.get[Sample](uuid) + result2 match { + case Some(r) => + r.uuid shouldBe uuid + case _ => + fail("Sample not found") + } + } + + "Update" should "work" in { + val uuid = UUID.randomUUID().toString + val sample = Sample(uuid) + val result = sClient.update(sample) + result shouldBe true + + sClient.updateAsync(sample).complete() match { + case Success(r) => r shouldBe true + case Failure(f) => fail(f.getMessage) + } + + val result2 = sClient.get[Sample](uuid) + result2 match { + case Some(r) => + r.uuid shouldBe uuid + case _ => + fail("Sample not found") + } + } + + "Delete" should "work" in { + val uuid = UUID.randomUUID().toString + val sample = Sample(uuid) + val result = sClient.index(sample) + result shouldBe true + + val result2 = sClient.delete(sample.uuid, Some("sample")) + result2 shouldBe true + + sClient.deleteAsync(sample.uuid, Some("sample")).complete() match { + case Success(r) => r shouldBe true + case Failure(f) => fail(f.getMessage) + } + + val result3 = sClient.get[Sample](uuid) + result3.isEmpty shouldBe true + } + + "Index binary data" should "work" in { + bClient.createIndex("binaries") shouldBe true + val mapping = + """{ + | "properties": { + | "lastUpdated": { + | "type": "date" + | }, + | "createdDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "content": { + | "type": "binary" + | }, + | "md5": { + | "type": "keyword" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + logger.info(s"mapping: $mapping") + bClient.setMapping("binaries", mapping) shouldBe true + val mappings = bClient.getMapping("binaries") + logger.info(s"mappings: $mappings") + assert("{\"mappings\":" + mapping + "}" == mappings) + for (uuid <- Seq("png", "jpg", "pdf")) { + val path = + Paths.get(Thread.currentThread().getContextClassLoader.getResource(s"avatar.$uuid").getPath) + import app.softnetwork.utils.ImageTools._ + import app.softnetwork.utils.HashTools._ + import app.softnetwork.utils.Base64Tools._ + val encoded = encodeImageBase64(path).getOrElse("") + val binary = Binary( + uuid, + content = encoded, + md5 = hashStream(new ByteArrayInputStream(decodeBase64(encoded))).getOrElse("") + ) + bClient.index(binary) shouldBe true + bClient.get[Binary](uuid) match { + case Some(result) => + val decoded = decodeBase64(result.content) + val out = Paths.get(s"/tmp/${path.getFileName}") + val fos = Files.newOutputStream(out) + fos.write(decoded) + fos.close() + hashFile(out).getOrElse("") shouldBe binary.md5 + case _ => fail("no result found for \"" + uuid + "\"") + } + } + } + + "Aggregations" should "work" in { + pClient.createIndex("person10") shouldBe true + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "keyword" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + logger.info(s"mapping: $mapping") + pClient.setMapping("person10", mapping) shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("person10", "person", 1000) + val indices = + pClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + pClient.flush("person10") + + indices should contain only "person10" + + blockUntilCount(3, "person10") + + "person10" should haveCount(3) + + pClient.get[Person]("A16", Some("person10")) match { + case Some(p) => + p.uuid shouldBe "A16" + p.birthDate shouldBe "1969-05-09" + case None => fail("Person A16 not found") + } + + // test distinct count aggregation + pClient + .aggregate( + "select count(distinct p.uuid) as c from person10 p" + ) + .complete() match { + case Success(s) => s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===(3d) + case Failure(f) => fail(f.getMessage) + } + + // test count aggregation + pClient.aggregate("select count(p.uuid) as c from person10 p").complete() match { + case Success(s) => s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===(3d) + case Failure(f) => fail(f.getMessage) + } + + // test max aggregation on date field + pClient.aggregate("select max(p.birthDate) as c from person10 p").complete() match { + case Success(s) => + s.headOption.flatMap(_.asStringOption).getOrElse("") should ===("1969-05-09T00:00:00.000Z") + case Failure(f) => fail(f.getMessage) + } + + // test min aggregation on date field + pClient.aggregate("select min(p.birthDate) as c from person10 p").complete() match { + case Success(s) => + s.headOption.flatMap(_.asStringOption).getOrElse("") should ===("1967-11-21T00:00:00.000Z") + case Failure(f) => fail(f.getMessage) + } + + // test avg aggregation on date field + pClient.aggregate("select avg(p.birthDate) as c from person10 p").complete() match { + case Success(s) => + s.headOption.flatMap(_.asStringOption).getOrElse("") should ===("1968-05-17T08:00:00.000Z") + case Failure(f) => fail(f.getMessage) + } + + // test sum aggregation on integer field + pClient + .aggregate( + "select sum(p.childrenCount) as c from person10 p" + ) + .complete() match { + case Success(s) => + s.headOption.flatMap(_.asDoubleOption).getOrElse(0d) should ===(2d) + case Failure(f) => fail(f.getMessage) + } + + } + + "Nested queries" should "work" in { + parentClient.createIndex("parent") shouldBe true + val mapping = + """{ + | "properties": { + | "birthDate": { + | "type": "date" + | }, + | "uuid": { + | "type": "keyword" + | }, + | "name": { + | "type": "keyword" + | }, + | "createdDate": { + | "type": "date", + | "null_value": "1970-01-01" + | }, + | "lastUpdated": { + | "type": "date", + | "null_value": "1970-01-01" + | }, + | "children": { + | "type": "nested", + | "include_in_parent": true, + | "properties": { + | "name": { + | "type": "keyword" + | }, + | "birthDate": { + | "type": "date" + | } + | } + | }, + | "childrenCount": { + | "type": "integer" + | } + | } + |} + """.stripMargin.replaceAll("\n", "").replaceAll("\\s+", "") + logger.info(s"mapping: $mapping") + parentClient.setMapping("parent", mapping) shouldBe true + + implicit val bulkOptions: BulkOptions = BulkOptions("parent", "parent", 1000) + val indices = + parentClient + .bulk[String](personsWithUpsert.iterator, identity, Some("uuid"), None, None, Some(true)) + refresh(indices) + parentClient.flush("parent") + parentClient.refresh("parent") + + indices should contain only "parent" + + blockUntilCount(3, "parent") + + "parent" should haveCount(3) + + val parents = parentClient.search[Parent]("select * from parent") + assert(parents.size == 3) + + val results = parentClient.searchWithInnerHits[Parent, Child]( + """SELECT + | p.uuid, + | p.name, + | p.birthDate, + | p.children, + | inner_children.name, + | inner_children.birthDate + |FROM + | parent as p, + | UNNEST(p.children) as inner_children + |WHERE + | inner_children.name is not null AND p.uuid = 'A16' + |""".stripMargin, + "inner_children" + ) + results.size shouldBe 1 + val result = results.head + result._1.uuid shouldBe "A16" + result._1.children.size shouldBe 2 + result._2.size shouldBe 2 + result._2.map(_.name) should contain allOf ("Steve Gumble", "Josh Gumble") + result._2.map( + _.birthDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) + ) should contain allOf ("1999-05-09", "2002-05-09") + result._2.map(_.parentId) should contain only "A16" + } +} diff --git a/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchClientSpec.scala b/es8/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchClientSpec.scala similarity index 100% rename from testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchClientSpec.scala rename to es8/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchClientSpec.scala diff --git a/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchProviders.scala b/es8/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchProviders.scala similarity index 100% rename from testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchProviders.scala rename to es8/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchProviders.scala diff --git a/es8/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala b/es8/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala new file mode 100644 index 00000000..55e82460 --- /dev/null +++ b/es8/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala @@ -0,0 +1,13 @@ +package app.softnetwork.elastic.model + +import app.softnetwork.persistence.model.Timestamped + +import java.time.Instant + +case class Binary( + uuid: String, + var createdDate: Instant = Instant.now(), + var lastUpdated: Instant = Instant.now(), + content: String, + md5: String +) extends Timestamped diff --git a/es8/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala b/es8/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala new file mode 100644 index 00000000..ace5d9b8 --- /dev/null +++ b/es8/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala @@ -0,0 +1,47 @@ +package app.softnetwork.elastic.model + +import app.softnetwork.persistence.{generateUUID, now} +import app.softnetwork.persistence.model.Timestamped +import app.softnetwork.time._ + +import java.time.{Instant, LocalDate} + +case class Parent( + uuid: String, + name: String, + birthDate: LocalDate, + children: Seq[Child] = Seq.empty[Child] +) extends Timestamped { + def addChild(child: Child): Parent = copy(children = children :+ child) + lazy val createdDate: Instant = Instant.now() + lazy val lastUpdated: Instant = Instant.now() +} + +case class Child(name: String, birthDate: LocalDate, parentId: String) + +object Parent { + def apply(name: String, birthDate: LocalDate): Parent = + apply( + generateUUID(), + name, + birthDate + ) + + def apply(uuid: String, name: String, birthDate: LocalDate): Parent = + apply( + uuid, + name, + birthDate, + Seq.empty[Child] + ) + + def apply(uuid: String, name: String, birthDate: LocalDate, children: Seq[Child]): Parent = { + Parent( + uuid = uuid, + name = name, + birthDate = birthDate, + children = children + ) + } + +} diff --git a/es8/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala b/es8/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala new file mode 100644 index 00000000..bf9cf5b3 --- /dev/null +++ b/es8/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala @@ -0,0 +1,13 @@ +package app.softnetwork.elastic.model + +import app.softnetwork.persistence.model.Timestamped + +import java.time.Instant + +/** Created by smanciot on 12/04/2020. + */ +case class Sample( + uuid: String, + var createdDate: Instant = Instant.now(), + var lastUpdated: Instant = Instant.now() +) extends Timestamped diff --git a/es8/testkit/src/test/scala/app/softnetwork/persistence/person/ElasticsearchClientPersonHandlerSpec.scala b/es8/testkit/src/test/scala/app/softnetwork/persistence/person/ElasticsearchClientPersonHandlerSpec.scala new file mode 100644 index 00000000..834718ec --- /dev/null +++ b/es8/testkit/src/test/scala/app/softnetwork/persistence/person/ElasticsearchClientPersonHandlerSpec.scala @@ -0,0 +1,35 @@ +package app.softnetwork.persistence.person + +import akka.actor.typed.ActorSystem +import app.softnetwork.elastic.client.java.ElasticsearchClientApi +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.person.model.Person +import app.softnetwork.persistence.person.query.PersonToExternalProcessorStream +import app.softnetwork.persistence.query.{ + ExternalPersistenceProvider, + PersonToElasticProcessorStream +} +import com.typesafe.config.Config +import org.slf4j.{Logger, LoggerFactory} + +class ElasticsearchClientPersonHandlerSpec extends ElasticPersonTestKit { + + override def externalPersistenceProvider: ExternalPersistenceProvider[Person] = + new ElasticProvider[Person] with ElasticsearchClientApi with ManifestWrapper[Person] { + override protected val manifestWrapper: ManifestW = ManifestW() + override lazy val config: Config = ElasticsearchClientPersonHandlerSpec.this.elasticConfig + } + + override def person2ExternalProcessorStream: ActorSystem[_] => PersonToExternalProcessorStream = + sys => + new PersonToElasticProcessorStream with ElasticsearchClientApi { + override val forTests: Boolean = true + override protected val manifestWrapper: ManifestW = ManifestW() + override implicit def system: ActorSystem[_] = sys + override def log: Logger = LoggerFactory getLogger getClass.getName + override lazy val config: Config = ElasticsearchClientPersonHandlerSpec.this.elasticConfig + } + + override def log: Logger = LoggerFactory getLogger getClass.getName +} diff --git a/es9/build.sbt b/es9/build.sbt new file mode 100644 index 00000000..28252611 --- /dev/null +++ b/es9/build.sbt @@ -0,0 +1,7 @@ +import SoftClient4es.* +organization := "app.softnetwork.elastic" +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}" +publish / skip := true +Compile / sources := Nil +Test / sources := Nil + diff --git a/es9/java/build.sbt b/es9/java/build.sbt new file mode 100644 index 00000000..53f53322 --- /dev/null +++ b/es9/java/build.sbt @@ -0,0 +1,8 @@ +import SoftClient4es.* + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-java-client" + +libraryDependencies ++= javaClientDependencies(elasticSearchVersion.value) ++ + elastic4sDependencies(elasticSearchVersion.value) diff --git a/es9/java/persistence/build.sbt b/es9/java/persistence/build.sbt new file mode 100644 index 00000000..4c7dc991 --- /dev/null +++ b/es9/java/persistence/build.sbt @@ -0,0 +1,6 @@ +import SoftClient4es.{elasticSearchMajorVersion, elasticSearchVersion} + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-java-persistence" + diff --git a/es9/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticsearchClientProvider.scala b/es9/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticsearchClientProvider.scala new file mode 100644 index 00000000..d83d9408 --- /dev/null +++ b/es9/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticsearchClientProvider.scala @@ -0,0 +1,12 @@ +package app.softnetwork.elastic.persistence.query + +import app.softnetwork.elastic.client.java.ElasticsearchClientApi +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.model.Timestamped + +trait ElasticsearchClientProvider[T <: Timestamped] + extends ElasticProvider[T] + with ElasticsearchClientApi { + _: ManifestWrapper[T] => + +} diff --git a/es9/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/State2ElasticProcessorStreamWithJavaProvider.scala b/es9/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/State2ElasticProcessorStreamWithJavaProvider.scala new file mode 100644 index 00000000..eaf53713 --- /dev/null +++ b/es9/java/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/State2ElasticProcessorStreamWithJavaProvider.scala @@ -0,0 +1,9 @@ +package app.softnetwork.elastic.persistence.query + +import app.softnetwork.persistence.message.CrudEvent +import app.softnetwork.persistence.model.Timestamped +import app.softnetwork.persistence.query.{JournalProvider, OffsetProvider} + +trait State2ElasticProcessorStreamWithJavaProvider[T <: Timestamped, E <: CrudEvent] + extends State2ElasticProcessorStream[T, E] + with ElasticsearchClientProvider[T] { _: JournalProvider with OffsetProvider => } diff --git a/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala similarity index 93% rename from java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala rename to es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala index 729e7fc6..a6cfb3a0 100644 --- a/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientApi.scala @@ -4,7 +4,8 @@ import akka.NotUsed import akka.actor.ActorSystem import akka.stream.scaladsl.Flow import app.softnetwork.elastic.client._ -import app.softnetwork.elastic.sql.SQLQuery +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.{SQLQuery, SQLSearchRequest} import app.softnetwork.elastic.{client, sql} import app.softnetwork.persistence.model.Timestamped import app.softnetwork.serialization.serialization @@ -316,7 +317,8 @@ trait ElasticsearchClientSingleValueAggregateApi override def aggregate( sqlQuery: SQLQuery )(implicit ec: ExecutionContext): Future[Seq[SingleValueAggregateResult]] = { - val futures = for (aggregation <- sqlQuery.aggregations) yield { + val aggregations: Seq[ElasticAggregation] = sqlQuery + val futures = for (aggregation <- aggregations) yield { val promise: Promise[SingleValueAggregateResult] = Promise() val field = aggregation.field val sourceField = aggregation.sourceField @@ -371,7 +373,7 @@ trait ElasticsearchClientSingleValueAggregateApi ) ) match { case Success(response) => - logger.whenDebugEnabled( + logger.debug( s"Aggregation response: ${response.toString}" ) val agg = aggName.split("\\.").last @@ -397,7 +399,7 @@ trait ElasticsearchClientSingleValueAggregateApi case sql.Count => NumericValue( if (aggregation.distinct) { - root.get(agg).cardinality().value() + root.get(agg).cardinality().value().toDouble } else { root.get(agg).valueCount().value() } @@ -587,7 +589,7 @@ trait ElasticsearchClientGetApi extends GetApi with ElasticsearchClientCompanion case Success(response) => if (response.found()) { val source = mapper.writeValueAsString(response.source()) - logger.whenDebugEnabled(s"Deserializing response $source for id: $id, index: ${index + logger.debug(s"Deserializing response $source for id: $id, index: ${index .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}") // Deserialize the source string to the expected type // Note: This assumes that the source is a valid JSON representation of U @@ -638,7 +640,7 @@ trait ElasticsearchClientGetApi extends GetApi with ElasticsearchClientCompanion ).flatMap { case response if response.found() => val source = mapper.writeValueAsString(response.source()) - logger.whenDebugEnabled(s"Deserializing response $source for id: $id, index: ${index + logger.debug(s"Deserializing response $source for id: $id, index: ${index .getOrElse("default")}, type: ${maybeType.getOrElse("_all")}") // Deserialize the source string to the expected type // Note: This assumes that the source is a valid JSON representation of U @@ -662,6 +664,9 @@ trait ElasticsearchClientGetApi extends GetApi with ElasticsearchClientCompanion } trait ElasticsearchClientSearchApi extends SearchApi with ElasticsearchClientCompanion { + override implicit def sqlSearchRequestToJsonQuery(sqlSearch: SQLSearchRequest): String = + implicitly[ElasticSearchRequest](sqlSearch).query + override def search[U]( jsonQuery: JSONQuery )(implicit m: Manifest[U], formats: Formats): List[U] = { @@ -683,7 +688,7 @@ trait ElasticsearchClientSearchApi extends SearchApi with ElasticsearchClientCom .asScala .flatMap { hit => val source = mapper.writeValueAsString(hit.source()) - logger.whenDebugEnabled(s"Deserializing hit: $source") + logger.debug(s"Deserializing hit: $source") Try(serialization.read[U](source)).toOption.orElse { logger.error( s"Failed to deserialize hit: $source" @@ -700,44 +705,36 @@ trait ElasticsearchClientSearchApi extends SearchApi with ElasticsearchClientCom override def searchAsync[U]( sqlQuery: SQLQuery )(implicit m: Manifest[U], ec: ExecutionContext, formats: Formats): Future[List[U]] = { - sqlQuery.search match { - case Some(searchRequest) => - val indices = collection.immutable.Seq(searchRequest.sources: _*) - fromCompletableFuture( - async() - .search( - new SearchRequest.Builder() - .index(indices.asJava) - .withJson(new StringReader(searchRequest.query)) - .build(), - classOf[JMap[String, Object]] - ) - ).flatMap { - case response if response.hits().total().value() > 0 => - Future.successful( - response - .hits() - .hits() - .asScala - .map { hit => - val source = mapper.writeValueAsString(hit.source()) - logger.whenDebugEnabled(s"Deserializing hit: $source") - serialization.read[U](source) - } - .toList - ) - case _ => - logger.warn( - s"No hits found for query: ${sqlQuery.query} on indices: ${indices.mkString(", ")}" - ) - Future.successful(List.empty[U]) - } - case None => - Future.failed( - throw new IllegalArgumentException( - s"SQL query ${sqlQuery.query} does not contain a valid search request" - ) + val jsonQuery: JSONQuery = sqlQuery + import jsonQuery._ + fromCompletableFuture( + async() + .search( + new SearchRequest.Builder() + .index(indices.asJava) + .withJson(new StringReader(query)) + .build(), + classOf[JMap[String, Object]] + ) + ).flatMap { + case response if response.hits().total().value() > 0 => + Future.successful( + response + .hits() + .hits() + .asScala + .map { hit => + val source = mapper.writeValueAsString(hit.source()) + logger.debug(s"Deserializing hit: $source") + serialization.read[U](source) + } + .toList + ) + case _ => + logger.warn( + s"No hits found for query: ${sqlQuery.query} on indices: ${indices.mkString(", ")}" ) + Future.successful(List.empty[U]) } } @@ -769,7 +766,7 @@ trait ElasticsearchClientSearchApi extends SearchApi with ElasticsearchClientCom Option(hitSource) .map(mapper.writeValueAsString) .flatMap { source => - logger.whenDebugEnabled(s"Deserializing hit: $source") + logger.debug(s"Deserializing hit: $source") Try(serialization.read[U](source)) match { case Success(mainObject) => Some(mainObject) @@ -795,7 +792,7 @@ trait ElasticsearchClientSearchApi extends SearchApi with ElasticsearchClientCom mapper.serialize(innerHit, generator) generator.close() val innerSource = writer.toString - logger.whenDebugEnabled(s"Processing inner hit: $innerSource") + logger.debug(s"Processing inner hit: $innerSource") val json = new JsonParser().parse(innerSource).getAsJsonObject val gson = new Gson() Try(serialization.read[I](gson.toJson(json.get("_source")))) match { diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientCompanion.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientCompanion.scala new file mode 100644 index 00000000..ead36f44 --- /dev/null +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/ElasticsearchClientCompanion.scala @@ -0,0 +1,81 @@ +package app.softnetwork.elastic.client.java + +import app.softnetwork.elastic.client.ElasticConfig +import co.elastic.clients.elasticsearch.{ElasticsearchAsyncClient, ElasticsearchClient} +import co.elastic.clients.json.jackson.JacksonJsonpMapper +import co.elastic.clients.transport.rest_client.RestClientTransport +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.ClassTagExtensions +import org.apache.http.HttpHost +import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.client.{RestClient, RestClientBuilder} +import org.slf4j.{Logger, LoggerFactory} + +import java.util.concurrent.CompletableFuture +import scala.concurrent.{Future, Promise} + +trait ElasticsearchClientCompanion { + + val logger: Logger = LoggerFactory getLogger getClass.getName + + def elasticConfig: ElasticConfig + + private var client: Option[ElasticsearchClient] = None + + private var asyncClient: Option[ElasticsearchAsyncClient] = None + + lazy val mapper: ObjectMapper with ClassTagExtensions = new ObjectMapper() with ClassTagExtensions + + def transport: RestClientTransport = { + val credentialsProvider = new BasicCredentialsProvider() + if (elasticConfig.credentials.username.nonEmpty) { + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials( + elasticConfig.credentials.username, + elasticConfig.credentials.password + ) + ) + } + val restClientBuilder: RestClientBuilder = RestClient + .builder( + HttpHost.create(elasticConfig.credentials.url) + ) + .setHttpClientConfigCallback((httpAsyncClientBuilder: HttpAsyncClientBuilder) => + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + ) + new RestClientTransport(restClientBuilder.build(), new JacksonJsonpMapper()) + } + + def apply(): ElasticsearchClient = { + client match { + case Some(c) => c + case _ => + val c = new ElasticsearchClient(transport) + client = Some(c) + c + } + } + + def async(): ElasticsearchAsyncClient = { + asyncClient match { + case Some(c) => c + case _ => + val c = new ElasticsearchAsyncClient(transport) + asyncClient = Some(c) + c + } + } + + def fromCompletableFuture[T](cf: CompletableFuture[T]): Future[T] = { + val promise = Promise[T]() + cf.whenComplete { (result: T, err: Throwable) => + if (err != null) promise.failure(err) + else promise.success(result) + } + promise.future + } + +} diff --git a/es9/testkit/build.sbt b/es9/testkit/build.sbt new file mode 100644 index 00000000..94b22a86 --- /dev/null +++ b/es9/testkit/build.sbt @@ -0,0 +1,42 @@ +import SoftClient4es.* + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-testkit" + +libraryDependencies ++= elastic4sTestkitDependencies(elasticSearchVersion.value) ++ Seq( + "org.apache.logging.log4j" % "log4j-api" % Versions.log4j, + // "org.apache.logging.log4j" % "log4j-slf4j-impl" % Versions.log4j, + "org.apache.logging.log4j" % "log4j-core" % Versions.log4j, + "app.softnetwork.persistence" %% "persistence-core-testkit" % Versions.genericPersistence, + "org.testcontainers" % "elasticsearch" % Versions.testContainers excludeAll (jacksonExclusions: _*) +) + +val testJavaOptions = { + val heapSize = sys.env.getOrElse("HEAP_SIZE", "1g") + val extraTestJavaArgs = Seq( + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED" + ).mkString(" ") + s"-Xmx$heapSize -Xss4m -XX:ReservedCodeCacheSize=128m -Dfile.encoding=UTF-8 $extraTestJavaArgs" + .split(" ") + .toSeq +} + +Test / javaOptions ++= testJavaOptions + +// Required by the Test container framework +Test / fork := true diff --git a/es9/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala b/es9/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala new file mode 100644 index 00000000..d3c0dfa6 --- /dev/null +++ b/es9/testkit/src/main/scala/app/softnetwork/elastic/client/MockElasticClientApi.scala @@ -0,0 +1,194 @@ +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Flow +import app.softnetwork.elastic.sql.SQLQuery +import app.softnetwork.persistence.model.Timestamped +import org.json4s.Formats +import org.slf4j.{Logger, LoggerFactory} + +import scala.concurrent.{ExecutionContext, Future} +import scala.language.implicitConversions +import scala.reflect.ClassTag + +/** Created by smanciot on 12/04/2020. + */ +trait MockElasticClientApi extends ElasticClientApi { + + protected lazy val log: Logger = LoggerFactory getLogger getClass.getName + + protected val elasticDocuments: ElasticDocuments = new ElasticDocuments() {} + + override def toggleRefresh(index: String, enable: Boolean): Boolean = true + + override def setReplicas(index: String, replicas: Int): Boolean = true + + override def updateSettings(index: String, settings: String) = true + + override def addAlias(index: String, alias: String): Boolean = true + + /** Remove an alias from the given index. + * + * @param index + * - the name of the index + * @param alias + * - the name of the alias + * @return + * true if the alias was removed successfully, false otherwise + */ + override def removeAlias(index: String, alias: String): Boolean = true + + override def createIndex(index: String, settings: String): Boolean = true + + override def setMapping(index: String, mapping: String): Boolean = true + + override def deleteIndex(index: String): Boolean = true + + override def closeIndex(index: String): Boolean = true + + override def openIndex(index: String): Boolean = true + + /** Reindex from source index to target index. + * + * @param sourceIndex + * - the name of the source index + * @param targetIndex + * - the name of the target index + * @param refresh + * - true to refresh the target index after reindexing, false otherwise + * @return + * true if the reindexing was successful, false otherwise + */ + override def reindex(sourceIndex: String, targetIndex: String, refresh: Boolean = true): Boolean = + true + + /** Check if an index exists. + * + * @param index + * - the name of the index to check + * @return + * true if the index exists, false otherwise + */ + override def indexExists(index: String): Boolean = false + + override def count(jsonQuery: JSONQuery): Option[Double] = + throw new UnsupportedOperationException + + override def get[U <: Timestamped]( + id: String, + index: Option[String] = None, + maybeType: Option[String] = None + )(implicit m: Manifest[U], formats: Formats): Option[U] = + elasticDocuments.get(id).asInstanceOf[Option[U]] + + override def search[U](sqlQuery: SQLQuery)(implicit m: Manifest[U], formats: Formats): List[U] = + elasticDocuments.getAll.toList.asInstanceOf[List[U]] + + override def multiSearch[U]( + jsonQueries: JSONQueries + )(implicit m: Manifest[U], formats: Formats): List[List[U]] = + throw new UnsupportedOperationException + + override def index(index: String, id: String, source: String): Boolean = + throw new UnsupportedOperationException + + override def update[U <: Timestamped]( + entity: U, + index: Option[String] = None, + maybeType: Option[String] = None, + upsert: Boolean = true + )(implicit u: ClassTag[U], formats: Formats): Boolean = { + elasticDocuments.createOrUpdate(entity) + true + } + + override def update( + index: String, + id: String, + source: String, + upsert: Boolean + ): Boolean = { + log.warn(s"MockElasticClient - $id not updated for $source") + false + } + + override def delete(uuid: String, index: String): Boolean = { + if (elasticDocuments.get(uuid).isDefined) { + elasticDocuments.delete(uuid) + true + } else { + false + } + } + + override def refresh(index: String): Boolean = true + + override def flush(index: String, force: Boolean, wait: Boolean): Boolean = true + + override type A = this.type + + override def bulk(implicit + bulkOptions: BulkOptions, + system: ActorSystem + ): Flow[Seq[A], R, NotUsed] = + throw new UnsupportedOperationException + + override def bulkResult: Flow[R, Set[String], NotUsed] = + throw new UnsupportedOperationException + + override type R = this.type + + override def toBulkAction(bulkItem: BulkItem): A = + throw new UnsupportedOperationException + + override implicit def toBulkElasticAction(a: A): BulkElasticAction = + throw new UnsupportedOperationException + + override implicit def toBulkElasticResult(r: R): BulkElasticResult = + throw new UnsupportedOperationException + + override def multiSearchWithInnerHits[U, I](jsonQueries: JSONQueries, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[List[(U, List[I])]] = List.empty + + override def search[U](jsonQuery: JSONQuery)(implicit m: Manifest[U], formats: Formats): List[U] = + List.empty + + override def searchWithInnerHits[U, I](jsonQuery: JSONQuery, innerField: String)(implicit + m1: Manifest[U], + m2: Manifest[I], + formats: Formats + ): List[(U, List[I])] = List.empty + + override def getMapping(index: String): String = + throw new UnsupportedOperationException + + override def aggregate(sqlQuery: SQLQuery)(implicit + ec: ExecutionContext + ): Future[Seq[SingleValueAggregateResult]] = + throw new UnsupportedOperationException + + override def loadSettings(index: String): String = + throw new UnsupportedOperationException +} + +trait ElasticDocuments { + + private[this] var documents: Map[String, Timestamped] = Map() + + def createOrUpdate(entity: Timestamped): Unit = { + documents = documents.updated(entity.uuid, entity) + } + + def delete(uuid: String): Unit = { + documents = documents - uuid + } + + def getAll: Iterable[Timestamped] = documents.values + + def get(uuid: String): Option[Timestamped] = documents.get(uuid) + +} diff --git a/es9/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala b/es9/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala new file mode 100644 index 00000000..b2e88a55 --- /dev/null +++ b/es9/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticDockerTestKit.scala @@ -0,0 +1,58 @@ +package app.softnetwork.elastic.scalatest + +import org.scalatest.Suite +import org.testcontainers.containers.BindMode +//import org.testcontainers.containers.wait.strategy.Wait +import org.testcontainers.elasticsearch.ElasticsearchContainer +import org.testcontainers.utility.DockerImageName + +import java.nio.file.Files +import java.time.Duration + +/** Created by smanciot on 28/06/2018. + */ +trait ElasticDockerTestKit extends ElasticTestKit { _: Suite => + + override lazy val elasticURL: String = s"http://${elasticContainer.getHttpHostAddress}" + + lazy val localExecution: Boolean = sys.props.get("LOCAL_EXECUTION") match { + case Some("true") => true + case _ => false + } + + lazy val elasticContainer: ElasticsearchContainer = { + val tmpDir = + if (localExecution) { + val tmp = Files.createTempDirectory("es-tmp") + tmp.toFile.setWritable(true, false) + tmp.toAbsolutePath.toString + } else { + "/tmp" + } + Console.println(s"Using temporary directory for Elasticsearch: $tmpDir") + val container = new ElasticsearchContainer( + DockerImageName + .parse("docker.elastic.co/elasticsearch/elasticsearch") + .withTag(elasticVersion) + ) + container.addEnv("ES_TMPDIR", "/usr/share/elasticsearch/tmp") + container.addEnv("discovery.type", "single-node") + container.addEnv("xpack.security.enabled", "false") + container.addEnv("xpack.ml.enabled", "false") + container.addEnv("xpack.watcher.enabled", "false") + container.addEnv("xpack.graph.enabled", "false") + container.addFileSystemBind( + tmpDir, + "/usr/share/elasticsearch/tmp", + BindMode.READ_WRITE + ) + // container.addEnv("ES_JAVA_OPTS", "-Xms1024m -Xmx1024m") + // container.setWaitStrategy(Wait.forHttp("/").forStatusCode(200)) + container.withStartupTimeout(Duration.ofMinutes(2)) + } + + override def start(): Unit = elasticContainer.start() + + override def stop(): Unit = elasticContainer.stop() + +} diff --git a/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala b/es9/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala similarity index 96% rename from testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala rename to es9/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala index efea2656..96ad823a 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala +++ b/es9/testkit/src/main/scala/app/softnetwork/elastic/scalatest/ElasticTestKit.scala @@ -1,7 +1,7 @@ package app.softnetwork.elastic.scalatest import app.softnetwork.concurrent.scalatest.CompletionTestKit -import app.softnetwork.elastic.ElasticTestkitBuildInfo +import app.softnetwork.elastic.Softclient4es9TestkitBuildInfo import com.sksamuel.elastic4s.http.JavaClient import com.sksamuel.elastic4s.requests.indexes.admin.RefreshIndexResponse import com.sksamuel.elastic4s.{ElasticClient, ElasticDsl, Indexes} @@ -23,7 +23,7 @@ trait ElasticTestKit extends ElasticDsl with CompletionTestKit with BeforeAndAft def log: Logger - def elasticVersion: String = ElasticTestkitBuildInfo.version + def elasticVersion: String = Softclient4es9TestkitBuildInfo.elasticVersion def elasticURL: String @@ -313,19 +313,19 @@ trait ElasticTestKit extends ElasticDsl with CompletionTestKit with BeforeAndAft } def blockUntilIndexExists(index: String): Unit = { - blockUntil(s"Expected exists index $index") { () ⇒ + blockUntil(s"Expected exists index $index") { () => doesIndexExists(index) } } def blockUntilIndexNotExists(index: String): Unit = { - blockUntil(s"Expected not exists index $index") { () ⇒ + blockUntil(s"Expected not exists index $index") { () => !doesIndexExists(index) } } def blockUntilAliasExists(alias: String): Unit = { - blockUntil(s"Expected exists alias $alias") { () ⇒ + blockUntil(s"Expected exists alias $alias") { () => doesAliasExists(alias) } } diff --git a/es9/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala b/es9/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala new file mode 100644 index 00000000..cc0f00ed --- /dev/null +++ b/es9/testkit/src/main/scala/app/softnetwork/persistence/person/ElasticPersonTestKit.scala @@ -0,0 +1,15 @@ +package app.softnetwork.persistence.person + +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.persistence.scalatest.InMemoryPersistenceTestKit + +trait ElasticPersonTestKit + extends PersonTestKit + with InMemoryPersistenceTestKit + with ElasticDockerTestKit { + + override def beforeAll(): Unit = { + super.beforeAll() + initAndJoinCluster() + } +} diff --git a/es9/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala b/es9/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala new file mode 100644 index 00000000..d3b6c3f9 --- /dev/null +++ b/es9/testkit/src/main/scala/app/softnetwork/persistence/query/PersonToElasticProcessorStream.scala @@ -0,0 +1,14 @@ +package app.softnetwork.persistence.query + +import app.softnetwork.elastic.client.ElasticClientApi +import app.softnetwork.elastic.persistence.query.{ElasticProvider, State2ElasticProcessorStream} +import app.softnetwork.persistence.person.message.PersonEvent +import app.softnetwork.persistence.person.model.Person +import app.softnetwork.persistence.person.query.PersonToExternalProcessorStream + +trait PersonToElasticProcessorStream + extends State2ElasticProcessorStream[Person, PersonEvent] + with PersonToExternalProcessorStream + with InMemoryJournalProvider + with InMemoryOffsetProvider + with ElasticProvider[Person] { _: ElasticClientApi => } diff --git a/es9/testkit/src/test/resources/application.conf b/es9/testkit/src/test/resources/application.conf new file mode 100644 index 00000000..ba8abfad --- /dev/null +++ b/es9/testkit/src/test/resources/application.conf @@ -0,0 +1,3 @@ +akka.coordinated-shutdown.exit-jvm = off +elastic.multithreaded = false +clustering.port = 0 diff --git a/es9/testkit/src/test/resources/avatar.jpg b/es9/testkit/src/test/resources/avatar.jpg new file mode 100644 index 00000000..7a214ba8 Binary files /dev/null and b/es9/testkit/src/test/resources/avatar.jpg differ diff --git a/es9/testkit/src/test/resources/avatar.pdf b/es9/testkit/src/test/resources/avatar.pdf new file mode 100644 index 00000000..cf44452f Binary files /dev/null and b/es9/testkit/src/test/resources/avatar.pdf differ diff --git a/es9/testkit/src/test/resources/avatar.png b/es9/testkit/src/test/resources/avatar.png new file mode 100644 index 00000000..a11b4dcd Binary files /dev/null and b/es9/testkit/src/test/resources/avatar.png differ diff --git a/es9/testkit/src/test/resources/mapping/person.mustache b/es9/testkit/src/test/resources/mapping/person.mustache new file mode 100644 index 00000000..21829e1c --- /dev/null +++ b/es9/testkit/src/test/resources/mapping/person.mustache @@ -0,0 +1,21 @@ +{ + "properties": { + "uuid": { + "type": "keyword", + "index": true + }, + "name": { + "type": "text", + "analyzer": "search_analyzer" + }, + "birthDate": { + "type": "keyword" + }, + "createdDate": { + "type": "date" + }, + "lastUpdated": { + "type": "date" + } + } +} \ No newline at end of file diff --git a/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala b/es9/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala similarity index 100% rename from testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala rename to es9/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticClientSpec.scala diff --git a/es9/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchClientSpec.scala b/es9/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchClientSpec.scala new file mode 100644 index 00000000..e0165b77 --- /dev/null +++ b/es9/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchClientSpec.scala @@ -0,0 +1,27 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.ElasticsearchProviders.{ + BinaryProvider, + ParentProvider, + PersonProvider, + SampleProvider +} +import app.softnetwork.elastic.model.{Binary, Parent, Sample} +import app.softnetwork.elastic.persistence.query.ElasticProvider +import app.softnetwork.persistence.person.model.Person + +class ElasticsearchClientSpec extends ElasticClientSpec { + + lazy val pClient: ElasticProvider[Person] with ElasticClientApi = new PersonProvider( + elasticConfig + ) + lazy val sClient: ElasticProvider[Sample] with ElasticClientApi = new SampleProvider( + elasticConfig + ) + lazy val bClient: ElasticProvider[Binary] with ElasticClientApi = new BinaryProvider( + elasticConfig + ) + lazy val parentClient: ElasticProvider[Parent] with ElasticClientApi = new ParentProvider( + elasticConfig + ) +} diff --git a/es9/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchProviders.scala b/es9/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchProviders.scala new file mode 100644 index 00000000..754a3416 --- /dev/null +++ b/es9/testkit/src/test/scala/app/softnetwork/elastic/client/ElasticsearchProviders.scala @@ -0,0 +1,51 @@ +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.model.{Binary, Parent, Sample} +import app.softnetwork.elastic.persistence.query.ElasticsearchClientProvider +import app.softnetwork.persistence.ManifestWrapper +import app.softnetwork.persistence.person.model.Person +import co.elastic.clients.elasticsearch.ElasticsearchClient +import com.typesafe.config.Config + +object ElasticsearchProviders { + + class PersonProvider(es: Config) + extends ElasticsearchClientProvider[Person] + with ManifestWrapper[Person] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val elasticsearchClient: ElasticsearchClient = apply() + } + + class SampleProvider(es: Config) + extends ElasticsearchClientProvider[Sample] + with ManifestWrapper[Sample] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val elasticsearchClient: ElasticsearchClient = apply() + } + + class BinaryProvider(es: Config) + extends ElasticsearchClientProvider[Binary] + with ManifestWrapper[Binary] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val elasticsearchClient: ElasticsearchClient = apply() + } + + class ParentProvider(es: Config) + extends ElasticsearchClientProvider[Parent] + with ManifestWrapper[Parent] { + override protected val manifestWrapper: ManifestW = ManifestW() + + override lazy val config: Config = es + + implicit lazy val elasticsearchClient: ElasticsearchClient = apply() + } +} diff --git a/es9/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala b/es9/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala new file mode 100644 index 00000000..55e82460 --- /dev/null +++ b/es9/testkit/src/test/scala/app/softnetwork/elastic/model/Binary.scala @@ -0,0 +1,13 @@ +package app.softnetwork.elastic.model + +import app.softnetwork.persistence.model.Timestamped + +import java.time.Instant + +case class Binary( + uuid: String, + var createdDate: Instant = Instant.now(), + var lastUpdated: Instant = Instant.now(), + content: String, + md5: String +) extends Timestamped diff --git a/es9/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala b/es9/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala new file mode 100644 index 00000000..ace5d9b8 --- /dev/null +++ b/es9/testkit/src/test/scala/app/softnetwork/elastic/model/Parent.scala @@ -0,0 +1,47 @@ +package app.softnetwork.elastic.model + +import app.softnetwork.persistence.{generateUUID, now} +import app.softnetwork.persistence.model.Timestamped +import app.softnetwork.time._ + +import java.time.{Instant, LocalDate} + +case class Parent( + uuid: String, + name: String, + birthDate: LocalDate, + children: Seq[Child] = Seq.empty[Child] +) extends Timestamped { + def addChild(child: Child): Parent = copy(children = children :+ child) + lazy val createdDate: Instant = Instant.now() + lazy val lastUpdated: Instant = Instant.now() +} + +case class Child(name: String, birthDate: LocalDate, parentId: String) + +object Parent { + def apply(name: String, birthDate: LocalDate): Parent = + apply( + generateUUID(), + name, + birthDate + ) + + def apply(uuid: String, name: String, birthDate: LocalDate): Parent = + apply( + uuid, + name, + birthDate, + Seq.empty[Child] + ) + + def apply(uuid: String, name: String, birthDate: LocalDate, children: Seq[Child]): Parent = { + Parent( + uuid = uuid, + name = name, + birthDate = birthDate, + children = children + ) + } + +} diff --git a/es9/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala b/es9/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala new file mode 100644 index 00000000..bf9cf5b3 --- /dev/null +++ b/es9/testkit/src/test/scala/app/softnetwork/elastic/model/Sample.scala @@ -0,0 +1,13 @@ +package app.softnetwork.elastic.model + +import app.softnetwork.persistence.model.Timestamped + +import java.time.Instant + +/** Created by smanciot on 12/04/2020. + */ +case class Sample( + uuid: String, + var createdDate: Instant = Instant.now(), + var lastUpdated: Instant = Instant.now() +) extends Timestamped diff --git a/testkit/src/test/scala/app/softnetwork/persistence/person/ElasticsearchClientPersonHandlerSpec.scala b/es9/testkit/src/test/scala/app/softnetwork/persistence/person/ElasticsearchClientPersonHandlerSpec.scala similarity index 100% rename from testkit/src/test/scala/app/softnetwork/persistence/person/ElasticsearchClientPersonHandlerSpec.scala rename to es9/testkit/src/test/scala/app/softnetwork/persistence/person/ElasticsearchClientPersonHandlerSpec.scala diff --git a/java/build.sbt b/java/build.sbt deleted file mode 100644 index a4146b35..00000000 --- a/java/build.sbt +++ /dev/null @@ -1,19 +0,0 @@ -organization := "app.softnetwork.elastic" - -name := "elastic-java-client" - -val jacksonExclusions = Seq( - ExclusionRule(organization = "com.fasterxml.jackson.core"), - ExclusionRule(organization = "com.fasterxml.jackson.dataformat"), - ExclusionRule(organization = "com.fasterxml.jackson.datatype"), - ExclusionRule(organization = "com.fasterxml.jackson.module"), - ExclusionRule(organization = "org.codehaus.jackson") -) - -val rest = Seq( - "org.elasticsearch" % "elasticsearch" % Versions.elasticSearch exclude ("org.apache.logging.log4j", "log4j-api"), - "co.elastic.clients" % "elasticsearch-java" % Versions.elasticSearch exclude ("org.elasticsearch", "elasticsearch"), - "org.elasticsearch.client" % "elasticsearch-rest-client" % Versions.elasticSearch -).map(_.excludeAll(jacksonExclusions: _*)) - -libraryDependencies ++= rest diff --git a/java/persistence/build.sbt b/java/persistence/build.sbt deleted file mode 100644 index 07392682..00000000 --- a/java/persistence/build.sbt +++ /dev/null @@ -1,3 +0,0 @@ -organization := "app.softnetwork.elastic" - -name := "elastic-java-persistence" diff --git a/persistence/build.sbt b/persistence/build.sbt index c62f517d..20c98dd7 100644 --- a/persistence/build.sbt +++ b/persistence/build.sbt @@ -1,4 +1,5 @@ -organization := "app.softnetwork.elastic" +import SoftClient4es._ -name := "elastic-persistence" +organization := "app.softnetwork.elastic" +name := "softclient4es-persistence" diff --git a/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala b/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala index 97245c93..145bc672 100644 --- a/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala +++ b/persistence/src/main/scala/app/softnetwork/elastic/persistence/query/ElasticProvider.scala @@ -16,7 +16,7 @@ import scala.util.{Failure, Success, Try} /** Created by smanciot on 16/05/2020. */ -trait ElasticProvider[T <: Timestamped] extends ExternalPersistenceProvider[T] with StrictLogging { +trait ElasticProvider[T <: Timestamped] extends ExternalPersistenceProvider[T] { _: ElasticClientApi with ManifestWrapper[T] => implicit def formats: Formats = commonFormats @@ -121,9 +121,7 @@ trait ElasticProvider[T <: Timestamped] extends ExternalPersistenceProvider[T] w * whether the operation is successful or not */ override def upsertDocument(uuid: String, data: String): Boolean = { - logger.whenDebugEnabled { - logger.debug(s"Upserting document $uuid for index $index with $data") - } + logger.debug(s"Upserting document $uuid for index $index with $data") Try( update( index, diff --git a/project/SoftClient4es.scala b/project/SoftClient4es.scala new file mode 100644 index 00000000..ef1d3381 --- /dev/null +++ b/project/SoftClient4es.scala @@ -0,0 +1,163 @@ +import sbt.* + +trait SoftClient4es { + + lazy val elasticSearchVersion = + settingKey[String]("The version of Elasticsearch used for this module") + + def elasticSearchMajorVersion(esVersion: String): Int = esVersion.split("\\.").head.toInt + + lazy val jacksonExclusions: Seq[ExclusionRule] = Seq( + ExclusionRule(organization = "com.fasterxml.jackson.core"), + ExclusionRule(organization = "com.fasterxml.jackson.dataformat"), + ExclusionRule(organization = "com.fasterxml.jackson.datatype"), + ExclusionRule(organization = "com.fasterxml.jackson.module"), + ExclusionRule(organization = "org.codehaus.jackson") + ) + + lazy val guavaExclusion = ExclusionRule(organization = "com.google.guava", name = "guava") + + lazy val httpComponentsExclusions: Seq[ExclusionRule] = Seq( + ExclusionRule( + organization = "org.apache.httpcomponents", + name = "httpclient", + artifact = "*", + configurations = Vector(ConfigRef("test")), + crossVersion = CrossVersion.disabled + ) + ) + + def jacksonDependencies(esVersion: String): Seq[ModuleID] = { + val jackson2_19 = "2.19.0" + val jackson2_13 = "2.13.3" + val jackson2_12 = "2.12.7" + (elasticSearchMajorVersion(esVersion) match { + case 6 => + Some(jackson2_12) + case 7 => + Some(jackson2_13) + case 8 | 9 => + Some(jackson2_19) + case _ => None + }) match { + case Some(version) => + Seq( + "com.fasterxml.jackson.core" % "jackson-databind" % version, + "com.fasterxml.jackson.core" % "jackson-core" % version, + "com.fasterxml.jackson.core" % "jackson-annotations" % version, + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % version, + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % version, + "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % version, + "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % version, + "com.fasterxml.jackson.module" % "jackson-module-parameter-names" % version, + "com.fasterxml.jackson.module" %% "jackson-module-scala" % version + ) + case None => Seq.empty + } + } + + def elastic4sDependencies(esVersion: String): Seq[ModuleID] = { + elasticSearchMajorVersion(esVersion) match { + case 6 => + Seq( + "com.sksamuel.elastic4s" %% "elastic4s-core" % Versions.elastic64s exclude ("org.elasticsearch", "elasticsearch") exclude ("org.slf4j", "slf4j-api"), + "com.sksamuel.elastic4s" %% "elastic4s-http" % Versions.elastic64s exclude ("org.elasticsearch", "elasticsearch") + ) + case 7 => + Seq( + "com.sksamuel.elastic4s" %% "elastic4s-core" % Versions.elastic74s exclude ("org.elasticsearch", "elasticsearch") exclude ("org.slf4j", "slf4j-api") + ) + case 8 => + Seq( + "nl.gn0s1s" %% "elastic4s-core" % Versions.elastic84s exclude ("org.elasticsearch", "elasticsearch") exclude ("org.slf4j", "slf4j-api") + ) + case 9 => + Seq( + "nl.gn0s1s" %% "elastic4s-core" % Versions.elastic94s exclude ("org.elasticsearch", "elasticsearch") exclude ("org.slf4j", "slf4j-api") + ) + case _ => Seq.empty + } + } + + def elastic4sTestkitDependencies(esVersion: String): Seq[ModuleID] = { + elastic4sDependencies(esVersion) ++ + (elasticSearchMajorVersion(esVersion) match { + case 6 => + Seq( + "com.sksamuel.elastic4s" %% "elastic4s-testkit" % Versions.elastic64s exclude ("org.elasticsearch", "elasticsearch") exclude ("org.slf4j", "slf4j-api"), + "com.sksamuel.elastic4s" %% "elastic4s-embedded" % Versions.elastic64s exclude ("org.elasticsearch", "elasticsearch"), + "pl.allegro.tech" % "embedded-elasticsearch" % "2.10.0" excludeAll (jacksonExclusions: _*) + ) + case 7 => + Seq( + "com.sksamuel.elastic4s" %% "elastic4s-testkit" % Versions.elastic74s exclude ("org.elasticsearch", "elasticsearch") exclude ("org.slf4j", "slf4j-api") + ) + case 8 => + Seq( + "nl.gn0s1s" %% "elastic4s-testkit" % Versions.elastic84s exclude ("org.elasticsearch", "elasticsearch") exclude ("org.slf4j", "slf4j-api") + ) + case 9 => + Seq( + "nl.gn0s1s" %% "elastic4s-testkit" % Versions.elastic94s exclude ("org.elasticsearch", "elasticsearch") exclude ("org.slf4j", "slf4j-api") + ) + case _ => Seq.empty + }) + } + + def elasticDependencies(esVersion: String): Seq[ModuleID] = { + elasticSearchMajorVersion(esVersion) match { + case 6 | 7 | 8 | 9 => + Seq( + "org.elasticsearch" % "elasticsearch" % esVersion exclude ("org.apache.logging.log4j", "log4j-api") exclude ("org.slf4j", "slf4j-api") excludeAll (jacksonExclusions: _*) + ).map(_.excludeAll(jacksonExclusions: _*)) + case _ => Seq.empty + } + } + + def elasticClientDependencies(esVersion: String): Seq[ModuleID] = { + elasticDependencies(esVersion) ++ + (elasticSearchMajorVersion(esVersion) match { + case 6 | 7 | 8 | 9 => + Seq( + "org.elasticsearch.client" % "elasticsearch-rest-client" % esVersion + ).map(_.excludeAll(jacksonExclusions: _*)) + case _ => Seq.empty + }) + } + + def javaClientDependencies(esVersion: String): Seq[ModuleID] = { + elasticClientDependencies(esVersion) ++ + (elasticSearchMajorVersion(esVersion) match { + case 8 | 9 => + Seq( + "co.elastic.clients" % "elasticsearch-java" % esVersion exclude ("org.elasticsearch", "elasticsearch") + ).map(_.excludeAll(jacksonExclusions: _*)) + case _ => Seq.empty + }) + } + + def restClientDependencies(esVersion: String): Seq[ModuleID] = { + elasticClientDependencies(esVersion) ++ + (elasticSearchMajorVersion(esVersion) match { + case 6 | 7 => + Seq( + "org.elasticsearch.client" % "elasticsearch-rest-high-level-client" % esVersion exclude ("org.elasticsearch", "elasticsearch") + ).map(_.excludeAll(jacksonExclusions: _*)) + case _ => Seq.empty + }) + } + + def jestClientDependencies(esVersion: String): Seq[ModuleID] = { + elasticClientDependencies(esVersion) ++ + (elasticSearchMajorVersion(esVersion) match { + case 6 => + Seq( + "io.searchbox" % "jest" % Versions.jest + ).map(_.excludeAll(httpComponentsExclusions ++ Seq(guavaExclusion): _*)) + case _ => Seq.empty + }) + } + +} + +object SoftClient4es extends SoftClient4es diff --git a/project/Versions.scala b/project/Versions.scala index 599adb38..fda845c4 100644 --- a/project/Versions.scala +++ b/project/Versions.scala @@ -1,6 +1,6 @@ object Versions { -// val akka = "2.6.20" + // val akka = "2.6.20" // TODO 2.6.20 -> 2.8.3 val scalatest = "3.2.19" @@ -20,9 +20,23 @@ object Versions { val log4s = "1.8.2" - val elasticSearch = "9.0.3" + val es6 = "6.8.23" - val elastic4s = "9.0.0" + val elastic64s = "6.7.8" + + val jest = "6.3.1" + + val es7 = "7.17.29" + + val elastic74s = "7.17.4" + + val es8 = "8.18.3" + + val elastic84s = "8.18.2" + + val es9 = "9.0.3" + + val elastic94s = "9.0.0" val log4j = "2.8.2" diff --git a/sql/bridge/build.sbt b/sql/bridge/build.sbt new file mode 100644 index 00000000..662d93d6 --- /dev/null +++ b/sql/bridge/build.sbt @@ -0,0 +1,10 @@ +import SoftClient4es.* + +organization := "app.softnetwork.elastic" + +name := s"softclient4es${elasticSearchMajorVersion(elasticSearchVersion.value)}-sql-bridge" + +target := baseDirectory.value / s"target-es${elasticSearchMajorVersion(elasticSearchVersion.value)}" + +libraryDependencies ++= elasticDependencies(elasticSearchVersion.value) ++ + elastic4sDependencies(elasticSearchVersion.value) diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala new file mode 100644 index 00000000..27f05588 --- /dev/null +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticAggregation.scala @@ -0,0 +1,120 @@ +package app.softnetwork.elastic.sql.bridge + +import app.softnetwork.elastic.sql.{ + AggregateFunction, + Avg, + Count, + ElasticBoolQuery, + Max, + Min, + SQLAggregate, + Sum +} +import com.sksamuel.elastic4s.ElasticApi.{ + avgAgg, + cardinalityAgg, + filterAgg, + matchAllQuery, + maxAgg, + minAgg, + nestedAggregation, + sumAgg, + valueCountAgg +} +import com.sksamuel.elastic4s.requests.searches.aggs.Aggregation + +import scala.language.implicitConversions + +case class ElasticAggregation( + aggName: String, + field: String, + sourceField: String, + sources: Seq[String] = Seq.empty, + query: Option[String] = None, + distinct: Boolean = false, + nested: Boolean = false, + filtered: Boolean = false, + aggType: AggregateFunction, + agg: Aggregation +) + +object ElasticAggregation { + def apply(sqlAgg: SQLAggregate): ElasticAggregation = { + import sqlAgg._ + val sourceField = identifier.columnName + + val field = alias match { + case Some(alias) => alias.alias + case _ => sourceField + } + + val distinct = identifier.distinct.isDefined + + val agg = + if (distinct) + s"${function}_distinct_${sourceField.replace(".", "_")}" + else + s"${function}_${sourceField.replace(".", "_")}" + + var aggPath = Seq[String]() + + val _agg = + function match { + case Count => + if (distinct) + cardinalityAgg(agg, sourceField) + else { + valueCountAgg(agg, sourceField) + } + case Min => minAgg(agg, sourceField) + case Max => maxAgg(agg, sourceField) + case Avg => avgAgg(agg, sourceField) + case Sum => sumAgg(agg, sourceField) + } + + def _filtered: Aggregation = filter match { + case Some(f) => + val boolQuery = Option(ElasticBoolQuery(group = true)) + val filteredAgg = s"filtered_agg" + aggPath ++= Seq(filteredAgg) + filterAgg( + filteredAgg, + f.criteria + .map( + _.asFilter(boolQuery) + .query(Set(identifier.innerHitsName).flatten, boolQuery) + ) + .getOrElse(matchAllQuery()) + ) subaggs { + aggPath ++= Seq(agg) + _agg + } + case _ => + aggPath ++= Seq(agg) + _agg + } + + val aggregation = + if (identifier.nested) { + val path = sourceField.split("\\.").head + val nestedAgg = s"nested_$agg" + aggPath ++= Seq(nestedAgg) + nestedAggregation(nestedAgg, path) subaggs { + _filtered + } + } else { + _filtered + } + + ElasticAggregation( + aggPath.mkString("."), + field, + sourceField, + distinct = distinct, + nested = identifier.nested, + filtered = filter.nonEmpty, + aggType = function, + agg = aggregation + ) + } +} diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala new file mode 100644 index 00000000..d6542758 --- /dev/null +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticCriteria.scala @@ -0,0 +1,15 @@ +package app.softnetwork.elastic.sql.bridge + +import app.softnetwork.elastic.sql.SQLCriteria +import com.sksamuel.elastic4s.requests.searches.queries.Query + +case class ElasticCriteria(criteria: SQLCriteria) { + + def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty): Query = { + val query = criteria.boolQuery.copy(group = group) + query + .filter(criteria.asFilter(Option(query))) + .unfilteredMatchCriteria() + .query(innerHitsNames, Option(query)) + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/ElasticMultiSearchRequest.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticMultiSearchRequest.scala similarity index 87% rename from sql/src/main/scala/app/softnetwork/elastic/sql/ElasticMultiSearchRequest.scala rename to sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticMultiSearchRequest.scala index 503c2323..554cdc4a 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/ElasticMultiSearchRequest.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticMultiSearchRequest.scala @@ -1,4 +1,4 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.bridge import com.sksamuel.elastic4s.requests.searches.{MultiSearchBuilderFn, MultiSearchRequest} diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala new file mode 100644 index 00000000..91506b63 --- /dev/null +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticQuery.scala @@ -0,0 +1,74 @@ +package app.softnetwork.elastic.sql.bridge + +import app.softnetwork.elastic.sql.{ + ElasticBoolQuery, + ElasticChild, + ElasticFilter, + ElasticGeoDistance, + ElasticMatch, + ElasticNested, + ElasticParent, + SQLBetween, + SQLExpression, + SQLIn, + SQLIsNotNull, + SQLIsNull +} +import com.sksamuel.elastic4s.ElasticApi.{bool, _} +import com.sksamuel.elastic4s.requests.searches.queries.Query + +case class ElasticQuery(filter: ElasticFilter) { + def query( + innerHitsNames: Set[String] = Set.empty, + currentQuery: Option[ElasticBoolQuery] + ): Query = { + filter match { + case boolQuery: ElasticBoolQuery => + import boolQuery._ + bool( + mustFilters.map(implicitly[ElasticQuery](_).query(innerHitsNames, currentQuery)), + shouldFilters.map(implicitly[ElasticQuery](_).query(innerHitsNames, currentQuery)), + notFilters.map(implicitly[ElasticQuery](_).query(innerHitsNames, currentQuery)) + ) + .filter(innerFilters.map(_.query(innerHitsNames, currentQuery))) + case nested: ElasticNested => + import nested._ + if (innerHitsNames.contains(innerHitsName.getOrElse(""))) { + criteria.asFilter(currentQuery).query(innerHitsNames, currentQuery) + } else { + val boolQuery = Option(ElasticBoolQuery(group = true)) + nestedQuery( + relationType.getOrElse(""), + criteria + .asFilter(boolQuery) + .query(innerHitsNames + innerHitsName.getOrElse(""), boolQuery) + ) /*.scoreMode(ScoreMode.None)*/ + .inner( + innerHits(innerHitsName.getOrElse("")).from(0).size(limit.map(_.limit).getOrElse(3)) + ) + } + case child: ElasticChild => + import child._ + hasChildQuery( + relationType.getOrElse(""), + criteria.asQuery(group = group, innerHitsNames = innerHitsNames) + ) + case parent: ElasticParent => + import parent._ + hasParentQuery( + relationType.getOrElse(""), + criteria.asQuery(group = group, innerHitsNames = innerHitsNames), + score = false + ) + case expression: SQLExpression => expression + case isNull: SQLIsNull => isNull + case isNotNull: SQLIsNotNull => isNotNull + case in: SQLIn[_, _] => in + case between: SQLBetween => between + case geoDistance: ElasticGeoDistance => geoDistance + case matchExpression: ElasticMatch => matchExpression + case other => + throw new IllegalArgumentException(s"Unsupported filter type: ${other.getClass.getName}") + } + } +} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/ElasticSearchRequest.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala similarity index 84% rename from sql/src/main/scala/app/softnetwork/elastic/sql/ElasticSearchRequest.scala rename to sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala index aa2a5b85..1d0530ff 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/ElasticSearchRequest.scala +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/ElasticSearchRequest.scala @@ -1,5 +1,6 @@ -package app.softnetwork.elastic.sql +package app.softnetwork.elastic.sql.bridge +import app.softnetwork.elastic.sql.{SQLCriteria, SQLExcept, SQLField} import com.sksamuel.elastic4s.requests.searches.{SearchBodyBuilderFn, SearchRequest} case class ElasticSearchRequest( diff --git a/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala new file mode 100644 index 00000000..8a8456f1 --- /dev/null +++ b/sql/bridge/src/main/scala/app/softnetwork/elastic/sql/bridge/package.scala @@ -0,0 +1,314 @@ +package app.softnetwork.elastic.sql + +import com.sksamuel.elastic4s.ElasticApi +import com.sksamuel.elastic4s.ElasticApi._ +import com.sksamuel.elastic4s.requests.searches.queries.Query +import com.sksamuel.elastic4s.requests.searches.sort.FieldSort +import com.sksamuel.elastic4s.requests.searches.{ + MultiSearchRequest, + SearchBodyBuilderFn, + SearchRequest +} + +import scala.language.implicitConversions + +package object bridge { + implicit def requestToElasticSearchRequest(request: SQLSearchRequest): ElasticSearchRequest = + ElasticSearchRequest( + request.select.fields, + request.select.except, + request.sources, + request.where.flatMap(_.criteria), + request.limit.map(_.limit), + request, + request.aggregates.map(ElasticAggregation(_)) + ).minScore(request.score) + + implicit def requestToSearchRequest(request: SQLSearchRequest): SearchRequest = { + import request._ + val aggregations = aggregates.map(ElasticAggregation(_)) + var _search: SearchRequest = search("") query { + where.flatMap(_.criteria.map(_.asQuery())).getOrElse(matchAllQuery()) + } sourceInclude fields + + _search = excludes match { + case Nil => _search + case excludes => _search sourceExclude excludes + } + + _search = aggregations match { + case Nil => _search + case _ => _search aggregations { aggregations.map(_.agg) } + } + + _search = orderBy match { + case Some(o) => + _search sortBy o.sorts.map(sort => + sort.order match { + case Some(Desc) => FieldSort(sort.field).desc() + case _ => FieldSort(sort.field).asc() + } + ) + case _ => _search + } + + if (aggregations.nonEmpty && fields.isEmpty) { + _search size 0 + } else { + limit match { + case Some(l) => _search limit l.limit from 0 + case _ => _search + } + } + } + + implicit def requestToMultiSearchRequest( + request: SQLMultiSearchRequest + ): MultiSearchRequest = { + MultiSearchRequest( + request.requests.map(implicitly[SearchRequest](_)) + ) + } + + implicit def expressionToQuery(expression: SQLExpression): Query = { + import expression._ + value match { + case n: SQLNumeric[Any] @unchecked => + operator match { + case _: Ge.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) lt n.sql + case _ => + rangeQuery(identifier.columnName) gte n.sql + } + case _: Gt.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) lte n.sql + case _ => + rangeQuery(identifier.columnName) gt n.sql + } + case _: Le.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) gt n.sql + case _ => + rangeQuery(identifier.columnName) lte n.sql + } + case _: Lt.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) gte n.sql + case _ => + rangeQuery(identifier.columnName) lt n.sql + } + case _: Eq.type => + maybeNot match { + case Some(_) => + not(termQuery(identifier.columnName, n.sql)) + case _ => + termQuery(identifier.columnName, n.sql) + } + case _: Ne.type => + maybeNot match { + case Some(_) => + termQuery(identifier.columnName, n.sql) + case _ => + not(termQuery(identifier.columnName, n.sql)) + } + case _ => matchAllQuery() + } + case l: SQLLiteral => + operator match { + case _: Like.type => + maybeNot match { + case Some(_) => + not(regexQuery(identifier.columnName, toRegex(l.value))) + case _ => + regexQuery(identifier.columnName, toRegex(l.value)) + } + case _: Ge.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) lt l.value + case _ => + rangeQuery(identifier.columnName) gte l.value + } + case _: Gt.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) lte l.value + case _ => + rangeQuery(identifier.columnName) gt l.value + } + case _: Le.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) gt l.value + case _ => + rangeQuery(identifier.columnName) lte l.value + } + case _: Lt.type => + maybeNot match { + case Some(_) => + rangeQuery(identifier.columnName) gte l.value + case _ => + rangeQuery(identifier.columnName) lt l.value + } + case _: Eq.type => + maybeNot match { + case Some(_) => + not(termQuery(identifier.columnName, l.value)) + case _ => + termQuery(identifier.columnName, l.value) + } + case _: Ne.type => + maybeNot match { + case Some(_) => + termQuery(identifier.columnName, l.value) + case _ => + not(termQuery(identifier.columnName, l.value)) + } + case _ => matchAllQuery() + } + case b: SQLBoolean => + operator match { + case _: Eq.type => + maybeNot match { + case Some(_) => + not(termQuery(identifier.columnName, b.value)) + case _ => + termQuery(identifier.columnName, b.value) + } + case _: Ne.type => + maybeNot match { + case Some(_) => + termQuery(identifier.columnName, b.value) + case _ => + not(termQuery(identifier.columnName, b.value)) + } + case _ => matchAllQuery() + } + case _ => matchAllQuery() + } + } + + implicit def isNullToQuery( + isNull: SQLIsNull + ): Query = { + import isNull._ + not(existsQuery(identifier.columnName)) + } + + implicit def isNotNullToQuery( + isNotNull: SQLIsNotNull + ): Query = { + import isNotNull._ + existsQuery(identifier.columnName) + } + + implicit def inToQuery[R, T <: SQLValue[R]](in: SQLIn[R, T]): Query = { + import in._ + val _values: Seq[Any] = values.innerValues + val t = + _values.headOption match { + case Some(_: Double) => + termsQuery(identifier.columnName, _values.asInstanceOf[Seq[Double]]) + case Some(_: Integer) => + termsQuery(identifier.columnName, _values.asInstanceOf[Seq[Integer]]) + case Some(_: Long) => + termsQuery(identifier.columnName, _values.asInstanceOf[Seq[Long]]) + case _ => termsQuery(identifier.columnName, _values.map(_.toString)) + } + maybeNot match { + case Some(_) => not(t) + case _ => t + } + } + + implicit def betweenToQuery( + between: SQLBetween + ): Query = { + import between._ + val r = rangeQuery(identifier.columnName) gte from.value lte to.value + maybeNot match { + case Some(_) => not(r) + case _ => r + } + } + + implicit def geoDistanceToQuery( + geoDistance: ElasticGeoDistance + ): Query = { + import geoDistance._ + geoDistanceQuery(identifier.columnName, lat.value, lon.value) distance distance.value + } + + implicit def matchToQuery( + matchExpression: ElasticMatch + ): Query = { + import matchExpression._ + matchQuery(identifier.columnName, value.value) + } + + implicit def criteriaToElasticCriteria( + criteria: SQLCriteria + ): ElasticCriteria = { + ElasticCriteria( + criteria + ) + } + + implicit def filterToQuery( + filter: ElasticFilter + ): ElasticQuery = { + ElasticQuery(filter) + } + + implicit def sqlQueryToAggregations( + query: SQLQuery + ): Seq[ElasticAggregation] = { + import query._ + request + .map { + case Left(l) => + l.aggregates + .map(ElasticAggregation(_)) + .map(aggregation => { + val queryFiltered = + l.where + .flatMap(_.criteria.map(ElasticCriteria(_).asQuery())) + .getOrElse(matchAllQuery()) + + aggregation.copy( + sources = l.sources, + query = Some( + (aggregation.aggType match { + case Count if aggregation.sourceField.equalsIgnoreCase("_id") => + SearchBodyBuilderFn( + ElasticApi.search("") query { + queryFiltered + } + ) + case _ => + SearchBodyBuilderFn( + ElasticApi.search("") query { + queryFiltered + } + aggregations { + aggregation.agg + } + size 0 + ) + }).string.replace("\"version\":true,", "") /*FIXME*/ + ) + ) + }) + + case _ => Seq.empty + + } + .getOrElse(Seq.empty) + } +} diff --git a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala similarity index 99% rename from sql/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala rename to sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala index c6db8489..2d1db9ec 100644 --- a/sql/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLCriteriaSpec.scala @@ -1,5 +1,6 @@ package app.softnetwork.elastic.sql +import app.softnetwork.elastic.sql.bridge._ import com.sksamuel.elastic4s.ElasticApi.matchAllQuery import com.sksamuel.elastic4s.requests.searches.{SearchBodyBuilderFn, SearchRequest} import org.scalatest.flatspec.AnyFlatSpec diff --git a/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala new file mode 100644 index 00000000..8d2e2271 --- /dev/null +++ b/sql/bridge/src/test/scala/app/softnetwork/elastic/sql/SQLQuerySpec.scala @@ -0,0 +1,773 @@ +package app.softnetwork.elastic.sql + +import app.softnetwork.elastic.sql.bridge._ +import app.softnetwork.elastic.sql.Queries._ +import com.google.gson.{JsonArray, JsonObject, JsonParser, JsonPrimitive} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import scala.jdk.CollectionConverters.IteratorHasAsScala + +/** Created by smanciot on 13/04/17. + */ +class SQLQuerySpec extends AnyFlatSpec with Matchers { + + import scala.language.implicitConversions + + implicit def sqlQueryToRequest(sqlQuery: SQLQuery): ElasticSearchRequest = { + sqlQuery.request match { + case Some(Left(value)) => + value.copy(score = sqlQuery.score) + case None => + throw new IllegalArgumentException( + s"SQL query ${sqlQuery.query} does not contain a valid search request" + ) + } + } + + "SQLQuery" should "perform native count" in { + val results: Seq[ElasticAggregation] = + SQLQuery("select count(t.id) c2 from Table t where t.nom = \"Nom\"") + results.size shouldBe 1 + val result = results.head + result.nested shouldBe false + result.distinct shouldBe false + result.aggName shouldBe "count_id" + result.field shouldBe "c2" + result.sources shouldBe Seq[String]("Table") + result.query.getOrElse("") shouldBe + """|{ + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "nom": { + | "value": "Nom" + | } + | } + | } + | ] + | } + | }, + | "size": 0, + | "aggs": { + | "count_id": { + | "value_count": { + | "field": "id" + | } + | } + | } + |}""".stripMargin.replaceAll("\\s+", "") + } + + it should "perform count distinct" in { + val results: Seq[ElasticAggregation] = + SQLQuery("select count(distinct t.id) as c2 from Table as t where nom = \"Nom\"") + results.size shouldBe 1 + val result = results.head + result.nested shouldBe false + result.distinct shouldBe true + result.aggName shouldBe "count_distinct_id" + result.field shouldBe "c2" + result.sources shouldBe Seq[String]("Table") + result.query.getOrElse("") shouldBe + """|{ + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "nom": { + | "value": "Nom" + | } + | } + | } + | ] + | } + | }, + | "size": 0, + | "aggs": { + | "count_distinct_id": { + | "cardinality": { + | "field": "id" + | } + | } + | } + |}""".stripMargin.replaceAll("\\s+", "") + } + + it should "perform nested count" in { + val results: Seq[ElasticAggregation] = + SQLQuery( + "select count(inner_emails.value) as email from index i, unnest(emails) as inner_emails where i.nom = \"Nom\"" + ) + results.size shouldBe 1 + val result = results.head + result.nested shouldBe true + result.distinct shouldBe false + result.aggName shouldBe "nested_count_emails_value.count_emails_value" + result.field shouldBe "email" + result.sources shouldBe Seq[String]("index") + result.query.getOrElse("") shouldBe + """{ + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "nom": { + | "value": "Nom" + | } + | } + | } + | ] + | } + | }, + | "size": 0, + | "aggs": { + | "nested_count_emails_value": { + | "nested": { + | "path": "emails" + | }, + | "aggs": { + | "count_emails_value": { + | "value_count": { + | "field": "emails.value" + | } + | } + | } + | } + | } + |}""".stripMargin.replaceAll("\\s+", "") + } + + it should "perform nested count with nested criteria" in { + val results: Seq[ElasticAggregation] = + SQLQuery( + "select count(inner_emails.value) as count_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))" + ) + results.size shouldBe 1 + val result = results.head + result.nested shouldBe true + result.distinct shouldBe false + result.aggName shouldBe "nested_count_emails_value.count_emails_value" + result.field shouldBe "count_emails" + result.sources shouldBe Seq[String]("index") + result.query.getOrElse("") shouldBe + """{ + | "query": { + | "bool":{ + | "filter": [ + | { + | "term": { + | "nom": { + | "value": "Nom" + | } + | } + | }, + | { + | "nested": { + | "path": "profiles", + | "query": { + | "terms": { + | "profiles.postalCode": [ + | "75001", + | "75002" + | ] + | } + | }, + | "inner_hits":{"name":"inner_profiles","from":0,"size":3} + | } + | } + | ] + | } + | }, + | "size": 0, + | "aggs": { + | "nested_count_emails_value": { + | "nested": { + | "path": "emails" + | }, + | "aggs": { + | "count_emails_value": { + | "value_count": { + | "field": "emails.value" + | } + | } + | } + | } + | } + |}""".stripMargin.replaceAll("\\s+", "") + } + + it should "perform nested count with filter" in { + val results: Seq[ElasticAggregation] = + SQLQuery( + "select count(inner_emails.value) as count_emails filter[inner_emails.context = \"profile\"] from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where nom = \"Nom\" and (inner_profiles.postalCode in (\"75001\",\"75002\"))" + ) + results.size shouldBe 1 + val result = results.head + result.nested shouldBe true + result.distinct shouldBe false + result.aggName shouldBe "nested_count_emails_value.filtered_agg.count_emails_value" + result.field shouldBe "count_emails" + result.sources shouldBe Seq[String]("index") + result.query.getOrElse("") shouldBe + """{ + | "query": { + | "bool":{ + | "filter": [ + | { + | "term": { + | "nom": { + | "value": "Nom" + | } + | } + | }, + | { + | "nested": { + | "path": "profiles", + | "query": { + | "terms": { + | "profiles.postalCode": [ + | "75001", + | "75002" + | ] + | } + | }, + | "inner_hits":{"name":"inner_profiles","from":0,"size":3} + | } + | } + | ] + | } + | }, + | "size": 0, + | "aggs": { + | "nested_count_emails_value": { + | "nested": { + | "path": "emails" + | }, + | "aggs": { + | "filtered_agg": { + | "filter": { + | "term": { + | "emails.context": { + | "value": "profile" + | } + | } + | }, + | "aggs": { + | "count_emails_value": { + | "value_count": { + | "field": "emails.value" + | } + | } + | } + | } + | } + | } + | } + |}""".stripMargin.replaceAll("\\s+", "") + } + + it should "perform nested count with \"and not\" operator" in { + val results: Seq[ElasticAggregation] = + SQLQuery( + "select count(distinct inner_emails.value) as count_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where ((inner_profiles.postalCode = \"33600\") and (inner_profiles.postalCode <> \"75001\"))" + ) + results.size shouldBe 1 + val result = results.head + result.nested shouldBe true + result.distinct shouldBe true + result.aggName shouldBe "nested_count_distinct_emails_value.count_distinct_emails_value" + result.field shouldBe "count_emails" + result.sources shouldBe Seq[String]("index") + result.query.getOrElse("") shouldBe + """ + |{ + | "query": { + | "bool": { + | "filter": [ + | { + | "nested": { + | "path": "profiles", + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "profiles.postalCode": { + | "value": "33600" + | } + | } + | }, + | { + | "bool": { + | "must_not": [ + | { + | "term": { + | "profiles.postalCode": { + | "value": "75001" + | } + | } + | } + | ] + | } + | } + | ] + | } + | }, + | "inner_hits": { + | "name": "inner_profiles", + | "from": 0, + | "size": 3 + | } + | } + | } + | ] + | } + | }, + | "size": 0, + | "aggs": { + | "nested_count_distinct_emails_value": { + | "nested": { + | "path": "emails" + | }, + | "aggs": { + | "count_distinct_emails_value": { + | "cardinality": { + | "field": "emails.value" + | } + | } + | } + | } + | } + |} + |""".stripMargin.replaceAll("\\s+", "") + } + + it should "perform nested count with date filtering" in { + val results: Seq[ElasticAggregation] = + SQLQuery( + "select count(distinct inner_emails.value) as count_distinct_emails from index, unnest(emails) as inner_emails, unnest(profiles) as inner_profiles where inner_profiles.postalCode = \"33600\" and inner_profiles.createdDate <= \"now-35M/M\"" + ) + results.size shouldBe 1 + val result = results.head + result.nested shouldBe true + result.distinct shouldBe true + result.aggName shouldBe "nested_count_distinct_emails_value.count_distinct_emails_value" + result.field shouldBe "count_distinct_emails" + result.sources shouldBe Seq[String]("index") + result.query.getOrElse("") shouldBe + """{ + "query": { + | "bool": { + | "filter": [ + | { + | "nested": { + | "path": "profiles", + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "profiles.postalCode": { + | "value": "33600" + | } + | } + | }, + | { + | "range": { + | "profiles.createdDate": { + | "lte": "now-35M/M" + | } + | } + | } + | ] + | } + | }, + | "inner_hits": { + | "name": "inner_profiles", + | "from": 0, + | "size": 3 + | } + | } + | } + | ] + | } + | }, + | "size": 0, + | "aggs": { + | "nested_count_distinct_emails_value": { + | "nested": { + | "path": "emails" + | }, + | "aggs": { + | "count_distinct_emails_value": { + | "cardinality": { + | "field": "emails.value" + | } + | } + | } + | } + | } + |}""".stripMargin.replaceAll("\\s+", "") + } + + it should "perform nested select" in { + val select: ElasticSearchRequest = + SQLQuery(""" + |SELECT + |profileId, + |profile_ccm.email as email, + |profile_ccm.city as city, + |profile_ccm.firstName as firstName, + |profile_ccm.lastName as lastName, + |profile_ccm.postalCode as postalCode, + |profile_ccm.birthYear as birthYear + |FROM index, unnest(profiles) as profile_ccm + |WHERE + |((profile_ccm.postalCode BETWEEN "10" AND "99999") + |AND + |(profile_ccm.birthYear <= 2000)) + |limit 100""".stripMargin) + val query = select.query + val queryWithoutSource = query.substring(0, query.indexOf("_source") - 2) + "}" + queryWithoutSource shouldBe + """{ + | "query": { + | "bool": { + | "filter": [ + | { + | "nested": { + | "path": "profiles", + | "query": { + | "bool": { + | "filter": [ + | { + | "range": { + | "profiles.postalCode": { + | "gte": "10", + | "lte": "99999" + | } + | } + | }, + | { + | "range": { + | "profiles.birthYear": { + | "lte": "2000" + | } + | } + | } + | ] + | } + | }, + | "inner_hits": { + | "name": "profile_ccm", + | "from": 0, + | "size": 3 + | } + | } + | } + | ] + | } + | }, + | "from": 0, + | "size": 100 + |}""".stripMargin.replaceAll("\\s+", "") + val includes = new JsonParser() + .parse(query.substring(query.indexOf("_source") + 9, query.length - 1)) + .asInstanceOf[JsonObject] + .get("includes") + .asInstanceOf[JsonArray] + .iterator() + .asScala + val sourceIncludes: Seq[String] = ( + for (i <- includes) yield i.asInstanceOf[JsonPrimitive].getAsString + ).toSeq + val expectedSourceIncludes = Seq( + "profileId", + "profile_ccm.email", + "profile_ccm.city", + "profile_ccm.firstName", + "profile_ccm.lastName", + "profile_ccm.postalCode", + "profile_ccm.birthYear" + ) + sourceIncludes should contain theSameElementsAs expectedSourceIncludes + } + + it should "exclude fields from select" in { + val select: ElasticSearchRequest = + SQLQuery( + except + ) + select.query shouldBe + """ + |{ + | "query":{ + | "match_all":{} + | }, + | "_source":{ + | "includes":["*"], + | "excludes":["col1","col2"] + | } + |}""".stripMargin.replaceAll("\\s+", "") + } + + it should "perform complex query" in { + val select: ElasticSearchRequest = + SQLQuery( + s"""SELECT + | inner_products.name, + | inner_products.category, + | inner_products.price, + | min(inner_products.price) as min_price, + | max(inner_products.price) as max_price + |FROM + | stores store, + | UNNEST(store.products LIMIT 10) as inner_products + |WHERE + | ( + | firstName is not null AND + | lastName is not null AND + | description is not null AND + | preparationTime <= 120 AND + | store.deliveryPeriods.dayOfWeek=6 AND + | blockedCustomers not like "%uuid%" AND + | NOT receiptOfOrdersDisabled=true AND + | ( + | distance(pickup.location,(0.0,0.0)) <= "7000m" OR + | distance(withdrawals.location,(0.0,0.0)) <= "7000m" + | ) AND + | ( + | inner_products.deleted=false AND + | inner_products.upForSale=true AND + | inner_products.stock > 0 + | ) + | ) AND + | ( + | match(products.name, "lasagnes") AND + | ( + | match(products.description, "lasagnes") OR + | match(products.ingredients, "lasagnes") + | ) + | ) + |ORDER BY preparationTime ASC, nbOrders DESC + |LIMIT 100""".stripMargin + ).minScore(1.0) + val query = select.query + println(query) + query shouldBe + """ + |{ + | "query": { + | "bool": { + | "must": [ + | { + | "match": { + | "products.name": { + | "query": "lasagnes" + | } + | } + | }, + | { + | "bool": { + | "should": [ + | { + | "match": { + | "products.description": { + | "query": "lasagnes" + | } + | } + | }, + | { + | "match": { + | "products.ingredients": { + | "query": "lasagnes" + | } + | } + | } + | ] + | } + | } + | ], + | "filter": [ + | { + | "bool": { + | "filter": [ + | { + | "exists": { + | "field": "firstName" + | } + | }, + | { + | "exists": { + | "field": "lastName" + | } + | }, + | { + | "exists": { + | "field": "description" + | } + | }, + | { + | "range": { + | "preparationTime": { + | "lte": "120" + | } + | } + | }, + | { + | "term": { + | "deliveryPeriods.dayOfWeek": { + | "value": "6" + | } + | } + | }, + | { + | "bool": { + | "must_not": [ + | { + | "regexp": { + | "blockedCustomers": { + | "value": ".*?uuid.*?" + | } + | } + | } + | ] + | } + | }, + | { + | "bool": { + | "must_not": [ + | { + | "term": { + | "receiptOfOrdersDisabled": { + | "value": true + | } + | } + | } + | ] + | } + | }, + | { + | "bool": { + | "should": [ + | { + | "geo_distance": { + | "distance": "7000m", + | "pickup.location": [ + | 0.0, + | 0.0 + | ] + | } + | }, + | { + | "geo_distance": { + | "distance": "7000m", + | "withdrawals.location": [ + | 0.0, + | 0.0 + | ] + | } + | } + | ] + | } + | }, + | { + | "nested": { + | "path": "products", + | "query": { + | "bool": { + | "filter": [ + | { + | "term": { + | "products.deleted": { + | "value": false + | } + | } + | }, + | { + | "term": { + | "products.upForSale": { + | "value": true + | } + | } + | }, + | { + | "range": { + | "products.stock": { + | "gt": "0" + | } + | } + | } + | ] + | } + | }, + | "inner_hits": { + | "name": "inner_products", + | "from": 0, + | "size": 10 + | } + | } + | } + | ] + | } + | } + | ] + | } + | }, + | "from": 0, + | "size": 100, + | "min_score": 1.0, + | "sort": [ + | { + | "preparationTime": { + | "order": "asc" + | } + | }, + | { + | "nbOrders": { + | "order": "desc" + | } + | } + | ], + | "_source": { + | "includes": [ + | "inner_products.name", + | "inner_products.category", + | "inner_products.price" + | ] + | }, + | "aggs": { + | "nested_min_products_price": { + | "nested": { + | "path": "products" + | }, + | "aggs": { + | "min_products_price": { + | "min": { + | "field": "products.price" + | } + | } + | } + | }, + | "nested_max_products_price": { + | "nested": { + | "path": "products" + | }, + | "aggs": { + | "max_products_price": { + | "max": { + | "field": "products.price" + | } + | } + | } + | } + | } + |}""".stripMargin.replaceAll("\\s+", "") + } + +} diff --git a/sql/build.sbt b/sql/build.sbt index aa00a927..0a065f7f 100644 --- a/sql/build.sbt +++ b/sql/build.sbt @@ -1,30 +1,22 @@ -organization := "app.softnetwork.elastic" +import SoftClient4es._ -name := "elastic-sql" +organization := "app.softnetwork.elastic" -val jackson = Seq( - "com.fasterxml.jackson.core" % "jackson-databind" % Versions.jackson, - "com.fasterxml.jackson.core" % "jackson-core" % Versions.jackson, - "com.fasterxml.jackson.core" % "jackson-annotations" % Versions.jackson, - "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % Versions.jackson, - "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % Versions.jackson, - "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % Versions.jackson, - "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % Versions.jackson, - "com.fasterxml.jackson.module" % "jackson-module-parameter-names" % Versions.jackson, - "com.fasterxml.jackson.module" %% "jackson-module-scala" % Versions.jackson, -) +elasticSearchVersion := Versions.es9 -val elastic4s = Seq( - "nl.gn0s1s" %% "elastic4s-core" % Versions.elastic4s exclude ("org.elasticsearch", "elasticsearch") exclude("org.slf4j", "slf4j-api"), -) +name := s"softclient4es-sql" val scalatest = Seq( "org.scalatest" %% "scalatest" % Versions.scalatest % Test ) -libraryDependencies ++= jackson ++ elastic4s ++ scalatest ++ Seq( - "javax.activation" % "activation" % "1.1.1" % Test -) :+ +libraryDependencies ++= jacksonDependencies(elasticSearchVersion.value) ++ +// elastic4sDependencies(elasticSearchVersion.value) ++ + scalatest ++ + Seq( + "javax.activation" % "activation" % "1.1.1" % Test + ) :+ +// ("app.softnetwork.persistence" %% "persistence-core" % Versions.genericPersistence excludeAll(jacksonExclusions: _*)) :+ "org.scala-lang" % "scala-reflect" % "2.13.16" :+ "com.google.code.gson" % "gson" % Versions.gson % Test diff --git a/sql/src/main/resources/mapping/default.mustache b/sql/src/main/resources/mapping/default.mustache deleted file mode 100644 index f3e19d45..00000000 --- a/sql/src/main/resources/mapping/default.mustache +++ /dev/null @@ -1,16 +0,0 @@ -{ - "{{type}}": { - "properties": { - "uuid": { - "type": "keyword", - "index": true - }, - "createdDate": { - "type": "date" - }, - "lastUpdated": { - "type": "date" - } - } - } -} \ No newline at end of file diff --git a/sql/src/main/resources/softnetwork-elastic.conf b/sql/src/main/resources/softnetwork-elastic.conf deleted file mode 100644 index d884e512..00000000 --- a/sql/src/main/resources/softnetwork-elastic.conf +++ /dev/null @@ -1,21 +0,0 @@ -elastic { - ip = "localhost" - ip = ${?ELASTIC_IP} - port = 9200 - port = ${?ELASTIC_PORT} - - credentials { - url = "http://"${elastic.ip}":"${elastic.port} - username = "" - password = "" - - url = ${?ELASTIC_CREDENTIALS_URL} - username = ${?ELASTIC_CREDENTIALS_USERNAME} - password = ${?ELASTIC_CREDENTIALS_PASSWORD} - - } - - multithreaded = true - discovery-enabled = false - -} \ No newline at end of file diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/ElasticAggregation.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/ElasticAggregation.scala deleted file mode 100644 index 5a5b4d48..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/ElasticAggregation.scala +++ /dev/null @@ -1,16 +0,0 @@ -package app.softnetwork.elastic.sql - -import com.sksamuel.elastic4s.requests.searches.aggs.Aggregation - -case class ElasticAggregation( - aggName: String, - field: String, - sourceField: String, - sources: Seq[String] = Seq.empty, - query: Option[String] = None, - distinct: Boolean = false, - nested: Boolean = false, - filtered: Boolean = false, - aggType: AggregateFunction, - agg: Aggregation -) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/ElasticQuery.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/ElasticQuery.scala deleted file mode 100644 index 08c9cf32..00000000 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/ElasticQuery.scala +++ /dev/null @@ -1,11 +0,0 @@ -package app.softnetwork.elastic.sql - -import com.sksamuel.elastic4s.requests.searches.{MultiSearchRequest, SearchRequest} - -/** Created by smanciot on 27/06/2018. - */ -object ElasticQuery { - def search(request: SearchRequest): SQLSearchRequest = ??? - - def multiSearch(request: MultiSearchRequest): SQLMultiSearchRequest = ??? -} diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala index cb72fa86..b25fcbb8 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLImplicits.scala @@ -24,17 +24,6 @@ object SQLImplicits { } } - implicit def requestToElasticSearchRequest(request: SQLSearchRequest): ElasticSearchRequest = - ElasticSearchRequest( - request.select.fields, - request.select.except, - request.sources, - request.where.flatMap(_.criteria), - request.limit.map(_.limit), - request.searchRequest, - request.aggregations - ) - implicit def sqllikeToRegex(value: String): Regex = toRegex(value).r } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLMultiSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLMultiSearchRequest.scala index 6d33586e..9184eef0 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLMultiSearchRequest.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLMultiSearchRequest.scala @@ -1,13 +1,8 @@ package app.softnetwork.elastic.sql -import com.sksamuel.elastic4s.requests.searches.MultiSearchRequest - case class SQLMultiSearchRequest(requests: Seq[SQLSearchRequest]) extends SQLToken { override def sql: String = s"${requests.map(_.sql).mkString(" union ")}" def update(): SQLMultiSearchRequest = this.copy(requests = requests.map(_.update())) - lazy val multiSearchRequest: MultiSearchRequest = MultiSearchRequest( - requests.map(_.searchRequest) - ) } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLQuery.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLQuery.scala index 752776d1..9d86903e 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLQuery.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLQuery.scala @@ -1,69 +1,9 @@ package app.softnetwork.elastic.sql -import com.sksamuel.elastic4s.ElasticApi -import com.sksamuel.elastic4s.requests.searches.SearchBodyBuilderFn - case class SQLQuery(query: String, score: Option[Double] = None) { import SQLImplicits._ - - lazy val select: Option[Either[ElasticSearchRequest, ElasticMultiSearchRequest]] = { - val select: Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = query - select map { - case Left(s) => Left(s) - case Right(m) => - Right(ElasticMultiSearchRequest(m.requests.map(_.asInstanceOf), m.multiSearchRequest)) - } - } - - lazy val aggregations: Seq[ElasticAggregation] = { - import com.sksamuel.elastic4s.ElasticApi._ - val select: Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = this.query - select - .map { - case Left(l) => - l.aggregations.map(aggregation => { - - val queryFiltered = l.where.map(_.asQuery()).getOrElse(matchAllQuery) - - aggregation.copy( - sources = l.sources, - query = Some( - (aggregation.aggType match { - case Count if aggregation.sourceField.equalsIgnoreCase("_id") => - SearchBodyBuilderFn( - ElasticApi.search("") query { - queryFiltered - } - ) - case _ => - SearchBodyBuilderFn( - ElasticApi.search("") query { - queryFiltered - } - aggregations { - aggregation.agg - } - size 0 - ) - }).string.replace("\"version\":true,", "") /*FIXME*/ - ) - ) - }) - - case _ => Seq.empty - - } - .getOrElse(Seq.empty) - } - - lazy val search: Option[ElasticSearchRequest] = select match { - case Some(Left(value)) => Some(value.minScore(score)) - case _ => None - } - - lazy val multiSearch: Option[ElasticMultiSearchRequest] = select match { - case Some(Right(value)) => Some(value) - case _ => None + lazy val request: Option[Either[SQLSearchRequest, SQLMultiSearchRequest]] = { + query } def minScore(score: Double): SQLQuery = this.copy(score = Some(score)) diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala index 39176b5e..5975bdec 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSearchRequest.scala @@ -1,15 +1,12 @@ package app.softnetwork.elastic.sql -import com.sksamuel.elastic4s.ElasticApi.{matchAllQuery, search} -import com.sksamuel.elastic4s.requests.searches.SearchRequest -import com.sksamuel.elastic4s.requests.searches.sort.FieldSort - case class SQLSearchRequest( select: SQLSelect = SQLSelect(), from: SQLFrom, where: Option[SQLWhere], orderBy: Option[SQLOrderBy] = None, - limit: Option[SQLLimit] = None + limit: Option[SQLLimit] = None, + score: Option[Double] = None ) extends SQLToken { override def sql: String = s"$select$from${asString(where)}${asString(orderBy)}${asString(limit)}" @@ -30,47 +27,10 @@ case class SQLSearchRequest( lazy val aggregates: Seq[SQLAggregate] = select.fields.collect { case a: SQLAggregate => a } - lazy val aggregations: Seq[ElasticAggregation] = aggregates.map(_.asAggregation()) - lazy val excludes: Seq[String] = select.except.map(_.fields.map(_.sourceField)).getOrElse(Nil) lazy val sources: Seq[String] = from.tables.collect { case SQLTable(source: SQLIdentifier, _) => source.sql } - lazy val searchRequest: SearchRequest = { - var _search: SearchRequest = search("") query { - where.map(_.asQuery()).getOrElse(matchAllQuery) - } sourceInclude fields - - _search = excludes match { - case Nil => _search - case excludes => _search sourceExclude excludes - } - - _search = aggregations match { - case Nil => _search - case _ => _search aggregations { aggregations.map(_.agg) } - } - - _search = orderBy match { - case Some(o) => - _search sortBy o.sorts.map(sort => - sort.order match { - case Some(Desc) => FieldSort(sort.field).desc() - case _ => FieldSort(sort.field).asc() - } - ) - case _ => _search - } - - if (aggregations.nonEmpty && fields.isEmpty) { - _search size 0 - } else { - limit match { - case Some(l) => _search limit l.limit from 0 - case _ => _search - } - } - } } diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSelect.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSelect.scala index b037384a..f0b0df71 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSelect.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLSelect.scala @@ -1,18 +1,5 @@ package app.softnetwork.elastic.sql -import com.sksamuel.elastic4s.ElasticApi.{ - avgAgg, - cardinalityAgg, - filterAgg, - matchAllQuery, - maxAgg, - minAgg, - nestedAggregation, - sumAgg, - valueCountAgg -} -import com.sksamuel.elastic4s.requests.searches.aggs.Aggregation - case object Select extends SQLExpr("select") with SQLRegex case class SQLField( @@ -60,84 +47,6 @@ class SQLAggregate( override def sql: String = s"$function($identifier)${asString(alias)}" override def update(request: SQLSearchRequest): SQLAggregate = new SQLAggregate(function, identifier.update(request), alias, filter.map(_.update(request))) - - def asAggregation(): ElasticAggregation = { - val sourceField = identifier.columnName - - val field = alias match { - case Some(alias) => alias.alias - case _ => sourceField - } - - val distinct = identifier.distinct.isDefined - - val agg = - if (distinct) - s"${function}_distinct_${sourceField.replace(".", "_")}" - else - s"${function}_${sourceField.replace(".", "_")}" - - var aggPath = Seq[String]() - - val _agg = - function match { - case Count => - if (distinct) - cardinalityAgg(agg, sourceField) - else { - valueCountAgg(agg, sourceField) - } - case Min => minAgg(agg, sourceField) - case Max => maxAgg(agg, sourceField) - case Avg => avgAgg(agg, sourceField) - case Sum => sumAgg(agg, sourceField) - } - - def _filtered: Aggregation = filter match { - case Some(f) => - val boolQuery = Option(ElasticBoolQuery(group = true)) - val filteredAgg = s"filtered_agg" - aggPath ++= Seq(filteredAgg) - filterAgg( - filteredAgg, - f.criteria - .map( - _.asFilter(boolQuery) - .query(Set(identifier.innerHitsName).flatten, boolQuery) - ) - .getOrElse(matchAllQuery()) - ) subaggs { - aggPath ++= Seq(agg) - _agg - } - case _ => - aggPath ++= Seq(agg) - _agg - } - - val aggregation = - if (identifier.nested) { - val path = sourceField.split("\\.").head - val nestedAgg = s"nested_$agg" - aggPath ++= Seq(nestedAgg) - nestedAggregation(nestedAgg, path) subaggs { - _filtered - } - } else { - _filtered - } - - ElasticAggregation( - aggPath.mkString("."), - field, - sourceField, - distinct = distinct, - nested = identifier.nested, - filtered = filter.nonEmpty, - aggType = function, - agg = aggregation - ) - } } case class SQLSelect( diff --git a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLWhere.scala b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLWhere.scala index 00410732..8d30c661 100644 --- a/sql/src/main/scala/app/softnetwork/elastic/sql/SQLWhere.scala +++ b/sql/src/main/scala/app/softnetwork/elastic/sql/SQLWhere.scala @@ -1,8 +1,5 @@ package app.softnetwork.elastic.sql -import com.sksamuel.elastic4s.ElasticApi._ -import com.sksamuel.elastic4s.requests.searches.queries.Query - case object Where extends SQLExpr("where") with SQLRegex sealed trait SQLCriteria extends Updateable { @@ -31,14 +28,6 @@ sealed trait SQLCriteria extends Updateable { } } - def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty): Query = { - val query = boolQuery.copy(group = group) - query - .filter(this.asFilter(Option(query))) - .unfilteredMatchCriteria() - .query(innerHitsNames, Option(query)) - } - } case class SQLPredicate( @@ -97,12 +86,7 @@ case class SQLPredicate( override def matchCriteria: Boolean = leftCriteria.matchCriteria || rightCriteria.matchCriteria } -sealed trait ElasticFilter { - def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query -} +sealed trait ElasticFilter sealed trait SQLCriteriaWithIdentifier extends SQLCriteria { def identifier: SQLIdentifier @@ -146,19 +130,6 @@ case class ElasticBoolQuery( this } - def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query = { - import com.sksamuel.elastic4s.ElasticApi._ - bool( - mustFilters.map(_.query(innerHitsNames, currentQuery)), - shouldFilters.map(_.query(innerHitsNames, currentQuery)), - notFilters.map(_.query(innerHitsNames, currentQuery)) - ) - .filter(innerFilters.map(_.query(innerHitsNames, currentQuery))) - } - def unfilteredMatchCriteria(): ElasticBoolQuery = { val query = ElasticBoolQuery().copy( mustFilters = this.mustFilters, @@ -196,132 +167,6 @@ case class SQLExpression( } override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this - - override def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query = { - value match { - case n: SQLNumeric[Any] @unchecked => - operator match { - case _: Ge.type => - maybeNot match { - case Some(_) => - rangeQuery(identifier.columnName) lt n.sql - case _ => - rangeQuery(identifier.columnName) gte n.sql - } - case _: Gt.type => - maybeNot match { - case Some(_) => - rangeQuery(identifier.columnName) lte n.sql - case _ => - rangeQuery(identifier.columnName) gt n.sql - } - case _: Le.type => - maybeNot match { - case Some(_) => - rangeQuery(identifier.columnName) gt n.sql - case _ => - rangeQuery(identifier.columnName) lte n.sql - } - case _: Lt.type => - maybeNot match { - case Some(_) => - rangeQuery(identifier.columnName) gte n.sql - case _ => - rangeQuery(identifier.columnName) lt n.sql - } - case _: Eq.type => - maybeNot match { - case Some(_) => - not(termQuery(identifier.columnName, n.sql)) - case _ => - termQuery(identifier.columnName, n.sql) - } - case _: Ne.type => - maybeNot match { - case Some(_) => - termQuery(identifier.columnName, n.sql) - case _ => - not(termQuery(identifier.columnName, n.sql)) - } - case _ => matchAllQuery - } - case l: SQLLiteral => - operator match { - case _: Like.type => - maybeNot match { - case Some(_) => - not(regexQuery(identifier.columnName, toRegex(l.value))) - case _ => - regexQuery(identifier.columnName, toRegex(l.value)) - } - case _: Ge.type => - maybeNot match { - case Some(_) => - rangeQuery(identifier.columnName) lt l.value - case _ => - rangeQuery(identifier.columnName) gte l.value - } - case _: Gt.type => - maybeNot match { - case Some(_) => - rangeQuery(identifier.columnName) lte l.value - case _ => - rangeQuery(identifier.columnName) gt l.value - } - case _: Le.type => - maybeNot match { - case Some(_) => - rangeQuery(identifier.columnName) gt l.value - case _ => - rangeQuery(identifier.columnName) lte l.value - } - case _: Lt.type => - maybeNot match { - case Some(_) => - rangeQuery(identifier.columnName) gte l.value - case _ => - rangeQuery(identifier.columnName) lt l.value - } - case _: Eq.type => - maybeNot match { - case Some(_) => - not(termQuery(identifier.columnName, l.value)) - case _ => - termQuery(identifier.columnName, l.value) - } - case _: Ne.type => - maybeNot match { - case Some(_) => - termQuery(identifier.columnName, l.value) - case _ => - not(termQuery(identifier.columnName, l.value)) - } - case _ => matchAllQuery - } - case b: SQLBoolean => - operator match { - case _: Eq.type => - maybeNot match { - case Some(_) => - not(termQuery(identifier.columnName, b.value)) - case _ => - termQuery(identifier.columnName, b.value) - } - case _: Ne.type => - maybeNot match { - case Some(_) => - termQuery(identifier.columnName, b.value) - case _ => - not(termQuery(identifier.columnName, b.value)) - } - case _ => matchAllQuery - } - case _ => matchAllQuery - } - } } case class SQLIsNull(identifier: SQLIdentifier) @@ -338,13 +183,6 @@ case class SQLIsNull(identifier: SQLIdentifier) } override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this - - override def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query = { - not(existsQuery(identifier.columnName)) - } } case class SQLIsNotNull(identifier: SQLIdentifier) @@ -361,13 +199,6 @@ case class SQLIsNotNull(identifier: SQLIdentifier) } override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this - - override def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query = { - existsQuery(identifier.columnName) - } } case class SQLIn[R, +T <: SQLValue[R]]( @@ -388,27 +219,6 @@ case class SQLIn[R, +T <: SQLValue[R]]( } override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this - - override def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query = { - val _values: Seq[Any] = values.innerValues - val t = - _values.headOption match { - case Some(_: Double) => - termsQuery(identifier.columnName, _values.asInstanceOf[Seq[Double]]) - case Some(_: Integer) => - termsQuery(identifier.columnName, _values.asInstanceOf[Seq[Integer]]) - case Some(_: Long) => - termsQuery(identifier.columnName, _values.asInstanceOf[Seq[Long]]) - case _ => termsQuery(identifier.columnName, _values.map(_.toString)) - } - maybeNot match { - case Some(_) => not(t) - case _ => t - } - } } case class SQLBetween( @@ -430,17 +240,6 @@ case class SQLBetween( } override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this - - override def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query = { - val r = rangeQuery(identifier.columnName) gte from.value lte to.value - maybeNot match { - case Some(_) => not(r) - case _ => r - } - } } case class ElasticGeoDistance( @@ -456,13 +255,6 @@ case class ElasticGeoDistance( this.copy(identifier = identifier.update(request)) override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this - - override def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query = { - geoDistanceQuery(identifier.columnName, lat.value, lon.value) distance distance.value - } } case class ElasticMatch( @@ -479,13 +271,6 @@ case class ElasticMatch( override def asFilter(currentQuery: Option[ElasticBoolQuery]): ElasticFilter = this - override def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query = { - matchQuery(identifier.columnName, value.value) - } - override def matchCriteria: Boolean = true } @@ -526,59 +311,18 @@ case class ElasticNested(override val criteria: SQLCriteria, override val limit: } lazy val innerHitsName: Option[String] = name(criteria) - - override def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query = { - if (innerHitsNames.contains(innerHitsName.getOrElse(""))) { - criteria.asFilter(currentQuery).query(innerHitsNames, currentQuery) - } else { - val boolQuery = Option(ElasticBoolQuery(group = true)) - nestedQuery( - relationType.getOrElse(""), - criteria - .asFilter(boolQuery) - .query(innerHitsNames + innerHitsName.getOrElse(""), boolQuery) - ) /*.scoreMode(ScoreMode.None)*/ - .inner( - innerHits(innerHitsName.getOrElse("")).from(0).size(limit.map(_.limit).getOrElse(3)) - ) - } - } } case class ElasticChild(override val criteria: SQLCriteria) extends ElasticRelation(criteria, Child) { override def update(request: SQLSearchRequest): ElasticChild = this.copy(criteria = criteria.update(request)) - - override def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query = - hasChildQuery( - relationType.getOrElse(""), - // criteria.asFilter(currentQuery).query(innerHitsNames, currentQuery), - criteria.asQuery(group = group, innerHitsNames = innerHitsNames) - ) } case class ElasticParent(override val criteria: SQLCriteria) extends ElasticRelation(criteria, Parent) { override def update(request: SQLSearchRequest): ElasticParent = this.copy(criteria = criteria.update(request)) - - override def query( - innerHitsNames: Set[String] = Set.empty, - currentQuery: Option[ElasticBoolQuery] - ): Query = - hasParentQuery( - relationType.getOrElse(""), - // criteria.asFilter(currentQuery).query(innerHitsNames, currentQuery), - criteria.asQuery(group = group, innerHitsNames = innerHitsNames), - score = false - ) } case class SQLWhere(criteria: Option[SQLCriteria]) extends Updateable { @@ -589,7 +333,4 @@ case class SQLWhere(criteria: Option[SQLCriteria]) extends Updateable { def update(request: SQLSearchRequest): SQLWhere = this.copy(criteria = criteria.map(_.update(request))) - def asQuery(group: Boolean = true, innerHitsNames: Set[String] = Set.empty): Query = criteria - .map(_.asQuery(group = group, innerHitsNames = innerHitsNames)) - .getOrElse(matchAllQuery) } diff --git a/testkit/build.sbt b/testkit/build.sbt deleted file mode 100644 index a428b25e..00000000 --- a/testkit/build.sbt +++ /dev/null @@ -1,49 +0,0 @@ -organization := "app.softnetwork.elastic" - -name := "elastic-testkit" - -val jacksonExclusions = Seq( - ExclusionRule(organization = "com.fasterxml.jackson.core"), - ExclusionRule(organization = "com.fasterxml.jackson.dataformat"), - ExclusionRule(organization = "com.fasterxml.jackson.datatype"), - ExclusionRule(organization = "com.fasterxml.jackson.module") -) - -val elastic = Seq( - "nl.gn0s1s" %% "elastic4s-core" % Versions.elastic4s exclude ("org.elasticsearch", "elasticsearch") exclude("org.slf4j", "slf4j-api"), - "org.elasticsearch" % "elasticsearch" % Versions.elasticSearch exclude ("org.apache.logging.log4j", "log4j-api") exclude("org.slf4j", "slf4j-api") excludeAll(jacksonExclusions:_*), - "nl.gn0s1s" %% "elastic4s-testkit" % Versions.elastic4s exclude ("org.elasticsearch", "elasticsearch") exclude("org.slf4j", "slf4j-api"), - "org.apache.logging.log4j" % "log4j-api" % Versions.log4j, -// "org.apache.logging.log4j" % "log4j-slf4j-impl" % Versions.log4j, - "org.apache.logging.log4j" % "log4j-core" % Versions.log4j, - "org.testcontainers" % "elasticsearch" % Versions.testContainers excludeAll(jacksonExclusions:_*) -) - -libraryDependencies ++= elastic :+ - "app.softnetwork.persistence" %% "persistence-core-testkit" % Versions.genericPersistence - -val testJavaOptions = { - val heapSize = sys.env.getOrElse("HEAP_SIZE", "1g") - val extraTestJavaArgs = Seq("-XX:+IgnoreUnrecognizedVMOptions", - "--add-opens=java.base/java.lang=ALL-UNNAMED", - "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", - "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", - "--add-opens=java.base/java.io=ALL-UNNAMED", - "--add-opens=java.base/java.net=ALL-UNNAMED", - "--add-opens=java.base/java.nio=ALL-UNNAMED", - "--add-opens=java.base/java.util=ALL-UNNAMED", - "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", - "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", - "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED", - "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", - "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", - "--add-opens=java.base/sun.security.action=ALL-UNNAMED", - "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED").mkString(" ") - s"-Xmx$heapSize -Xss4m -XX:ReservedCodeCacheSize=128m -Dfile.encoding=UTF-8 $extraTestJavaArgs" - .split(" ").toSeq -} - -Test / javaOptions ++= testJavaOptions - -// Required by the Test container framework -Test / fork := true