[RORDEV-1554] ES 8.19.0 support

ci/supported-es-versions/es8x.txt

@@ -1,3 +1,4 @@
+8.19.0
 8.18.4
 8.18.3
 8.18.2

es818x/gradle.properties

@@ -1 +1 @@
-latestSupportedEsVersion=8.18.4
+latestSupportedEsVersion=8.19.0

es818x/src/main/scala/tech/beshu/ror/es/IndexLevelActionFilter.scala

@@ -162,7 +162,8 @@ class IndexLevelActionFilter(clusterService: ClusterService,
             request,
             listener,
             chain,
-            JavaConverters.flattenPair(threadPool.getThreadContext.getResponseHeaders).toCovariantSet
+            JavaConverters.flattenPair(threadPool.getThreadContext.getResponseHeaders).toCovariantSet,
+            esEnv.esVersion
           )
         )
     }

es818x/src/main/scala/tech/beshu/ror/es/handler/AclAwareRequestFilter.scala

@@ -74,7 +74,7 @@ import tech.beshu.ror.es.handler.request.context.types.repositories.*
 import tech.beshu.ror.es.handler.request.context.types.ror.*
 import tech.beshu.ror.es.handler.request.context.types.snapshots.*
 import tech.beshu.ror.es.handler.request.context.types.templates.*
-import tech.beshu.ror.es.{RorClusterService, RorRestChannel}
+import tech.beshu.ror.es.{EsVersion, RorClusterService, RorRestChannel}
 import tech.beshu.ror.implicits.*
 import tech.beshu.ror.syntax.*
 
@@ -274,7 +274,8 @@ object AclAwareRequestFilter {
                         val actionRequest: ActionRequest,
                         val listener: ActionListener[ActionResponse],
                         val chain: EsChain,
-                        val threadContextResponseHeaders: Set[(String, String)]) {
+                        val threadContextResponseHeaders: Set[(String, String)],
+                        val esVersion: EsVersion) {
 
     val timestamp: Instant = Instant.now()
 

es818x/src/main/scala/tech/beshu/ror/es/handler/request/context/types/EsqlIndicesEsRequestContext.scala

@@ -34,7 +34,7 @@ import tech.beshu.ror.es.handler.request.context.ModificationResult
 import tech.beshu.ror.es.handler.request.context.ModificationResult.{CannotModify, UpdateResponse}
 import tech.beshu.ror.es.handler.response.FLSContextHeaderHandler
 import tech.beshu.ror.es.utils.EsqlRequestHelper
-import tech.beshu.ror.es.utils.EsqlRequestHelper.{ClassificationError, EsqlRequestClassification}
+import tech.beshu.ror.es.utils.EsqlRequestHelper.{ClassificationError, EsqlRequestClassification, ModificationError}
 import tech.beshu.ror.exceptions.SecurityPermissionException
 import tech.beshu.ror.implicits.*
 import tech.beshu.ror.syntax.*
@@ -43,12 +43,13 @@ class EsqlIndicesEsRequestContext private(actionRequest: ActionRequest with Comp
                                           esContext: EsContext,
                                           aclContext: AccessControlStaticContext,
                                           clusterService: RorClusterService,
-                                          override val threadPool: ThreadPool)
+                                          override val threadPool: ThreadPool,
+                                          esqlRequestHelper: EsqlRequestHelper)
   extends BaseFilterableEsRequestContext[ActionRequest with CompositeIndicesRequest](actionRequest, esContext, aclContext, clusterService, threadPool) {
 
   override protected def requestFieldsUsage: RequestFieldsUsage = RequestFieldsUsage.NotUsingFields
 
-  private lazy val requestClassification = EsqlRequestHelper.classifyEsqlRequest(actionRequest) match {
+  private lazy val requestClassification = esqlRequestHelper.classifyEsqlRequest(actionRequest) match {
     case result@Right(_) => result
     case result@Left(ClassificationError.ParsingException) => result
     case Left(ClassificationError.UnexpectedException(ex)) =>
@@ -70,7 +71,7 @@ class EsqlIndicesEsRequestContext private(actionRequest: ActionRequest with Comp
                                 filteredRequestedIndices: NonEmptyList[RequestedIndex[ClusterIndexName]],
                                 filter: Option[Filter],
                                 fieldLevelSecurity: Option[FieldLevelSecurity]): ModificationResult = {
-    val result: Either[EsqlRequestHelper.ModificationError, UpdateResponse] = for {
+    val result: Either[ModificationError, UpdateResponse] = for {
       _ <- modifyRequestIndices(request, filteredRequestedIndices)
       _ <- Right(applyFieldLevelSecurityTo(request, fieldLevelSecurity))
       _ <- Right(applyFilterTo(request, filter))
@@ -80,7 +81,7 @@ class EsqlIndicesEsRequestContext private(actionRequest: ActionRequest with Comp
           applyFieldLevelSecurityTo(response, fieldLevelSecurity) match {
             case Right(modifiedResponse) =>
               modifiedResponse
-            case Left(EsqlRequestHelper.ModificationError.UnexpectedException(ex)) =>
+            case Left(ModificationError.UnexpectedException(ex)) =>
               throw new SecurityPermissionException("Cannot apply field level security to the ESQL response", ex)
           }
         }
@@ -96,14 +97,14 @@ class EsqlIndicesEsRequestContext private(actionRequest: ActionRequest with Comp
   }
 
   private def modifyRequestIndices(request: ActionRequest with CompositeIndicesRequest,
-                                   filteredIndices: NonEmptyList[RequestedIndex[ClusterIndexName]]): Either[EsqlRequestHelper.ModificationError, CompositeIndicesRequest] = {
+                                   filteredIndices: NonEmptyList[RequestedIndex[ClusterIndexName]]): Either[ModificationError, CompositeIndicesRequest] = {
     requestClassification match {
       case Right(EsqlRequestClassification.NonIndicesRelated) =>
         Right(request)
       case Right([email protected](tables)) =>
         val filteredIndicesStrings = filteredIndices.stringify.toCovariantSet
         if (filteredIndicesStrings != r.indices) {
-          EsqlRequestHelper.modifyIndicesOf(request, tables, filteredIndicesStrings)
+          esqlRequestHelper.modifyIndicesOf(request, tables, filteredIndicesStrings)
         } else {
           Right(request)
         }
@@ -135,7 +136,7 @@ class EsqlIndicesEsRequestContext private(actionRequest: ActionRequest with Comp
                                         fieldLevelSecurity: Option[FieldLevelSecurity]) = {
     fieldLevelSecurity match {
       case Some(fls) =>
-        EsqlRequestHelper.modifyResponseAccordingToFieldLevelSecurity(response, fls)
+        esqlRequestHelper.modifyResponseAccordingToFieldLevelSecurity(response, fls)
       case None =>
         Right(response)
     }
@@ -159,7 +160,8 @@ object EsqlIndicesEsRequestContext {
         arg.esContext,
         arg.aclContext,
         arg.clusterService,
-        arg.threadPool
+        arg.threadPool,
+        new EsqlRequestHelper(arg.esContext.esVersion)
       ))
     } else {
       None

es818x/src/main/scala/tech/beshu/ror/es/utils/EsqlRequestHelper.scala

@@ -23,29 +23,22 @@ import org.elasticsearch.action.{ActionResponse, CompositeIndicesRequest}
 import org.joor.Reflect.*
 import tech.beshu.ror.accesscontrol.___domain.FieldLevelSecurity
 import tech.beshu.ror.accesscontrol.___domain.FieldLevelSecurity.FieldsRestrictions
+import tech.beshu.ror.es.EsVersion
 import tech.beshu.ror.es.handler.response.FieldsFiltering
 import tech.beshu.ror.es.handler.response.FieldsFiltering.NonMetadataDocumentFields
-import tech.beshu.ror.es.utils.*
+import tech.beshu.ror.es.utils.EsqlRequestHelper.*
 import tech.beshu.ror.syntax.*
 import tech.beshu.ror.utils.ReflecUtils
 import tech.beshu.ror.utils.ScalaOps.*
 
 import java.lang.reflect.Modifier
-import java.util.List as JList
+import java.time.ZoneOffset
+import java.util.{Locale, List as JList}
 import java.util.regex.Pattern
 import scala.jdk.CollectionConverters.*
 import scala.util.{Failure, Success, Try}
 
-object EsqlRequestHelper {
-
-  final case class IndexTable(tableStringInQuery: String, indices: NonEmptyList[String])
-
-  sealed trait ModificationError
-  object ModificationError {
-    final case class UnexpectedException(ex: Throwable) extends ModificationError
-
-    implicit val show: Show[ModificationError] = Show.show(_.toString)
-  }
+class EsqlRequestHelper(esVersion: EsVersion) {
 
   def modifyIndicesOf(request: CompositeIndicesRequest,
                       requestTables: NonEmptyList[IndexTable],
@@ -61,37 +54,21 @@ object EsqlRequestHelper {
       .toEither.left.map(ModificationError.UnexpectedException.apply)
   }
 
-  sealed trait EsqlRequestClassification
-  object EsqlRequestClassification {
-    final case class IndicesRelated(tables: NonEmptyList[IndexTable]) extends EsqlRequestClassification {
-      lazy val indices: Set[String] = tables.toCovariantSet.flatMap(_.indices.toIterable)
-    }
-    case object NonIndicesRelated extends EsqlRequestClassification
-  }
-
-  sealed trait ClassificationError
-  object ClassificationError {
-    final case class UnexpectedException(ex: Throwable) extends ClassificationError
-    case object ParsingException extends ClassificationError
-  }
-
-  import EsqlRequestClassification._
+  import EsqlRequestClassification.*
 
   def classifyEsqlRequest(request: CompositeIndicesRequest): Either[ClassificationError, EsqlRequestClassification] = {
-    val result = Try {
-      val query = getQuery(request)
-      val params = ReflecUtils.invokeMethodCached(request, request.getClass, "params")
-
-      implicit val classLoader: ClassLoader = request.getClass.getClassLoader
-      Try(new EsqlParser().createStatement(query, params)) match {
-        case Success(statement: IndicesRelatedStatement) => Right(IndicesRelated(statement.indices))
-        case Success(command: OtherCommand) => Right(NonIndicesRelated)
-        case Failure(_) => Left(ClassificationError.ParsingException: ClassificationError)
-      }
+    createStatement(request) match {
+      case Right(statement: IndicesRelatedStatement) => Right(IndicesRelated(statement.indices))
+      case Right(command: OtherCommand) => Right(NonIndicesRelated)
+      case Left(error) => Left(error)
     }
-    result match {
-      case Success(value) => value
-      case Failure(exception) => Left(ClassificationError.UnexpectedException(exception))
+  }
+
+  private def createStatement(request: CompositeIndicesRequest): Either[ClassificationError, Statement] = {
+    implicit val classLoader: ClassLoader = request.getClass.getClassLoader
+    Try(new EsqlParser(esVersion)) match {
+      case Success(parser) => parser.createStatementBasedOn(request)
+      case Failure(ex) => Left(ClassificationError.UnexpectedException(ex))
     }
   }
 
@@ -106,6 +83,30 @@ object EsqlRequestHelper {
     request
   }
 
+  private def getParams(request: CompositeIndicesRequest): AnyRef = {
+    ReflecUtils.invokeMethodCached(request, request.getClass, "params")
+  }
+
+  private def createConfiguration(request: CompositeIndicesRequest): AnyRef = {
+    val classLoader = request.getClass.getClassLoader
+    onClass(classLoader.loadClass("org.elasticsearch.xpack.esql.session.Configuration"))
+      .create(
+        ZoneOffset.UTC,
+        Option(on(request).call("locale").get[Locale]).getOrElse(Locale.US),
+        null, // at the moment it's not used anywhere, so it's null here - probably to be fixed in the future
+        "ROR", // at the moment it's not used anywhere, so it's placeholder here - probably to be fixed in the future
+        on(request).call("pragmas").get[AnyRef],
+        Int.MaxValue,
+        Int.MaxValue,
+        getQuery(request),
+        on(request).call("profile").get[AnyRef],
+        on(request).call("tables").get[AnyRef],
+        System.nanoTime(),
+        Option(on(request).call("allowPartialResults").get[Any]).getOrElse(true)
+      )
+      .get[AnyRef]()
+  }
+
   private def newQueryFrom(oldQuery: String, requestTables: NonEmptyList[IndexTable], finalIndices: Set[String]) = {
     requestTables.toList.foldLeft(oldQuery) {
       case (currentQuery, table) =>
@@ -123,18 +124,43 @@ object EsqlRequestHelper {
     currentQuery.replaceAll(Pattern.quote(originTable), finalIndices.mkString(","))
   }
 
-  private final class EsqlParser(implicit classLoader: ClassLoader) {
+  private final class EsqlParser(esVersion: EsVersion)
+                                (implicit classLoader: ClassLoader) {
 
     private val underlyingObject =
       onClass(classLoader.loadClass("org.elasticsearch.xpack.esql.parser.EsqlParser"))
         .create().get[Any]()
 
-    def createStatement(query: String, params: AnyRef): Statement = {
-      val statement = on(underlyingObject).call("createStatement", query, params).get[Any]
-      NonEmptyList.fromList(indicesFrom(statement)) match {
-        case Some(indices) => new IndicesRelatedStatement(statement, indices)
-        case None => OtherCommand(statement)
+    def createStatementBasedOn(request: CompositeIndicesRequest): Either[ClassificationError, Statement] = {
+      val statement = esVersion match {
+        case v if v >= EsVersion(8, 19, 0) => createStatementForEsEqualOrAbove8190(request)
+        case v => createStatementForEsBelow8190(request)
       }
+      statement.map { s =>
+        NonEmptyList.fromList(indicesFrom(s)) match {
+          case Some(indices) => new IndicesRelatedStatement(statement, indices)
+          case None => OtherCommand(statement)
+        }
+      }
+    }
+
+    private def createStatementForEsBelow8190(request: CompositeIndicesRequest) = {
+      for {
+        query <- Try(getQuery(request)).toEither.left.map(ClassificationError.UnexpectedException.apply)
+        params <- Try(getParams(request)).toEither.left.map(ClassificationError.UnexpectedException.apply)
+        statement <- Try(on(underlyingObject).call("createStatement", query, params).get[Any])
+          .toEither.left.map(_ => ClassificationError.ParsingException)
+      } yield statement
+    }
+
+    private def createStatementForEsEqualOrAbove8190(request: CompositeIndicesRequest) = {
+      for {
+        query <- Try(getQuery(request)).toEither.left.map(ClassificationError.UnexpectedException.apply)
+        params <- Try(getParams(request)).toEither.left.map(ClassificationError.UnexpectedException.apply)
+        configuration <- Try(createConfiguration(request)).toEither.left.map(ClassificationError.UnexpectedException.apply)
+        statement <- Try(on(underlyingObject).call("createStatement", query, params, configuration).get[Any])
+          .toEither.left.map(_ => ClassificationError.ParsingException)
+      } yield statement
     }
 
     private def indicesFrom(statement: Any) = {
@@ -273,3 +299,29 @@ object EsqlRequestHelper {
     }
   }
 }
+object EsqlRequestHelper {
+
+  final case class IndexTable(tableStringInQuery: String, indices: NonEmptyList[String])
+
+  sealed trait ModificationError
+  object ModificationError {
+    final case class UnexpectedException(ex: Throwable) extends ModificationError
+
+    implicit val show: Show[ModificationError] = Show.show(_.toString)
+  }
+
+  sealed trait EsqlRequestClassification
+  object EsqlRequestClassification {
+    final case class IndicesRelated(tables: NonEmptyList[IndexTable]) extends EsqlRequestClassification {
+      lazy val indices: Set[String] = tables.toCovariantSet.flatMap(_.indices.toIterable)
+    }
+    case object NonIndicesRelated extends EsqlRequestClassification
+  }
+
+  sealed trait ClassificationError
+  object ClassificationError {
+    final case class UnexpectedException(ex: Throwable) extends ClassificationError
+    case object ParsingException extends ClassificationError
+  }
+
+}

ror-tools-core/src/main/scala/tech/beshu/ror/tools/core/patches/internal/modifiers/bytecodeJars/ModifyFilesEntitlementsValidationClass.scala

@@ -19,12 +19,12 @@ package tech.beshu.ror.tools.core.patches.internal.modifiers.bytecodeJars
 import just.semver.SemVer
 import org.objectweb.asm.{ClassReader, ClassVisitor, ClassWriter, Label, MethodVisitor, Opcodes}
 import tech.beshu.ror.tools.core.patches.internal.modifiers.BytecodeJarModifier
-import tech.beshu.ror.tools.core.utils.EsUtil.es8182
+import tech.beshu.ror.tools.core.utils.EsUtil.{es8182, es8190, es900}
 
 import java.io.{File, InputStream}
 
 /**
- * Modifies the EntitlementInitialization class to bypass forbidden file path validation
+ * Modifies the FilesEntitlementsValidation class to bypass forbidden file path validation
  * specifically for the ReadonlyREST plugin. This is necessary because ReadonlyREST
  * requires access to certain paths that would otherwise be blocked by Elasticsearch's
  * security entitlements system in versions 8.18.2+
@@ -35,7 +35,11 @@ private[patches] class ModifyFilesEntitlementsValidationClass(esVersion: SemVer)
   override def apply(jar: File): Unit = {
     modifyFileInJar(
       jar = jar,
-      filePathString = "org/elasticsearch/entitlement/initialization/FilesEntitlementsValidation.class",
+      filePathString = esVersion match {
+        case v if v >= es900 => "org/elasticsearch/entitlement/initialization/FilesEntitlementsValidation.class"
+        case v if v >= es8190 => "org/elasticsearch/entitlement/bootstrap/FilesEntitlementsValidation.class"
+        case v => "org/elasticsearch/entitlement/initialization/FilesEntitlementsValidation.class"
+      },
       processFileContent = dontValidateForbiddenPathsInCaseOfRorPlugin
     )
   }

ror-tools-core/src/main/scala/tech/beshu/ror/tools/core/utils/EsUtil.scala

@@ -26,6 +26,7 @@ object EsUtil {
   val es902: SemVer = SemVer.unsafeParse("9.0.2")
   val es901: SemVer = SemVer.unsafeParse("9.0.1")
   val es900: SemVer = SemVer.unsafeParse("9.0.0")
+  val es8190: SemVer = SemVer.unsafeParse("8.19.0")
   val es8182: SemVer = SemVer.unsafeParse("8.18.2")
   val es8181: SemVer = SemVer.unsafeParse("8.18.1")
   val es8180: SemVer = SemVer.unsafeParse("8.18.0")