diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03e0af5fb5..f2d0c1359f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,10 +47,10 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 21 distribution: 'temurin' - name: Setup Gradle uses: gradle/gradle-build-action@v2 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c43d90c2db..01cf579646 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,10 +37,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up JDK 17 + - name: Set up JDK 21 uses: actions/setup-java@v4 with: - java-version: 17 + java-version: 21 distribution: 'temurin' # Initializes the CodeQL tools for scanning. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9d0bec195f..024589de71 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: - name: Set up JDK uses: actions/setup-java@v4 with: - java-version: '17' + java-version: '21' distribution: 'temurin' - name: Setup Gradle uses: gradle/gradle-build-action@v2 diff --git a/.idea/misc.xml b/.idea/misc.xml index 4336a05b15..78ee99e08d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -54,5 +54,5 @@ - + \ No newline at end of file diff --git a/build.gradle b/build.gradle index 02f86f4e6d..ca229379c3 100644 --- a/build.gradle +++ b/build.gradle @@ -20,6 +20,12 @@ scmVersion { versionCreator 'versionWithBranch' } +java { + toolchain { + languageVersion = JavaLanguageVersion.of(21) + } +} + nexusPublishing { connectTimeout = Duration.ofMinutes(getIntProperty('publishingTimeoutInMin', 10)) clientTimeout = Duration.ofMinutes(getIntProperty('publishingTimeoutInMin', 10)) @@ -43,9 +49,6 @@ allprojects { group = 'pl.allegro.tech.hermes' version = scmVersion.version - sourceCompatibility = JavaVersion.VERSION_17 - targetCompatibility = JavaVersion.VERSION_17 - project.ext.versions = [ kafka : '2.8.2', guava : '33.1.0-jre', @@ -195,9 +198,9 @@ subprojects { test { reports { - html.enabled = false - junitXml.enabled = true - junitXml.destination = file("$buildDir/test-results/$name") + html.required = false + junitXml.required = true + junitXml.outputLocation = file("$buildDir/test-results/$name") } testLogging { diff --git a/docker/latest/consumers/Dockerfile b/docker/latest/consumers/Dockerfile index 5601a496c1..5137644606 100644 --- a/docker/latest/consumers/Dockerfile +++ b/docker/latest/consumers/Dockerfile @@ -1,10 +1,10 @@ -FROM gradle:7.6-jdk17 AS builder +FROM gradle:7.6-jdk21 AS builder COPY --chown=gradle:gradle . /home/gradle/src/ WORKDIR /home/gradle/src/ RUN gradle clean distZip -Pdistribution -FROM eclipse-temurin:17.0.7_7-jre +FROM eclipse-temurin:21-jre RUN apt-get update \ && apt-get -y install unzip wget bash \ diff --git a/docker/latest/frontend/Dockerfile b/docker/latest/frontend/Dockerfile index d3cc46f16a..938b1ca76d 100644 --- a/docker/latest/frontend/Dockerfile +++ b/docker/latest/frontend/Dockerfile @@ -1,10 +1,10 @@ -FROM gradle:7.6-jdk17 AS builder +FROM gradle:7.6-jdk21 AS builder COPY --chown=gradle:gradle . /home/gradle/src/ WORKDIR /home/gradle/src/ RUN gradle clean distZip -Pdistribution -FROM eclipse-temurin:17.0.7_7-jre +FROM eclipse-temurin:21-jre RUN apt-get update \ && apt-get -y install unzip wget bash \ diff --git a/docker/latest/management/Dockerfile b/docker/latest/management/Dockerfile index 5f6e8f32a9..2e2180083d 100644 --- a/docker/latest/management/Dockerfile +++ b/docker/latest/management/Dockerfile @@ -1,10 +1,10 @@ -FROM gradle:7.6-jdk17 AS builder +FROM gradle:7.6-jdk21 AS builder COPY --chown=gradle:gradle . /home/gradle/src/ WORKDIR /home/gradle/src/ RUN gradle clean distZip -Pdistribution -FROM eclipse-temurin:17.0.7_7-jre +FROM eclipse-temurin:21-jre RUN apt-get update \ && apt-get -y install unzip wget bash \ diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e708b1c023..a4b76b9530 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 070cb702f0..9355b41557 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 1b6c787337..f5feea6d6b 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,13 +82,12 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,22 +134,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -193,11 +201,15 @@ if "$cygwin" || "$msys" ; then done fi -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ @@ -205,6 +217,12 @@ set -- \ org.gradle.wrapper.GradleWrapperMain \ "$@" +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. diff --git a/gradlew.bat b/gradlew.bat index ac1b06f938..9b42019c79 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,8 +13,10 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +27,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,13 +43,13 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -56,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -75,13 +78,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/hermes-frontend/build.gradle b/hermes-frontend/build.gradle index dcb9018c40..440388400b 100644 --- a/hermes-frontend/build.gradle +++ b/hermes-frontend/build.gradle @@ -14,7 +14,7 @@ dependencies { api group: 'org.springframework.boot', name: 'spring-boot-starter', version: versions.spring api group: 'io.undertow', name: 'undertow-core', version: versions.undertow // Did not update that as we're trying to abandon buffers - api(group: 'net.openhft', name: 'chronicle-map', version: '3.22.9') { + api(group: 'net.openhft', name: 'chronicle-map', version: '3.25ea6') { exclude group: 'net.openhft', module: 'chronicle-analytics' } implementation group: 'commons-io', name: 'commons-io', version: '2.16.1' diff --git a/hermes-frontend/src/test/groovy/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducerIntegrationTest.groovy b/hermes-frontend/src/test/groovy/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducerIntegrationTest.groovy index 46b1913121..abfc90a484 100644 --- a/hermes-frontend/src/test/groovy/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducerIntegrationTest.groovy +++ b/hermes-frontend/src/test/groovy/pl/allegro/tech/hermes/frontend/producer/kafka/LocalDatacenterMessageProducerIntegrationTest.groovy @@ -1,6 +1,5 @@ package pl.allegro.tech.hermes.frontend.producer.kafka -import com.codahale.metrics.MetricRegistry import io.micrometer.core.instrument.simple.SimpleMeterRegistry import org.apache.commons.lang3.tuple.ImmutablePair import org.apache.kafka.clients.admin.AdminClient @@ -85,7 +84,7 @@ class LocalDatacenterMessageProducerIntegrationTest extends Specification { String datacenter = "dc" @Shared - ScheduledExecutorService chaosScheduler = Executors.newSingleThreadScheduledExecutor(); + ScheduledExecutorService chaosScheduler = Executors.newSingleThreadScheduledExecutor() @Shared MetricsFacade metricsFacade = new MetricsFacade(new SimpleMeterRegistry()) diff --git a/hermes-management/build.gradle b/hermes-management/build.gradle index 1370341e63..30be0fce91 100644 --- a/hermes-management/build.gradle +++ b/hermes-management/build.gradle @@ -89,6 +89,14 @@ tasks.register('prepareIndexTemplate') { } } +tasks.named('compileTestGroovy') { + dependsOn 'attachHermesConsole' +} + +tasks.named('javadoc') { + dependsOn 'attachHermesConsole' +} + jar.dependsOn(attachHermesConsole, 'prepareIndexTemplate') run.dependsOn(attachHermesConsole, 'prepareIndexTemplate') diff --git a/hermes-tracker-elasticsearch/build.gradle b/hermes-tracker-elasticsearch/build.gradle index 6e18bada83..7bbc669fa1 100644 --- a/hermes-tracker-elasticsearch/build.gradle +++ b/hermes-tracker-elasticsearch/build.gradle @@ -5,15 +5,12 @@ dependencies { implementation project(':hermes-common') implementation project(':hermes-tracker') implementation 'org.slf4j:slf4j-api:2.0.13' - // TODO: client has to have the same major version as backend: https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/client.html - // we need to use test-containers for elasticsearch server to able to use newer client. - api group: 'org.elasticsearch.client', name: 'transport', version: '6.8.12' + api group: 'org.elasticsearch.client', name: 'transport', version: '7.10.2' testImplementation project(path: ":hermes-tracker", configuration: "testArtifacts") testImplementation project(path: ":hermes-test-helper") testImplementation group: 'org.spockframework', name: 'spock-core', version: versions.spock testImplementation group: 'org.spockframework', name: 'spock-junit4', version: versions.spock - // this should be migrated to testcontainers - testImplementation 'pl.allegro.tech:embedded-elasticsearch:2.10.0' + testImplementation "org.testcontainers:elasticsearch:1.20.3" testRuntimeOnly group: 'org.junit.vintage', name: 'junit-vintage-engine', version: versions.junit_jupiter } \ No newline at end of file diff --git a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchResource.java b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchResource.java index efc54f195a..d746738184 100644 --- a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchResource.java +++ b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/ElasticsearchResource.java @@ -1,50 +1,34 @@ package pl.allegro.tech.hermes.tracker.elasticsearch; -import static pl.allegro.tech.embeddedelasticsearch.PopularProperties.CLUSTER_NAME; -import static pl.allegro.tech.embeddedelasticsearch.PopularProperties.HTTP_PORT; -import static pl.allegro.tech.embeddedelasticsearch.PopularProperties.TRANSPORT_TCP_PORT; - import java.net.InetAddress; -import java.nio.file.Files; -import java.util.concurrent.TimeUnit; +import java.time.Duration; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.transport.client.PreBuiltTransportClient; import org.junit.rules.ExternalResource; -import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; -import pl.allegro.tech.hermes.test.helper.util.Ports; +import org.testcontainers.elasticsearch.ElasticsearchContainer; public class ElasticsearchResource extends ExternalResource implements LogSchemaAware { - private static final String ELASTIC_VERSION = "6.1.4"; + private static final String ELASTIC_VERSION = + "docker.elastic.co/elasticsearch/elasticsearch:7.10.2"; private static final String CLUSTER_NAME_VALUE = "myTestCluster"; - private final EmbeddedElastic embeddedElastic; + private final ElasticsearchContainer elasticsearchContainer; private Client client; public ElasticsearchResource() { - int port = Ports.nextAvailable(); - int httpPort = Ports.nextAvailable(); - try { - embeddedElastic = - EmbeddedElastic.builder() - .withElasticVersion(ELASTIC_VERSION) - .withSetting(TRANSPORT_TCP_PORT, port) - .withSetting(HTTP_PORT, httpPort) - .withSetting(CLUSTER_NAME, CLUSTER_NAME_VALUE) - // embedded elastic search runs with "UseConcMarkSweepGC" which is invalid in Java 17 - .withEsJavaOpts("-Xms128m -Xmx512m -XX:+IgnoreUnrecognizedVMOptions") - .withStartTimeout(1, TimeUnit.MINUTES) - .withCleanInstallationDirectoryOnStop(true) - .withInstallationDirectory( - Files.createTempDirectory("elasticsearch-installation-" + port).toFile()) - .build(); - + elasticsearchContainer = + new ElasticsearchContainer(ELASTIC_VERSION) + .withEnv("cluster.name", CLUSTER_NAME_VALUE) + .withEnv("discovery.type", "single-node") + .withEnv("ES_JAVA_OPTS", "-Xms128m -Xmx512m -XX:+IgnoreUnrecognizedVMOptions") + .withStartupTimeout(Duration.ofMinutes(1)); } catch (Exception e) { throw new RuntimeException("Unchecked exception", e); } @@ -52,19 +36,20 @@ public ElasticsearchResource() { @Override public void before() throws Throwable { - embeddedElastic.start(); + elasticsearchContainer.start(); client = new PreBuiltTransportClient( - Settings.builder().put(CLUSTER_NAME, CLUSTER_NAME_VALUE).build()) + Settings.builder().put("cluster.name", CLUSTER_NAME_VALUE).build()) .addTransportAddress( new TransportAddress( - InetAddress.getByName("localhost"), embeddedElastic.getTransportTcpPort())); + InetAddress.getByName(elasticsearchContainer.getHost()), + elasticsearchContainer.getMappedPort(9300))); } @Override public void after() { - embeddedElastic.stop(); + elasticsearchContainer.stop(); client.close(); } @@ -76,7 +61,7 @@ public AdminClient adminClient() { return client.admin(); } - public ImmutableOpenMap getIndices() { + public ImmutableOpenMap getIndices() { return client .admin() .cluster() @@ -84,12 +69,12 @@ public ImmutableOpenMap getIndices() { .execute() .actionGet() .getState() - .getMetaData() + .getMetadata() .getIndices(); } public void cleanStructures() { - embeddedElastic.deleteIndices(); - embeddedElastic.deleteTemplates(); + client.admin().indices().prepareDelete("*").get(); + client.admin().indices().prepareDeleteTemplate("*").get(); } } diff --git a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepositoryTest.java b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepositoryTest.java index 4bad544276..515844057d 100644 --- a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepositoryTest.java +++ b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/consumers/ConsumersElasticsearchLogRepositoryTest.java @@ -89,7 +89,7 @@ private void awaitUntilPersisted(QueryBuilder query) { .setQuery(query) .execute() .get(); - return response.getHits().getTotalHits() == 1; + return response.getHits().getTotalHits().value == 1; }); } diff --git a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepositoryTest.java b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepositoryTest.java index de7deaa1c4..eb93d26a70 100644 --- a/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepositoryTest.java +++ b/hermes-tracker-elasticsearch/src/test/java/pl/allegro/tech/hermes/tracker/elasticsearch/frontend/FrontendElasticsearchLogRepositoryTest.java @@ -146,7 +146,7 @@ private void awaitUntilMessageIsIndexed(QueryBuilder query) { .setQuery(query) .execute() .get(); - return response.getHits().getTotalHits() == 1; + return response.getHits().getTotalHits().value == 1; }); } } diff --git a/hermes-tracker/build.gradle b/hermes-tracker/build.gradle index b97f37ce21..77dabfc6cd 100644 --- a/hermes-tracker/build.gradle +++ b/hermes-tracker/build.gradle @@ -10,7 +10,7 @@ configurations { } task testJar (type: Jar) { - classifier = 'tests' + archiveClassifier = 'tests' from sourceSets.test.output }