From fb88aa03e2b41a54be5a27382f895dc6c2ef9817 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 31 Dec 2024 14:55:57 -0800 Subject: [PATCH 01/27] RANGER-5080: Add docker support for MS SQL Server --- dev-support/ranger-docker/.env | 1 + dev-support/ranger-docker/Dockerfile.ranger | 4 + .../ranger-docker/Dockerfile.ranger-sqlserver | 44 +++++++++ .../ranger-docker/config/init_mssql.sh | 71 ++++++++++++++ .../docker-compose.ranger-sqlserver.yml | 28 ++++++ .../ranger-docker/download-archives.sh | 1 + .../ranger-admin-install-sqlserver.properties | 94 +++++++++++++++++++ 7 files changed, 243 insertions(+) create mode 100644 dev-support/ranger-docker/Dockerfile.ranger-sqlserver create mode 100644 dev-support/ranger-docker/config/init_mssql.sh create mode 100644 dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml create mode 100644 dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index ca27d0799e..25ae55537b 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -36,6 +36,7 @@ UBI_VERSION=latest MARIADB_VERSION=10.7.3 POSTGRES_VERSION=12 ORACLE_VERSION=23.6 +SQLSERVER_VERSION=2022-latest ENABLE_DB_MOUNT=true ZK_VERSION=3.9.2 SOLR_VERSION=8.11.3 diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger index 087430dd72..9d14a1eb9c 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger @@ -58,6 +58,10 @@ FROM ranger AS ranger_oracle COPY ./downloads/ojdbc8.jar /home/ranger/dist/ RUN mv /home/ranger/dist/ojdbc8.jar /usr/share/java/oracle.jar +FROM ranger AS ranger_sqlserver +COPY ./downloads/mssql-jdbc-12.8.1.jre8.jar /home/ranger/dist/ +RUN mv /home/ranger/dist/mssql-jdbc-12.8.1.jre8.jar /usr/share/java/mssql.jar + FROM ranger_${RANGER_DB_TYPE} USER ranger diff --git a/dev-support/ranger-docker/Dockerfile.ranger-sqlserver b/dev-support/ranger-docker/Dockerfile.ranger-sqlserver new file mode 100644 index 0000000000..0691eaf6f0 --- /dev/null +++ b/dev-support/ranger-docker/Dockerfile.ranger-sqlserver @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG SQLSERVER_VERSION + +#FROM mcr.microsoft.com/mssql/server:${SQLSERVER_VERSION} + +FROM mcr.microsoft.com/azure-sql-edge:latest + +USER 0 + +RUN ACCEPT_EULA=Y apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + apt-transport-https \ + gnupg \ + unixodbc-dev + +RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \ + curl https://packages.microsoft.com/config/ubuntu/18.04/prod.list > /etc/apt/sources.list.d/mssql-tools.list + +RUN ACCEPT_EULA=Y apt-get update && \ + ACCEPT_EULA=Y apt-get install -y --no-install-recommends mssql-tools unixodbc-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /docker-entrypoint-initdb.d +COPY config/init_mssql.sh /docker-entrypoint-initdb.d/ +RUN chown -R mssql /docker-entrypoint-initdb.d/ +ENV MSSQL_PASSWORD=rangerR0cks! +USER mssql diff --git a/dev-support/ranger-docker/config/init_mssql.sh b/dev-support/ranger-docker/config/init_mssql.sh new file mode 100644 index 0000000000..45ed3b243c --- /dev/null +++ b/dev-support/ranger-docker/config/init_mssql.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Wait for SQL Server to be ready +echo "Waiting for SQL Server to start..." +RETRIES=30 # Number of retries +SLEEP_INTERVAL=5 # Seconds to wait between retries +for i in $(seq 1 $RETRIES); do + # Try to connect to SQL Server + /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "rangerR0cks!" -Q "SELECT 1" > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "SQL Server is ready!" + break + else + echo "SQL Server is not ready yet. Waiting..." + sleep $SLEEP_INTERVAL + fi +done + +if [ $i -eq $RETRIES ]; then + echo "SQL Server did not become ready in time. Exiting." + exit 1 +fi + + +sqlcmd -S localhost -U mssql -P 'rangerR0cks!' -Q " + +-- Set the database context +USE master; + +-- Create databases +CREATE DATABASE ranger; +CREATE DATABASE rangerkms; +CREATE DATABASE hive; +GO + +-- Create users and assign permissions +USE ranger; +CREATE LOGIN rangeradmin WITH PASSWORD = 'rangerR0cks!'; +CREATE USER rangeradmin FOR LOGIN rangeradmin; +ALTER ROLE db_owner ADD MEMBER rangeradmin; -- Grant equivalent high-level permissions +GO + +USE rangerkms; +CREATE LOGIN rangerkms WITH PASSWORD = 'rangerR0cks!'; +CREATE USER rangerkms FOR LOGIN rangerkms; +ALTER ROLE db_owner ADD MEMBER rangerkms; -- Grant equivalent high-level permissions +GO + +USE hive; +CREATE LOGIN hive WITH PASSWORD = 'rangerR0cks!'; +CREATE USER hive FOR LOGIN hive; +ALTER ROLE db_owner ADD MEMBER hive; -- Grant equivalent high-level permissions +GO +" diff --git a/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml b/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml new file mode 100644 index 0000000000..9cb35d4ddd --- /dev/null +++ b/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml @@ -0,0 +1,28 @@ +services: + ranger-db: + build: + context: . + dockerfile: Dockerfile.ranger-sqlserver + args: + - SQLSERVER_VERSION=${SQLSERVER_VERSION} + image: ranger-sqlserver + container_name: ranger-sqlserver + hostname: ranger-db.example.com + networks: + - ranger + healthcheck: + test: [ + "CMD-SHELL", + "sqlcmd -S localhost -U SA -P 'rangerR0cks!' -Q \"SELECT 1\" || exit 1" + ] + interval: 10s + timeout: 2s + retries: 30 + environment: + - ACCEPT_EULA=Y + - MSSQL_SA_PASSWORD=rangerR0cks! + - SA_PASSWORD=rangerR0cks! + +networks: + ranger: + name: rangernw diff --git a/dev-support/ranger-docker/download-archives.sh b/dev-support/ranger-docker/download-archives.sh index e7a4bf217b..7f11e21a29 100755 --- a/dev-support/ranger-docker/download-archives.sh +++ b/dev-support/ranger-docker/download-archives.sh @@ -44,6 +44,7 @@ downloadIfNotPresent() { downloadIfNotPresent postgresql-42.2.16.jre7.jar "https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7" downloadIfNotPresent mysql-connector-java-8.0.28.jar "https://search.maven.org/remotecontent?filepath=mysql/mysql-connector-java/8.0.28" downloadIfNotPresent ojdbc8.jar https://download.oracle.com/otn-pub/otn_software/jdbc/236 +downloadIfNotPresent mssql-jdbc-12.8.1.jre8.jar https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/12.8.1.jre8/mssql-jdbc-12.8.1.jre8.jar downloadIfNotPresent log4jdbc-1.2.jar https://repo1.maven.org/maven2/com/googlecode/log4jdbc/log4jdbc/1.2 if [[ $# -eq 0 ]] diff --git a/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties new file mode 100644 index 0000000000..5bbe1fc581 --- /dev/null +++ b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This file provides a list of the deployment variables for the Policy Manager Web Application +# + +PYTHON_COMMAND_INVOKER=python3 +RANGER_ADMIN_LOG_DIR=/var/log/ranger +RANGER_PID_DIR_PATH=/var/run/ranger +DB_FLAVOR=MSSQL +SQL_CONNECTOR_JAR=/usr/share/java/mssql.jar +RANGER_ADMIN_LOGBACK_CONF_FILE=/opt/ranger/admin/ews/webapp/WEB-INF/classes/conf/logback.xml + +db_root_user=sa +db_root_password=rangerR0cks! +db_host=ranger-db + +db_name=ranger +db_user=rangeradmin +db_password=rangerR0cks! + +postgres_core_file=db/postgres/optimized/current/ranger_core_db_postgres.sql +postgres_audit_file=db/postgres/xa_audit_db_postgres.sql +mysql_core_file=db/mysql/optimized/current/ranger_core_db_mysql.sql +mysql_audit_file=db/mysql/xa_audit_db.sql +oracle_core_file=db/oracle/optimized/current/ranger_core_db_oracle.sql +oracle_audit_file=db/oracle/xa_audit_db_oracle.sql +sqlserver_core_file=db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql +sqlserver_audit_file=db/sqlserver/xa_audit_db_sqlserver.sql + +rangerAdmin_password=rangerR0cks! +rangerTagsync_password=rangerR0cks! +rangerUsersync_password=rangerR0cks! +keyadmin_password=rangerR0cks! + + +audit_store=solr +audit_solr_urls=http://ranger-solr:8983/solr/ranger_audits +audit_solr_collection_name=ranger_audits + +# audit_store=elasticsearch +audit_elasticsearch_urls= +audit_elasticsearch_port=9200 +audit_elasticsearch_protocol=http +audit_elasticsearch_user=elastic +audit_elasticsearch_password=elasticsearch +audit_elasticsearch_index=ranger_audits +audit_elasticsearch_bootstrap_enabled=true + +policymgr_external_url=http://ranger-admin:6080 +policymgr_http_enabled=true + +unix_user=ranger +unix_user_pwd=ranger +unix_group=ranger + +# Following variables are referenced in db_setup.py. Do not remove these +sqlanywhere_core_file= +cred_keystore_filename= + +# ################# DO NOT MODIFY ANY VARIABLES BELOW ######################### +# +# --- These deployment variables are not to be modified unless you understand the full impact of the changes +# +################################################################################ +XAPOLICYMGR_DIR=$PWD +app_home=$PWD/ews/webapp +TMPFILE=$PWD/.fi_tmp +LOGFILE=$PWD/logfile +LOGFILES="$LOGFILE" + +JAVA_BIN='java' +JAVA_VERSION_REQUIRED='1.8' + +ranger_admin_max_heap_size=1g +#retry DB and Java patches after the given time in seconds. +PATCH_RETRY_INTERVAL=120 +STALE_PATCH_ENTRY_HOLD_TIME=10 + +hadoop_conf= +authentication_method=UNIX From ada1e07c6f5334ddff429656dd975cd143285714 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 3 Jan 2025 13:44:14 -0800 Subject: [PATCH 02/27] RANGER-5080: Refactor Dockerfile and update CI workflow for testing --- .github/workflows/maven.yml | 49 ++----------------- .../ranger-docker/Dockerfile.ranger-sqlserver | 23 ++------- 2 files changed, 10 insertions(+), 62 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f24342e993..04ab8745a8 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -128,55 +128,16 @@ jobs: docker compose -f docker-compose.ranger-base.yml build export DOCKER_BUILDKIT=1 export COMPOSE_DOCKER_CLI_BUILD=1 - export RANGER_DB_TYPE=postgres + export RANGER_DB_TYPE=sqlserver docker compose \ -f docker-compose.ranger-${RANGER_DB_TYPE}.yml \ - -f docker-compose.ranger.yml \ - -f docker-compose.ranger-usersync.yml \ - -f docker-compose.ranger-tagsync.yml \ - -f docker-compose.ranger-kms.yml \ - -f docker-compose.ranger-hadoop.yml \ - -f docker-compose.ranger-hbase.yml \ - -f docker-compose.ranger-kafka.yml \ - -f docker-compose.ranger-hive.yml \ - -f docker-compose.ranger-knox.yml \ - -f docker-compose.ranger-ozone.yml build + -f docker-compose.ranger.yml build - name: Bring up containers run: | cd dev-support/ranger-docker - ./scripts/ozone-plugin-docker-setup.sh - export RANGER_DB_TYPE=postgres + export RANGER_DB_TYPE=sqlserver docker compose \ -f docker-compose.ranger-${RANGER_DB_TYPE}.yml \ - -f docker-compose.ranger.yml \ - -f docker-compose.ranger-usersync.yml \ - -f docker-compose.ranger-tagsync.yml \ - -f docker-compose.ranger-kms.yml \ - -f docker-compose.ranger-hadoop.yml \ - -f docker-compose.ranger-hbase.yml \ - -f docker-compose.ranger-kafka.yml \ - -f docker-compose.ranger-hive.yml \ - -f docker-compose.ranger-knox.yml \ - -f docker-compose.ranger-ozone.yml up -d - - name: Check status of containers and remove them - run: | - sleep 60 - containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode); - flag=true; - for container in "${containers[@]}"; do - if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then - echo "Container $container is running!"; - else - flag=false; - echo "Container $container is NOT running!"; - fi - done - - if [[ $flag == true ]]; then - echo "All required containers are up and running"; - docker stop $(docker ps -q) && docker rm $(docker ps -aq); - else - docker stop $(docker ps -q) && docker rm $(docker ps -aq); - exit 1; - fi + -f docker-compose.ranger.yml up -d + diff --git a/dev-support/ranger-docker/Dockerfile.ranger-sqlserver b/dev-support/ranger-docker/Dockerfile.ranger-sqlserver index 0691eaf6f0..01407fcb28 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-sqlserver +++ b/dev-support/ranger-docker/Dockerfile.ranger-sqlserver @@ -16,26 +16,13 @@ ARG SQLSERVER_VERSION -#FROM mcr.microsoft.com/mssql/server:${SQLSERVER_VERSION} +FROM mcr.microsoft.com/mssql/server:${SQLSERVER_VERSION} -FROM mcr.microsoft.com/azure-sql-edge:latest +USER root -USER 0 - -RUN ACCEPT_EULA=Y apt-get update && \ - apt-get install -y --no-install-recommends \ - curl \ - apt-transport-https \ - gnupg \ - unixodbc-dev - -RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - \ - curl https://packages.microsoft.com/config/ubuntu/18.04/prod.list > /etc/apt/sources.list.d/mssql-tools.list - -RUN ACCEPT_EULA=Y apt-get update && \ - ACCEPT_EULA=Y apt-get install -y --no-install-recommends mssql-tools unixodbc-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* +RUN ACCEPT_EULA=Y apt-get update +RUN ACCEPT_EULA=Y apt-get install -y --no-install-recommends curl gnupg unixodbc-dev mssql-tools +RUN ACCEPT_EULA=Y apt-get clean && rm -rf /var/lib/apt/lists/* RUN mkdir -p /docker-entrypoint-initdb.d COPY config/init_mssql.sh /docker-entrypoint-initdb.d/ From 6fc807230a70829fd28e6e6ad448663c6dc1d943 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 3 Jan 2025 14:34:27 -0800 Subject: [PATCH 03/27] RANGER-5080: Add command to view sqlserver container logs --- .github/workflows/maven.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 04ab8745a8..25d4a586ec 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -141,3 +141,7 @@ jobs: -f docker-compose.ranger-${RANGER_DB_TYPE}.yml \ -f docker-compose.ranger.yml up -d + - name: View sql server logs + run: | + docker logs ranger-sqlserver + From ed1abfb439b35de49c5f35a6dbb85b3e525b7009 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 15 Jan 2025 14:17:37 -0800 Subject: [PATCH 04/27] Add healthcheck, refactor commands and disable SSL checks --- dev-support/ranger-docker/.env | 2 +- .../ranger-docker/Dockerfile.ranger-sqlserver | 16 ++++++++++------ .../ranger-docker/config/init_mssql.sh | 19 +++++++++++++++++-- .../docker-compose.ranger-sqlserver.yml | 10 ++++++---- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index 25ae55537b..0ef00da03e 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -36,7 +36,7 @@ UBI_VERSION=latest MARIADB_VERSION=10.7.3 POSTGRES_VERSION=12 ORACLE_VERSION=23.6 -SQLSERVER_VERSION=2022-latest +SQLSERVER_VERSION=2019-latest ENABLE_DB_MOUNT=true ZK_VERSION=3.9.2 SOLR_VERSION=8.11.3 diff --git a/dev-support/ranger-docker/Dockerfile.ranger-sqlserver b/dev-support/ranger-docker/Dockerfile.ranger-sqlserver index 01407fcb28..3559fa49dd 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-sqlserver +++ b/dev-support/ranger-docker/Dockerfile.ranger-sqlserver @@ -18,14 +18,18 @@ ARG SQLSERVER_VERSION FROM mcr.microsoft.com/mssql/server:${SQLSERVER_VERSION} -USER root +ENV ACCEPT_EULA=Y +ENV MSSQL_SA_PASSWORD=rangerR0cks! + +EXPOSE 1433 -RUN ACCEPT_EULA=Y apt-get update -RUN ACCEPT_EULA=Y apt-get install -y --no-install-recommends curl gnupg unixodbc-dev mssql-tools -RUN ACCEPT_EULA=Y apt-get clean && rm -rf /var/lib/apt/lists/* +USER root RUN mkdir -p /docker-entrypoint-initdb.d COPY config/init_mssql.sh /docker-entrypoint-initdb.d/ -RUN chown -R mssql /docker-entrypoint-initdb.d/ -ENV MSSQL_PASSWORD=rangerR0cks! +RUN chown -R mssql /docker-entrypoint-initdb.d/ +RUN chmod +x /docker-entrypoint-initdb.d/init_mssql.sh + USER mssql + +ENTRYPOINT ["/docker-entrypoint-initdb.d/init_mssql.sh"] diff --git a/dev-support/ranger-docker/config/init_mssql.sh b/dev-support/ranger-docker/config/init_mssql.sh index 45ed3b243c..d179f4bcff 100644 --- a/dev-support/ranger-docker/config/init_mssql.sh +++ b/dev-support/ranger-docker/config/init_mssql.sh @@ -16,6 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +/opt/mssql/bin/sqlservr & # Wait for SQL Server to be ready echo "Waiting for SQL Server to start..." @@ -38,8 +39,18 @@ if [ $i -eq $RETRIES ]; then exit 1 fi +# Disable SSL encryption by setting 'force encryption' to 0 +echo "Disabling SSL encryption..." +/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P 'rangerR0cks!' -Q " +EXEC sp_configure 'show advanced options', 1; +RECONFIGURE; +EXEC sp_configure 'force encryption', 0; +RECONFIGURE; +" +echo "SSL encryption disabled." + -sqlcmd -S localhost -U mssql -P 'rangerR0cks!' -Q " +/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P 'rangerR0cks!' -Q " -- Set the database context USE master; @@ -68,4 +79,8 @@ CREATE LOGIN hive WITH PASSWORD = 'rangerR0cks!'; CREATE USER hive FOR LOGIN hive; ALTER ROLE db_owner ADD MEMBER hive; -- Grant equivalent high-level permissions GO -" +" -C + +# Bring SQL Server to the foreground +wait -n +exec /opt/mssql/bin/sqlservr diff --git a/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml b/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml index 9cb35d4ddd..ddda04f751 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml @@ -11,13 +11,15 @@ services: networks: - ranger healthcheck: + # -C bypasses SSL validation test: [ "CMD-SHELL", - "sqlcmd -S localhost -U SA -P 'rangerR0cks!' -Q \"SELECT 1\" || exit 1" + "/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P rangerR0cks! -Q \"SELECT 1\" -C" ] - interval: 10s - timeout: 2s - retries: 30 + interval: 15s + timeout: 10s + retries: 3 + start_period: 10s environment: - ACCEPT_EULA=Y - MSSQL_SA_PASSWORD=rangerR0cks! From e72346a523ef7d8cb16005f5515b60477d3644a5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 16 Jan 2025 20:40:27 -0800 Subject: [PATCH 05/27] Add support in dba_script.py for custom jdbc url in SQL Server --- .github/workflows/maven.yml | 3 +- dev-support/ranger-docker/Dockerfile.ranger | 3 + .../ranger-docker/config/init_mssql.sh | 13 +- .../ranger-admin-install-sqlserver.properties | 4 + .../ranger-kms-install-sqlserver.properties | 222 ++++++++++++++++++ kms/scripts/dba_script.py | 24 +- security-admin/scripts/dba_script.py | 25 +- 7 files changed, 272 insertions(+), 22 deletions(-) create mode 100644 dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 25d4a586ec..a6b2fa60bc 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -143,5 +143,6 @@ jobs: - name: View sql server logs run: | + sleep 30 docker logs ranger-sqlserver - + docker logs ranger diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger index 9d14a1eb9c..625d6bb40f 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger @@ -62,6 +62,9 @@ FROM ranger AS ranger_sqlserver COPY ./downloads/mssql-jdbc-12.8.1.jre8.jar /home/ranger/dist/ RUN mv /home/ranger/dist/mssql-jdbc-12.8.1.jre8.jar /usr/share/java/mssql.jar +# to trust the self-signed certificate +ENV JAVA_OPTS="${JAVA_OPTS} -DtrustServerCertificate=true" + FROM ranger_${RANGER_DB_TYPE} USER ranger diff --git a/dev-support/ranger-docker/config/init_mssql.sh b/dev-support/ranger-docker/config/init_mssql.sh index d179f4bcff..32036c9eae 100644 --- a/dev-support/ranger-docker/config/init_mssql.sh +++ b/dev-support/ranger-docker/config/init_mssql.sh @@ -24,7 +24,7 @@ RETRIES=30 # Number of retries SLEEP_INTERVAL=5 # Seconds to wait between retries for i in $(seq 1 $RETRIES); do # Try to connect to SQL Server - /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "rangerR0cks!" -Q "SELECT 1" > /dev/null 2>&1 + /opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P "rangerR0cks!" -Q "SELECT 1" -C > /dev/null 2>&1 if [ $? -eq 0 ]; then echo "SQL Server is ready!" break @@ -39,17 +39,6 @@ if [ $i -eq $RETRIES ]; then exit 1 fi -# Disable SSL encryption by setting 'force encryption' to 0 -echo "Disabling SSL encryption..." -/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P 'rangerR0cks!' -Q " -EXEC sp_configure 'show advanced options', 1; -RECONFIGURE; -EXEC sp_configure 'force encryption', 0; -RECONFIGURE; -" -echo "SSL encryption disabled." - - /opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P 'rangerR0cks!' -Q " -- Set the database context diff --git a/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties index 5bbe1fc581..2815e1abae 100644 --- a/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties +++ b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties @@ -41,6 +41,10 @@ oracle_audit_file=db/oracle/xa_audit_db_oracle.sql sqlserver_core_file=db/sqlserver/optimized/current/ranger_core_db_sqlserver.sql sqlserver_audit_file=db/sqlserver/xa_audit_db_sqlserver.sql +# For over-riding the jdbc url +is_override_db_connection_string=true +db_override_jdbc_connection_string="jdbc:sqlserver://ranger-db;databaseName=ranger;trustServerCertificate=true;" + rangerAdmin_password=rangerR0cks! rangerTagsync_password=rangerR0cks! rangerUsersync_password=rangerR0cks! diff --git a/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties new file mode 100644 index 0000000000..66e17a2725 --- /dev/null +++ b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties @@ -0,0 +1,222 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This file provides a list of the deployment variables for the Ranger KMS Web Application +# + +PYTHON_COMMAND_INVOKER=python3 +DB_FLAVOR=POSTGRES +SQL_CONNECTOR_JAR=/usr/share/java/postgresql.jar + +db_root_user=postgres +db_root_password=rangerR0cks! +db_host=ranger-db + +db_name=rangerkms +db_user=rangerkms +db_password=rangerR0cks! + +# Following variables are referenced in db_setup.py. Do not remove these +mysql_core_file=db/mysql/kms_core_db.sql +postgres_core_file=db/postgres/kms_core_db_postgres.sql +sqlserver_core_file=db/sqlserver/kms_core_db_sqlserver.sql +oracle_core_file= +sqlanywhere_core_file= + +# For over-riding the jdbc url +is_override_db_connection_string=true +db_override_jdbc_connection_string="jdbc:sqlserver://ranger-db;databaseName=rangerkms;trustServerCertificate=true;" + + +#SSL config +db_ssl_enabled=false +db_ssl_required=false +db_ssl_verifyServerCertificate=false +#db_ssl_auth_type=1-way|2-way, where 1-way represents standard one way ssl authentication and 2-way represents mutual ssl authentication +db_ssl_auth_type=2-way +javax_net_ssl_keyStore= +javax_net_ssl_keyStorePassword= +javax_net_ssl_trustStore= +javax_net_ssl_trustStorePassword= +javax_net_ssl_trustStore_type=jks +javax_net_ssl_keyStore_type=jks + +# For postgresql db +db_ssl_certificate_file= + +#------------------------- DB CONFIG - END ---------------------------------- +#KMS Server config +ranger_kms_http_enabled=true +ranger_kms_https_keystore_file= +ranger_kms_https_keystore_keyalias=rangerkms +ranger_kms_https_keystore_password= + +#------------------------- RANGER KMS Install Dir ------------------ +COMPONENT_INSTALL_DIR_NAME=/opt/ranger/kms + +#------------------------- RANGER KMS Master Key Crypt Key ------------------ +KMS_MASTER_KEY_PASSWD=Str0ngPassw0rd + +#------------------------- Ranger KMS Kerberos Configuration --------------------------- +kms_principal= +kms_keytab= +hadoop_conf= + +#------------------------- Ranger KMS HSM CONFIG ------------------------------ +HSM_TYPE=LunaProvider +HSM_ENABLED=false +HSM_PARTITION_NAME=par19 +HSM_PARTITION_PASSWORD=S@fenet123 + +#------------------------- Ranger SAFENET KEYSECURE CONFIG ------------------------------ +KEYSECURE_ENABLED=false +KEYSECURE_USER_PASSWORD_AUTHENTICATION=true +KEYSECURE_MASTERKEY_NAME=safenetkeysecure +KEYSECURE_USERNAME=user1 +KEYSECURE_PASSWORD=t1e2s3t4 +KEYSECURE_HOSTNAME=SunPKCS11-keysecurehn +KEYSECURE_MASTER_KEY_SIZE=256 +KEYSECURE_LIB_CONFIG_PATH=/opt/safenetConf/64/8.3.1/sunpkcs11.cfg + +#------------------------- Ranger Azure Key Vault ------------------------------ +AZURE_KEYVAULT_ENABLED=false +AZURE_KEYVAULT_SSL_ENABLED=false +AZURE_CLIENT_ID=50fd7ca6-fd4f-4785-a13f-1a6cc4e95e42 +AZURE_CLIENT_SECRET= +AZURE_AUTH_KEYVAULT_CERTIFICATE_PATH=/home/machine/Desktop/azureAuthCertificate/keyvault-MyCert.pfx +# Initialize below prop if your certificate file has any password +#AZURE_AUTH_KEYVAULT_CERTIFICATE_PASSWORD=certPass +AZURE_MASTERKEY_NAME=RangerMasterKey +# E.G. RSA, RSA_HSM, EC, EC_HSM, OCT +AZURE_MASTER_KEY_TYPE=RSA +# E.G. RSA_OAEP, RSA_OAEP_256, RSA1_5, RSA_OAEP +ZONE_KEY_ENCRYPTION_ALGO=RSA_OAEP +AZURE_KEYVAULT_URL=https://shahkeyvault.vault.azure.net/ + +#------------------------- Ranger Google Cloud HSM ------------------------------ +IS_GCP_ENABLED=false +GCP_KEYRING_ID= +GCP_CRED_JSON_FILE=/full/path/to/credfile.json +GCP_PROJECT_ID= +GCP_LOCATION_ID= +GCP_MASTER_KEY_NAME=MyMasterKeyNameChangeIt + +#------------------------- Ranger Tencent KMS ------------------------------ +TENCENT_KMS_ENABLED=false +TENCENT_MASTERKEY_ID=b756b016-6e11-11ec-a735-525400fe0300 +TENCENT_CLIENT_ID=AKIDrXx6ybx2qNdiaBWaNs76pGQJvFJ6crpW +TENCENT_CLIENT_SECRET= +TENCENT_CLIENT_REGION=ap-beijing + +# ------- UNIX User CONFIG ---------------- +# +unix_user=rangerkms +unix_user_pwd=kms +unix_group=ranger + +# +# ------- UNIX User CONFIG - END ---------------- +# + +POLICY_MGR_URL=http://ranger:6080 +REPOSITORY_NAME=dev_kms + +# AUDIT configuration with V3 properties +XAAUDIT.SOLR.IS_ENABLED=true +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://ranger-solr:8983/solr/ranger_audits +XAAUDIT.SUMMARY.ENABLE=true + +# Following properties are needed to get past installation script! Please don't remove +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=/ranger/audit +XAAUDIT.HDFS.DESTINTATION_FILE=hive +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/hive/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/hive/audit/archive +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +XAAUDIT.SOLR.ENABLE=true +XAAUDIT.SOLR.URL=http://ranger-solr:8983/solr/ranger_audits +XAAUDIT.SOLR.USER=NONE +XAAUDIT.SOLR.PASSWORD=NONE +XAAUDIT.SOLR.ZOOKEEPER=NONE +XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/hive/audit/solr/spool + +XAAUDIT.ELASTICSEARCH.ENABLE=false +XAAUDIT.ELASTICSEARCH.URL=NONE +XAAUDIT.ELASTICSEARCH.USER=NONE +XAAUDIT.ELASTICSEARCH.PASSWORD=NONE +XAAUDIT.ELASTICSEARCH.INDEX=NONE +XAAUDIT.ELASTICSEARCH.PORT=NONE +XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE + +XAAUDIT.HDFS.ENABLE=true +XAAUDIT.HDFS.HDFS_DIR=hdfs://ranger-hadoop:9000/ranger/audit +XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/hive/audit/hdfs/spool + +XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME +XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY +XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER +XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER + +XAAUDIT.LOG4J.ENABLE=false +XAAUDIT.LOG4J.IS_ASYNC=false +XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240 +XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000 +XAAUDIT.LOG4J.DESTINATION.LOG4J=false +XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit + +XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false +XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE +XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE +XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE + +SSL_KEYSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/hive/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit + + +# Custom log directory path +RANGER_KMS_LOG_DIR=/var/log/ranger/kms + +#PID file path +RANGER_KMS_PID_DIR_PATH=/var/run/ranger_kms +# ################# DO NOT MODIFY ANY VARIABLES BELOW ######################### +# +# --- These deployment variables are not to be modified unless you understand the full impact of the changes +# +################################################################################ +KMS_DIR=$PWD +app_home=$PWD/ews/webapp +TMPFILE=$PWD/.fi_tmp +LOGFILE=$PWD/logfile + +JAVA_BIN='java' +JAVA_VERSION_REQUIRED='1.8' +JAVA_ORACLE='Java(TM) SE Runtime Environment' + +cred_keystore_filename=$app_home/WEB-INF/classes/conf/.jceks/rangerkms.jceks + +KMS_BLACKLIST_DECRYPT_EEK=hdfs diff --git a/kms/scripts/dba_script.py b/kms/scripts/dba_script.py index 7e7b4e1ce3..5d19a7d215 100755 --- a/kms/scripts/dba_script.py +++ b/kms/scripts/dba_script.py @@ -825,19 +825,27 @@ def writeDrymodeCmd(self, xa_db_root_user, xa_db_root_password, db_user, db_pass class SqlServerConf(BaseDB): # Constructor - def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN): + def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN, is_db_override_jdbc_connection_string, db_override_jdbc_connection_string): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN + self.is_db_override_jdbc_connection_string = is_db_override_jdbc_connection_string + self.db_override_jdbc_connection_string = db_override_jdbc_connection_string def get_jisql_cmd(self, user, password, db_name): #TODO: User array for forming command path = RANGER_KMS_HOME self.JAVA_BIN = self.JAVA_BIN.strip("'") if is_unix: - jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name) + if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: + jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring %s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.db_override_jdbc_connection_string) + else: + jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name) elif os_name == "WINDOWS": - jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name) + if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: + jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring %s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.db_override_jdbc_connection_string) + else: + jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name) return jisql_cmd def verify_user(self, root_user, db_root_password, db_user,dryMode): @@ -1398,6 +1406,14 @@ def main(argv): javax_net_ssl_keyStorePassword='' javax_net_ssl_trustStore='' javax_net_ssl_trustStorePassword='' + + is_override_db_connection_string='false' + db_override_jdbc_connection_string='' + if 'is_override_db_connection_string' in globalDict: + is_override_db_connection_string=globalDict['is_override_db_connection_string'].lower() + if 'db_override_jdbc_connection_string' in globalDict: + db_override_jdbc_connection_string=globalDict['db_override_jdbc_connection_string'].strip() + if XA_DB_FLAVOR == "MYSQL" or XA_DB_FLAVOR == "POSTGRES": if 'db_ssl_enabled' in globalDict: db_ssl_enabled=globalDict['db_ssl_enabled'].lower() @@ -1451,7 +1467,7 @@ def main(argv): elif XA_DB_FLAVOR == "MSSQL": SQLSERVER_CONNECTOR_JAR=CONNECTOR_JAR - xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN) + xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN, is_override_db_connection_string, db_override_jdbc_connection_string) xa_db_core_file = os.path.join(RANGER_KMS_HOME,sqlserver_core_file) elif XA_DB_FLAVOR == "SQLA": diff --git a/security-admin/scripts/dba_script.py b/security-admin/scripts/dba_script.py index 5f96ef26d5..8a334de063 100644 --- a/security-admin/scripts/dba_script.py +++ b/security-admin/scripts/dba_script.py @@ -1004,19 +1004,27 @@ def writeDrymodeCmd(self, xa_db_host, audit_db_host, xa_db_root_user, xa_db_root class SqlServerConf(BaseDB): # Constructor - def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN): + def __init__(self, host, SQL_CONNECTOR_JAR, JAVA_BIN, is_db_override_jdbc_connection_string, db_override_jdbc_connection_string): self.host = host self.SQL_CONNECTOR_JAR = SQL_CONNECTOR_JAR self.JAVA_BIN = JAVA_BIN + self.is_db_override_jdbc_connection_string = is_db_override_jdbc_connection_string + self.db_override_jdbc_connection_string = db_override_jdbc_connection_string def get_jisql_cmd(self, user, password, db_name): #TODO: User array for forming command path = RANGER_ADMIN_HOME self.JAVA_BIN = self.JAVA_BIN.strip("'") if is_unix: - jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name) + if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: + jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring %s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.db_override_jdbc_connection_string) + else: + jisql_cmd = "%s -cp %s:%s/jisql/lib/* org.apache.util.sql.Jisql -user %s -p '%s' -driver mssql -cstring jdbc:sqlserver://%s\\;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path,user, password, self.host,db_name) elif os_name == "WINDOWS": - jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name) + if self.is_db_override_jdbc_connection_string == 'true' and self.db_override_jdbc_connection_string is not None and len(self.db_override_jdbc_connection_string) > 0: + jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring %s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.db_override_jdbc_connection_string) + else: + jisql_cmd = "%s -cp %s;%s\\jisql\\lib\\* org.apache.util.sql.Jisql -user %s -p \"%s\" -driver mssql -cstring jdbc:sqlserver://%s;databaseName=%s -noheader -trim"%(self.JAVA_BIN, self.SQL_CONNECTOR_JAR, path, user, password, self.host,db_name) return jisql_cmd def verify_user(self, root_user, db_root_password, db_user,dryMode): @@ -1714,6 +1722,13 @@ def main(argv): log("[E] Invalid ssl keystore password!","error") sys.exit(1) + is_override_db_connection_string='false' + db_override_jdbc_connection_string='' + if 'is_override_db_connection_string' in globalDict: + is_override_db_connection_string=globalDict['is_override_db_connection_string'].lower() + if 'db_override_jdbc_connection_string' in globalDict: + db_override_jdbc_connection_string=globalDict['db_override_jdbc_connection_string'].strip() + if XA_DB_FLAVOR == "MYSQL": MYSQL_CONNECTOR_JAR=CONNECTOR_JAR xa_sqlObj = MysqlConf(xa_db_host, MYSQL_CONNECTOR_JAR, JAVA_BIN,db_ssl_enabled,db_ssl_required,db_ssl_verifyServerCertificate,javax_net_ssl_keyStore,javax_net_ssl_keyStorePassword,javax_net_ssl_trustStore,javax_net_ssl_trustStorePassword,db_ssl_auth_type) @@ -1740,7 +1755,7 @@ def main(argv): elif XA_DB_FLAVOR == "MSSQL": SQLSERVER_CONNECTOR_JAR=CONNECTOR_JAR - xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN) + xa_sqlObj = SqlServerConf(xa_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN, is_override_db_connection_string, db_override_jdbc_connection_string) xa_db_version_file = os.path.join(RANGER_ADMIN_HOME,sqlserver_dbversion_catalog) xa_db_core_file = os.path.join(RANGER_ADMIN_HOME,sqlserver_core_file) xa_patch_file = os.path.join(RANGER_ADMIN_HOME,sqlserver_patches) @@ -1779,7 +1794,7 @@ def main(argv): elif AUDIT_DB_FLAVOR == "MSSQL": SQLSERVER_CONNECTOR_JAR=CONNECTOR_JAR - audit_sqlObj = SqlServerConf(audit_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN) + audit_sqlObj = SqlServerConf(audit_db_host, SQLSERVER_CONNECTOR_JAR, JAVA_BIN, is_override_db_connection_string, db_override_jdbc_connection_string) audit_db_file = os.path.join(RANGER_ADMIN_HOME,sqlserver_audit_file) elif AUDIT_DB_FLAVOR == "SQLA": From 9bdc2b29dd23cbbd63ca0075ec2cc0b369e90129 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 16 Jan 2025 21:42:16 -0800 Subject: [PATCH 06/27] Remove downloads cache and download mssql jdbc driver --- .github/workflows/maven.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index a6b2fa60bc..d87ee719b6 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -105,19 +105,11 @@ jobs: run: | cp ranger-*.tar.gz dev-support/ranger-docker/dist cp version dev-support/ranger-docker/dist - - - name: Cache downloaded archives - uses: actions/cache@v4 - with: - path: dev-support/ranger-docker/downloads - key: ${{ runner.os }}-ranger-downloads-${{ hashFiles('dev-support/ranger-docker/.env') }} - restore-keys: | - ${{ runner.os }}-ranger-downloads- - name: Run download-archives.sh run: | cd dev-support/ranger-docker - ./download-archives.sh hadoop hive hbase kafka knox ozone + ./download-archives.sh kafka - name: Clean up Docker space run: docker system prune --all --force --volumes From ac477f50bc846929239ccf487378823c88b51ff0 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 16 Jan 2025 22:51:47 -0800 Subject: [PATCH 07/27] Fix maven download url for SQL Server JDBC Driver --- dev-support/ranger-docker/download-archives.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/ranger-docker/download-archives.sh b/dev-support/ranger-docker/download-archives.sh index 7f11e21a29..0bfca41ade 100755 --- a/dev-support/ranger-docker/download-archives.sh +++ b/dev-support/ranger-docker/download-archives.sh @@ -44,7 +44,7 @@ downloadIfNotPresent() { downloadIfNotPresent postgresql-42.2.16.jre7.jar "https://search.maven.org/remotecontent?filepath=org/postgresql/postgresql/42.2.16.jre7" downloadIfNotPresent mysql-connector-java-8.0.28.jar "https://search.maven.org/remotecontent?filepath=mysql/mysql-connector-java/8.0.28" downloadIfNotPresent ojdbc8.jar https://download.oracle.com/otn-pub/otn_software/jdbc/236 -downloadIfNotPresent mssql-jdbc-12.8.1.jre8.jar https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/12.8.1.jre8/mssql-jdbc-12.8.1.jre8.jar +downloadIfNotPresent mssql-jdbc-12.8.1.jre8.jar https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/12.8.1.jre8 downloadIfNotPresent log4jdbc-1.2.jar https://repo1.maven.org/maven2/com/googlecode/log4jdbc/log4jdbc/1.2 if [[ $# -eq 0 ]] From 279871f3ab4c7a00d7e8e924096b62442211135f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 18 Jan 2025 01:27:01 -0800 Subject: [PATCH 08/27] Add -DtrustServerCertificate=true in db_setup.py --- security-admin/scripts/db_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security-admin/scripts/db_setup.py b/security-admin/scripts/db_setup.py index fcc57ef2f0..fb7d1b133b 100644 --- a/security-admin/scripts/db_setup.py +++ b/security-admin/scripts/db_setup.py @@ -654,7 +654,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s %s -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,JAVA_OPTS,ranger_log_dir,logback_conf_file,os_user,client_host,path,className,'"'+userName+'"','"'+oldPassword+'"','"'+newPassword+'"') + get_java_cmd = "%s %s -DtrustServerCertificate=true -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,JAVA_OPTS,ranger_log_dir,logback_conf_file,os_user,client_host,path,className,'"'+userName+'"','"'+oldPassword+'"','"'+newPassword+'"') if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": From 95007fabc364e6ceb02b3ab40941dcfb71922a17 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 18 Jan 2025 02:03:45 -0800 Subject: [PATCH 09/27] Add -DtrustServerCertificate=true in db_setup.py 2 --- security-admin/scripts/db_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security-admin/scripts/db_setup.py b/security-admin/scripts/db_setup.py index fb7d1b133b..409253afb5 100644 --- a/security-admin/scripts/db_setup.py +++ b/security-admin/scripts/db_setup.py @@ -752,7 +752,7 @@ def change_all_admin_default_password(self, xa_db_host, db_user, db_password, db path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s %s -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.cliutil.%s %s -default"%(self.JAVA_BIN,JAVA_OPTS,ranger_log_dir,logback_conf_file,os_user,client_host,path,className, userPwdString) + get_java_cmd = "%s %s -DtrustServerCertificate=true -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.cliutil.%s %s -default"%(self.JAVA_BIN,JAVA_OPTS,ranger_log_dir,logback_conf_file,os_user,client_host,path,className, userPwdString) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": From 89afc63197112d40d235424e59e8c286535785bd Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 18 Jan 2025 17:10:27 -0800 Subject: [PATCH 10/27] Add -DtrustServerCertificate=true in db_setup.py 3 --- security-admin/scripts/db_setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/security-admin/scripts/db_setup.py b/security-admin/scripts/db_setup.py index 409253afb5..007a0f6a18 100644 --- a/security-admin/scripts/db_setup.py +++ b/security-admin/scripts/db_setup.py @@ -564,6 +564,7 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name, my_dic elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) get_java_cmd = "%s %s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,JAVA_OPTS,globalDict['ranger_admin_max_heap_size'],ranger_log_dir,logback_conf_file,os_user,client_host,path,className) + print(get_java_cmd) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -655,6 +656,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) get_java_cmd = "%s %s -DtrustServerCertificate=true -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,JAVA_OPTS,ranger_log_dir,logback_conf_file,os_user,client_host,path,className,'"'+userName+'"','"'+oldPassword+'"','"'+newPassword+'"') + print(get_java_cmd) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -753,6 +755,7 @@ def change_all_admin_default_password(self, xa_db_host, db_user, db_password, db elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) get_java_cmd = "%s %s -DtrustServerCertificate=true -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.cliutil.%s %s -default"%(self.JAVA_BIN,JAVA_OPTS,ranger_log_dir,logback_conf_file,os_user,client_host,path,className, userPwdString) + print(get_java_cmd) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": From 59fc66a1d519394a515311a7c5248854922478b5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 20 Jan 2025 01:06:07 -0800 Subject: [PATCH 11/27] Add -DtrustServerCertificate=true in setup.sh --- dev-support/ranger-docker/Dockerfile.ranger | 1 + security-admin/scripts/setup.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger index 625d6bb40f..4adae45925 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger @@ -23,6 +23,7 @@ ARG TARGETARCH ARG RANGER_ADMIN_JAVA_VERSION ENV JAVA_HOME=/usr/lib/jvm/java-1.${RANGER_ADMIN_JAVA_VERSION}.0-openjdk-${TARGETARCH} +# ENV RANGER_ADMIN_CONF=/opt/ranger/admin/conf RUN if [ "${OS_NAME}" = "UBUNTU" ]; then\ update-java-alternatives --set "$JAVA_HOME";\ diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh index b5eec25dd8..96acb4b39e 100755 --- a/security-admin/scripts/setup.sh +++ b/security-admin/scripts/setup.sh @@ -746,7 +746,7 @@ update_properties() { if [ "${DB_FLAVOR}" == "MSSQL" ] then propertyName=ranger.jpa.jdbc.url - newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};trustServerCertificate=true;" updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger propertyName=ranger.jpa.jdbc.dialect From ceca20361def549ded366941f1d4f76da6f33b33 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 23 Jan 2025 15:48:31 -0800 Subject: [PATCH 12/27] Refactoring and cleaning out unrequired stuff --- .github/workflows/maven.yml | 55 ++++++++++++++++++- dev-support/ranger-docker/Dockerfile.ranger | 6 -- .../docker-compose.ranger-sqlserver.yml | 7 +-- security-admin/scripts/db_setup.py | 7 +-- 4 files changed, 55 insertions(+), 20 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index d87ee719b6..16f5d8fc4d 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -106,10 +106,18 @@ jobs: cp ranger-*.tar.gz dev-support/ranger-docker/dist cp version dev-support/ranger-docker/dist + - name: Cache downloaded archives + uses: actions/cache@v4 + with: + path: dev-support/ranger-docker/downloads + key: ${{ runner.os }}-ranger-downloads-${{ hashFiles('dev-support/ranger-docker/.env') }} + restore-keys: | + ${{ runner.os }}-ranger-downloads- + - name: Run download-archives.sh run: | cd dev-support/ranger-docker - ./download-archives.sh kafka + ./download-archives.sh hadoop hive hbase kafka knox ozone - name: Clean up Docker space run: docker system prune --all --force --volumes @@ -123,18 +131,59 @@ jobs: export RANGER_DB_TYPE=sqlserver docker compose \ -f docker-compose.ranger-${RANGER_DB_TYPE}.yml \ - -f docker-compose.ranger.yml build + -f docker-compose.ranger.yml \ + -f docker-compose.ranger-usersync.yml \ + -f docker-compose.ranger-tagsync.yml \ + -f docker-compose.ranger-kms.yml \ + -f docker-compose.ranger-hadoop.yml \ + -f docker-compose.ranger-hbase.yml \ + -f docker-compose.ranger-kafka.yml \ + -f docker-compose.ranger-hive.yml \ + -f docker-compose.ranger-knox.yml \ + -f docker-compose.ranger-ozone.yml build - name: Bring up containers run: | cd dev-support/ranger-docker + ./scripts/ozone-plugin-docker-setup.sh export RANGER_DB_TYPE=sqlserver docker compose \ -f docker-compose.ranger-${RANGER_DB_TYPE}.yml \ - -f docker-compose.ranger.yml up -d + -f docker-compose.ranger.yml \ + -f docker-compose.ranger-usersync.yml \ + -f docker-compose.ranger-tagsync.yml \ + -f docker-compose.ranger-kms.yml \ + -f docker-compose.ranger-hadoop.yml \ + -f docker-compose.ranger-hbase.yml \ + -f docker-compose.ranger-kafka.yml \ + -f docker-compose.ranger-hive.yml \ + -f docker-compose.ranger-knox.yml \ + -f docker-compose.ranger-ozone.yml up -d - name: View sql server logs run: | sleep 30 docker logs ranger-sqlserver docker logs ranger + + - name: Check status of containers and remove them + run: | + sleep 60 + containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode); + flag=true; + for container in "${containers[@]}"; do + if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then + echo "Container $container is running!"; + else + flag=false; + echo "Container $container is NOT running!"; + fi + done + + if [[ $flag == true ]]; then + echo "All required containers are up and running"; + docker stop $(docker ps -q) && docker rm $(docker ps -aq); + else + docker stop $(docker ps -q) && docker rm $(docker ps -aq); + exit 1; + fi diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger index 4adae45925..708b459e6a 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger @@ -22,9 +22,6 @@ ARG RANGER_DB_TYPE ARG TARGETARCH ARG RANGER_ADMIN_JAVA_VERSION -ENV JAVA_HOME=/usr/lib/jvm/java-1.${RANGER_ADMIN_JAVA_VERSION}.0-openjdk-${TARGETARCH} -# ENV RANGER_ADMIN_CONF=/opt/ranger/admin/conf - RUN if [ "${OS_NAME}" = "UBUNTU" ]; then\ update-java-alternatives --set "$JAVA_HOME";\ fi @@ -63,9 +60,6 @@ FROM ranger AS ranger_sqlserver COPY ./downloads/mssql-jdbc-12.8.1.jre8.jar /home/ranger/dist/ RUN mv /home/ranger/dist/mssql-jdbc-12.8.1.jre8.jar /usr/share/java/mssql.jar -# to trust the self-signed certificate -ENV JAVA_OPTS="${JAVA_OPTS} -DtrustServerCertificate=true" - FROM ranger_${RANGER_DB_TYPE} USER ranger diff --git a/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml b/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml index ddda04f751..f1e790019f 100644 --- a/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml +++ b/dev-support/ranger-docker/docker-compose.ranger-sqlserver.yml @@ -11,19 +11,14 @@ services: networks: - ranger healthcheck: - # -C bypasses SSL validation test: [ "CMD-SHELL", - "/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P rangerR0cks! -Q \"SELECT 1\" -C" + "/opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P rangerR0cks! -Q \"SELECT 1\" -C" # -C bypasses SSL validation ] interval: 15s timeout: 10s retries: 3 start_period: 10s - environment: - - ACCEPT_EULA=Y - - MSSQL_SA_PASSWORD=rangerR0cks! - - SA_PASSWORD=rangerR0cks! networks: ranger: diff --git a/security-admin/scripts/db_setup.py b/security-admin/scripts/db_setup.py index 007a0f6a18..fcc57ef2f0 100644 --- a/security-admin/scripts/db_setup.py +++ b/security-admin/scripts/db_setup.py @@ -564,7 +564,6 @@ def execute_java_patches(self, xa_db_host, db_user, db_password, db_name, my_dic elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) get_java_cmd = "%s %s -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=200m -Xmx%s -Xms1g -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.%s"%(self.JAVA_BIN,JAVA_OPTS,globalDict['ranger_admin_max_heap_size'],ranger_log_dir,logback_conf_file,os_user,client_host,path,className) - print(get_java_cmd) if is_unix: ret = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -655,8 +654,7 @@ def change_admin_default_password(self, xa_db_host, db_user, db_password, db_nam path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s %s -DtrustServerCertificate=true -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,JAVA_OPTS,ranger_log_dir,logback_conf_file,os_user,client_host,path,className,'"'+userName+'"','"'+oldPassword+'"','"'+newPassword+'"') - print(get_java_cmd) + get_java_cmd = "%s %s -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.cliutil.%s %s %s %s -default"%(self.JAVA_BIN,JAVA_OPTS,ranger_log_dir,logback_conf_file,os_user,client_host,path,className,'"'+userName+'"','"'+oldPassword+'"','"'+newPassword+'"') if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": @@ -754,8 +752,7 @@ def change_all_admin_default_password(self, xa_db_host, db_user, db_password, db path = os.path.join("%s","WEB-INF","classes","conf:%s","WEB-INF","classes","lib","*:%s","WEB-INF",":%s","META-INF",":%s","WEB-INF","lib","*:%s","WEB-INF","classes",":%s","WEB-INF","classes","META-INF:%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) elif os_name == "WINDOWS": path = os.path.join("%s","WEB-INF","classes","conf;%s","WEB-INF","classes","lib","*;%s","WEB-INF",";%s","META-INF",";%s","WEB-INF","lib","*;%s","WEB-INF","classes",";%s","WEB-INF","classes","META-INF;%s" )%(app_home ,app_home ,app_home, app_home, app_home, app_home ,app_home ,self.SQL_CONNECTOR_JAR) - get_java_cmd = "%s %s -DtrustServerCertificate=true -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.cliutil.%s %s -default"%(self.JAVA_BIN,JAVA_OPTS,ranger_log_dir,logback_conf_file,os_user,client_host,path,className, userPwdString) - print(get_java_cmd) + get_java_cmd = "%s %s -Dlogdir=%s -Dlogback.configurationFile=%s -Duser=%s -Dhostname=%s -cp %s org.apache.ranger.patch.cliutil.%s %s -default"%(self.JAVA_BIN,JAVA_OPTS,ranger_log_dir,logback_conf_file,os_user,client_host,path,className, userPwdString) if is_unix: status = subprocess.call(shlex.split(get_java_cmd)) elif os_name == "WINDOWS": From 6569039aefa0a26f760f100fcbe08913538af432 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 23 Jan 2025 16:01:32 -0800 Subject: [PATCH 13/27] Add property to enable trustServerCertificate=true in docker env --- dev-support/ranger-docker/Dockerfile.ranger | 2 ++ .../scripts/ranger-admin-install-sqlserver.properties | 2 +- .../scripts/ranger-kms-install-sqlserver.properties | 2 +- kms/scripts/setup.sh | 8 +++++++- security-admin/scripts/setup.sh | 8 +++++++- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/dev-support/ranger-docker/Dockerfile.ranger b/dev-support/ranger-docker/Dockerfile.ranger index 708b459e6a..9d14a1eb9c 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger +++ b/dev-support/ranger-docker/Dockerfile.ranger @@ -22,6 +22,8 @@ ARG RANGER_DB_TYPE ARG TARGETARCH ARG RANGER_ADMIN_JAVA_VERSION +ENV JAVA_HOME=/usr/lib/jvm/java-1.${RANGER_ADMIN_JAVA_VERSION}.0-openjdk-${TARGETARCH} + RUN if [ "${OS_NAME}" = "UBUNTU" ]; then\ update-java-alternatives --set "$JAVA_HOME";\ fi diff --git a/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties index 2815e1abae..7ac28832c9 100644 --- a/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties +++ b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties @@ -16,7 +16,7 @@ # # This file provides a list of the deployment variables for the Policy Manager Web Application # - +DOCKER_ENV=true PYTHON_COMMAND_INVOKER=python3 RANGER_ADMIN_LOG_DIR=/var/log/ranger RANGER_PID_DIR_PATH=/var/run/ranger diff --git a/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties index 66e17a2725..78e92ebc60 100644 --- a/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties +++ b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties @@ -16,7 +16,7 @@ # # This file provides a list of the deployment variables for the Ranger KMS Web Application # - +DOCKER_ENV=true PYTHON_COMMAND_INVOKER=python3 DB_FLAVOR=POSTGRES SQL_CONNECTOR_JAR=/usr/share/java/postgresql.jar diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh index 9e31f5d347..34e72b7c32 100755 --- a/kms/scripts/setup.sh +++ b/kms/scripts/setup.sh @@ -66,6 +66,7 @@ get_prop_or_default() { } PYTHON_COMMAND_INVOKER=$(get_prop 'PYTHON_COMMAND_INVOKER' $PROPFILE) +DOCKER_ENV=$(get_prop 'DOCKER_ENV' $PROPFILE) DB_FLAVOR=$(get_prop 'DB_FLAVOR' $PROPFILE) SQL_CONNECTOR_JAR=$(get_prop 'SQL_CONNECTOR_JAR' $PROPFILE) db_root_user=$(get_prop 'db_root_user' $PROPFILE) @@ -605,7 +606,12 @@ update_properties() { if [ "${DB_FLAVOR}" == "MSSQL" ] then propertyName=ranger.ks.jpa.jdbc.url - newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" + if [ "${DOCKER_ENV}" == "true" ] + then + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};trustServerCertificate=true;" + else + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" + fi updatePropertyToFilePy $propertyName $newPropertyValue $to_file propertyName=ranger.ks.jpa.jdbc.dialect diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh index 96acb4b39e..fd1997a0f7 100755 --- a/security-admin/scripts/setup.sh +++ b/security-admin/scripts/setup.sh @@ -69,6 +69,7 @@ fi LOGFILE=$(eval echo " $(get_prop 'LOGFILE' $PROPFILE)") PYTHON_COMMAND_INVOKER=$(get_prop 'PYTHON_COMMAND_INVOKER' $PROPFILE) +DOCKER_ENV=$(get_prop 'DOCKER_ENV' $PROPFILE) DB_FLAVOR=$(get_prop 'DB_FLAVOR' $PROPFILE) SQL_CONNECTOR_JAR=$(get_prop 'SQL_CONNECTOR_JAR' $PROPFILE) db_root_user=$(get_prop 'db_root_user' $PROPFILE) @@ -746,7 +747,12 @@ update_properties() { if [ "${DB_FLAVOR}" == "MSSQL" ] then propertyName=ranger.jpa.jdbc.url - newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};trustServerCertificate=true;" + if [ "${DOCKER_ENV}" == "true" ] + then + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};trustServerCertificate=true;" + else + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" + fi updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger propertyName=ranger.jpa.jdbc.dialect From d3668d9da3faf5eb1d940bdbd3795ecc812e48aa Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 23 Jan 2025 16:05:22 -0800 Subject: [PATCH 14/27] Update setup.sh --- security-admin/scripts/setup.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh index fd1997a0f7..83718e93ad 100755 --- a/security-admin/scripts/setup.sh +++ b/security-admin/scripts/setup.sh @@ -749,10 +749,10 @@ update_properties() { propertyName=ranger.jpa.jdbc.url if [ "${DOCKER_ENV}" == "true" ] then - newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};trustServerCertificate=true;" - else - newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" - fi + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};trustServerCertificate=true;" + else + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" + fi updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger propertyName=ranger.jpa.jdbc.dialect From 3a5e2ce17e12c2af7d4def6efe4582be187615ba Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 23 Jan 2025 16:06:24 -0800 Subject: [PATCH 15/27] Update setup.sh --- kms/scripts/setup.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh index 34e72b7c32..ef9ab784c3 100755 --- a/kms/scripts/setup.sh +++ b/kms/scripts/setup.sh @@ -606,12 +606,12 @@ update_properties() { if [ "${DB_FLAVOR}" == "MSSQL" ] then propertyName=ranger.ks.jpa.jdbc.url - if [ "${DOCKER_ENV}" == "true" ] - then - newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};trustServerCertificate=true;" - else - newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" - fi + if [ "${DOCKER_ENV}" == "true" ] + then + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};trustServerCertificate=true;" + else + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" + fi updatePropertyToFilePy $propertyName $newPropertyValue $to_file propertyName=ranger.ks.jpa.jdbc.dialect From adfd6996794d8555aa9a5ff4b2d7f9420f4bb5bb Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 23 Jan 2025 17:23:52 -0800 Subject: [PATCH 16/27] Add sqlserver connector in Dockerfile.ranger-kms --- dev-support/ranger-docker/Dockerfile.ranger-kms | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev-support/ranger-docker/Dockerfile.ranger-kms b/dev-support/ranger-docker/Dockerfile.ranger-kms index 5e70d0da53..be85bbcff8 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-kms +++ b/dev-support/ranger-docker/Dockerfile.ranger-kms @@ -58,6 +58,10 @@ FROM ranger-kms AS ranger_oracle COPY ./downloads/ojdbc8.jar /home/ranger/dist/ RUN mv /home/ranger/dist/ojdbc8.jar /usr/share/java/oracle.jar +FROM ranger-kms AS ranger_sqlserver +COPY ./downloads/mssql-jdbc-12.8.1.jre8.jar /home/ranger/dist/ +RUN mv /home/ranger/dist/mssql-jdbc-12.8.1.jre8.jar /usr/share/java/mssql.jar + FROM ranger_${RANGER_DB_TYPE} ENTRYPOINT [ "/home/ranger/scripts/ranger-kms.sh" ] From 012517a07a0b2ff001247586febb4c26708ac22c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 23 Jan 2025 18:44:38 -0800 Subject: [PATCH 17/27] add hive-site-sqlserver.xml --- .../scripts/hive-site-sqlserver.xml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 dev-support/ranger-docker/scripts/hive-site-sqlserver.xml diff --git a/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml b/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml new file mode 100644 index 0000000000..5bed21c2d8 --- /dev/null +++ b/dev-support/ranger-docker/scripts/hive-site-sqlserver.xml @@ -0,0 +1,50 @@ + + + + + javax.jdo.option.ConnectionURL + jdbc:sqlserver://ranger-db/hive + + + javax.jdo.option.ConnectionDriverName + com.microsoft.sqlserver.jdbc.SQLServerDriver + + + javax.jdo.option.ConnectionUserName + hive + + + javax.jdo.option.ConnectionPassword + rangerR0cks! + + + hive.server2.enable.doAs + false + + + hive.zookeeper.quorum + ranger-zk.example.com + + + hive.zookeeper.client.port + 2181 + + From 5d5368dfb2345369440550a7b3a95932ac33d0bb Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 23 Jan 2025 20:19:36 -0800 Subject: [PATCH 18/27] Container check for sqlserver --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 16f5d8fc4d..1575d17d72 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -169,7 +169,7 @@ jobs: - name: Check status of containers and remove them run: | sleep 60 - containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode); + containers=(ranger ranger-zk ranger-solr ranger-sqlserver ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode); flag=true; for container in "${containers[@]}"; do if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then From 68f5a3f55f263492be7f643593dbc98fadcf42d3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 24 Jan 2025 10:36:37 -0800 Subject: [PATCH 19/27] Update CONNECTION_STRING_ADDITIONAL_PARAMS in setup.sh --- security-admin/scripts/setup.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh index 83718e93ad..3bfd0d0431 100755 --- a/security-admin/scripts/setup.sh +++ b/security-admin/scripts/setup.sh @@ -69,9 +69,10 @@ fi LOGFILE=$(eval echo " $(get_prop 'LOGFILE' $PROPFILE)") PYTHON_COMMAND_INVOKER=$(get_prop 'PYTHON_COMMAND_INVOKER' $PROPFILE) -DOCKER_ENV=$(get_prop 'DOCKER_ENV' $PROPFILE) + DB_FLAVOR=$(get_prop 'DB_FLAVOR' $PROPFILE) SQL_CONNECTOR_JAR=$(get_prop 'SQL_CONNECTOR_JAR' $PROPFILE) +CONNECTION_STRING_ADDITIONAL_PARAMS=$(get_prop 'CONNECTION_STRING_ADDITIONAL_PARAMS' $PROPFILE) db_root_user=$(get_prop 'db_root_user' $PROPFILE) db_root_password=$(get_prop 'db_root_password' $PROPFILE) db_host=$(get_prop 'db_host' $PROPFILE) @@ -747,9 +748,9 @@ update_properties() { if [ "${DB_FLAVOR}" == "MSSQL" ] then propertyName=ranger.jpa.jdbc.url - if [ "${DOCKER_ENV}" == "true" ] + if [ "${CONNECTION_STRING_ADDITIONAL_PARAMS}" != "" ] then - newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};trustServerCertificate=true;" + newPropertyValue=${CONNECTION_STRING_ADDITIONAL_PARAMS} else newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" fi From 5c812f717929a72e9c33b1339143011d5e24487c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 24 Jan 2025 10:39:03 -0800 Subject: [PATCH 20/27] Update CONNECTION_STRING_ADDITIONAL_PARAMS in setup.sh --- kms/scripts/setup.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh index ef9ab784c3..5f9861cb80 100755 --- a/kms/scripts/setup.sh +++ b/kms/scripts/setup.sh @@ -66,9 +66,10 @@ get_prop_or_default() { } PYTHON_COMMAND_INVOKER=$(get_prop 'PYTHON_COMMAND_INVOKER' $PROPFILE) -DOCKER_ENV=$(get_prop 'DOCKER_ENV' $PROPFILE) + DB_FLAVOR=$(get_prop 'DB_FLAVOR' $PROPFILE) SQL_CONNECTOR_JAR=$(get_prop 'SQL_CONNECTOR_JAR' $PROPFILE) +CONNECTION_STRING_ADDITIONAL_PARAMS=$(get_prop 'CONNECTION_STRING_ADDITIONAL_PARAMS' $PROPFILE) db_root_user=$(get_prop 'db_root_user' $PROPFILE) db_root_password=$(get_prop 'db_root_password' $PROPFILE) db_host=$(get_prop 'db_host' $PROPFILE) @@ -606,9 +607,9 @@ update_properties() { if [ "${DB_FLAVOR}" == "MSSQL" ] then propertyName=ranger.ks.jpa.jdbc.url - if [ "${DOCKER_ENV}" == "true" ] + if [ "${CONNECTION_STRING_ADDITIONAL_PARAMS}" != "" ] then - newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};trustServerCertificate=true;" + newPropertyValue=${CONNECTION_STRING_ADDITIONAL_PARAMS} else newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" fi From ee10741c4ab6768b96ab2c507657b56ea5931016 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 24 Jan 2025 10:41:05 -0800 Subject: [PATCH 21/27] Update setup.sh --- security-admin/scripts/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh index 3bfd0d0431..1fef99ee45 100755 --- a/security-admin/scripts/setup.sh +++ b/security-admin/scripts/setup.sh @@ -750,7 +750,7 @@ update_properties() { propertyName=ranger.jpa.jdbc.url if [ "${CONNECTION_STRING_ADDITIONAL_PARAMS}" != "" ] then - newPropertyValue=${CONNECTION_STRING_ADDITIONAL_PARAMS} + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};${CONNECTION_STRING_ADDITIONAL_PARAMS}" else newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" fi From 565e1d59d3c4de3f3e2b512b5727a057d429d42c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 24 Jan 2025 10:41:57 -0800 Subject: [PATCH 22/27] Update ranger-admin-install-sqlserver.properties --- .../scripts/ranger-admin-install-sqlserver.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties index 7ac28832c9..b69e22d1e4 100644 --- a/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties +++ b/dev-support/ranger-docker/scripts/ranger-admin-install-sqlserver.properties @@ -16,12 +16,13 @@ # # This file provides a list of the deployment variables for the Policy Manager Web Application # -DOCKER_ENV=true + PYTHON_COMMAND_INVOKER=python3 RANGER_ADMIN_LOG_DIR=/var/log/ranger RANGER_PID_DIR_PATH=/var/run/ranger DB_FLAVOR=MSSQL SQL_CONNECTOR_JAR=/usr/share/java/mssql.jar +CONNECTION_STRING_ADDITIONAL_PARAMS="trustServerCertificate=true;" RANGER_ADMIN_LOGBACK_CONF_FILE=/opt/ranger/admin/ews/webapp/WEB-INF/classes/conf/logback.xml db_root_user=sa From b3b84e5ca5f98f8e2965ea5ee239efa383434611 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 24 Jan 2025 10:42:26 -0800 Subject: [PATCH 23/27] Update ranger-kms-install-sqlserver.properties --- .../scripts/ranger-kms-install-sqlserver.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties index 78e92ebc60..a4b6fdbfe2 100644 --- a/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties +++ b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties @@ -16,10 +16,11 @@ # # This file provides a list of the deployment variables for the Ranger KMS Web Application # -DOCKER_ENV=true + PYTHON_COMMAND_INVOKER=python3 DB_FLAVOR=POSTGRES SQL_CONNECTOR_JAR=/usr/share/java/postgresql.jar +CONNECTION_STRING_ADDITIONAL_PARAMS="trustServerCertificate=true;" db_root_user=postgres db_root_password=rangerR0cks! From c86fc5edabd7c56869c418739f80f35ceb0826bf Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 24 Jan 2025 10:44:39 -0800 Subject: [PATCH 24/27] Update setup.sh --- kms/scripts/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh index 5f9861cb80..b547b1751b 100755 --- a/kms/scripts/setup.sh +++ b/kms/scripts/setup.sh @@ -609,7 +609,7 @@ update_properties() { propertyName=ranger.ks.jpa.jdbc.url if [ "${CONNECTION_STRING_ADDITIONAL_PARAMS}" != "" ] then - newPropertyValue=${CONNECTION_STRING_ADDITIONAL_PARAMS} + newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name};${CONNECTION_STRING_ADDITIONAL_PARAMS}" else newPropertyValue="jdbc:sqlserver://${DB_HOST};databaseName=${db_name}" fi From cbd6fab3907cd2ae1b19808f1c009cfb7e7ceaf5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 24 Jan 2025 11:15:53 -0800 Subject: [PATCH 25/27] Debug ranger-kms logs --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 1575d17d72..3f88d12400 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -165,6 +165,7 @@ jobs: sleep 30 docker logs ranger-sqlserver docker logs ranger + docker logs ranger-kms - name: Check status of containers and remove them run: | From 44c0e945f1a7cc1aaf738db54022f5763be46e22 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 24 Jan 2025 12:40:28 -0800 Subject: [PATCH 26/27] Fix ranger-kms-install-sqlserver.properties --- .../scripts/ranger-kms-install-sqlserver.properties | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties index a4b6fdbfe2..04c96989be 100644 --- a/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties +++ b/dev-support/ranger-docker/scripts/ranger-kms-install-sqlserver.properties @@ -18,11 +18,11 @@ # PYTHON_COMMAND_INVOKER=python3 -DB_FLAVOR=POSTGRES -SQL_CONNECTOR_JAR=/usr/share/java/postgresql.jar +DB_FLAVOR=MSSQL +SQL_CONNECTOR_JAR=/usr/share/java/mssql.jar CONNECTION_STRING_ADDITIONAL_PARAMS="trustServerCertificate=true;" -db_root_user=postgres +db_root_user=sa db_root_password=rangerR0cks! db_host=ranger-db @@ -33,8 +33,8 @@ db_password=rangerR0cks! # Following variables are referenced in db_setup.py. Do not remove these mysql_core_file=db/mysql/kms_core_db.sql postgres_core_file=db/postgres/kms_core_db_postgres.sql +oracle_core_file=db/oracle/kms_core_db_oracle.sql sqlserver_core_file=db/sqlserver/kms_core_db_sqlserver.sql -oracle_core_file= sqlanywhere_core_file= # For over-riding the jdbc url From 29f92b2a5e6b4a5964b99e72397a752936007035 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 24 Jan 2025 13:09:24 -0800 Subject: [PATCH 27/27] Undo changes for sqlserver and switch it back to postgres --- .github/workflows/maven.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 3f88d12400..b556905e58 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -128,7 +128,7 @@ jobs: docker compose -f docker-compose.ranger-base.yml build export DOCKER_BUILDKIT=1 export COMPOSE_DOCKER_CLI_BUILD=1 - export RANGER_DB_TYPE=sqlserver + export RANGER_DB_TYPE=postgres docker compose \ -f docker-compose.ranger-${RANGER_DB_TYPE}.yml \ -f docker-compose.ranger.yml \ @@ -146,7 +146,7 @@ jobs: run: | cd dev-support/ranger-docker ./scripts/ozone-plugin-docker-setup.sh - export RANGER_DB_TYPE=sqlserver + export RANGER_DB_TYPE=postgres docker compose \ -f docker-compose.ranger-${RANGER_DB_TYPE}.yml \ -f docker-compose.ranger.yml \ @@ -160,17 +160,10 @@ jobs: -f docker-compose.ranger-knox.yml \ -f docker-compose.ranger-ozone.yml up -d - - name: View sql server logs - run: | - sleep 30 - docker logs ranger-sqlserver - docker logs ranger - docker logs ranger-kms - - name: Check status of containers and remove them run: | sleep 60 - containers=(ranger ranger-zk ranger-solr ranger-sqlserver ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode); + containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox ozone-om ozone-scm ozone-datanode); flag=true; for container in "${containers[@]}"; do if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then