From 91b139e0f98f671e2522fbbc70ab734c956c3c82 Mon Sep 17 00:00:00 2001 From: kiranNukal Date: Thu, 23 Jan 2025 10:32:15 +0000 Subject: [PATCH 1/4] Currency: Updated build_script and build_info.json for pyarrow v15.0.1 --- p/pyarrow/build_info.json | 8 +- p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh | 136 ++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh diff --git a/p/pyarrow/build_info.json b/p/pyarrow/build_info.json index 64e44b8eed..a69a98bfad 100644 --- a/p/pyarrow/build_info.json +++ b/p/pyarrow/build_info.json @@ -1,16 +1,16 @@ { - "maintainer": "vinodk99", + "maintainer": "kiranNukal", "package_name": "pyarrow", "github_url": "https://github.com/apache/arrow.git", - "version": "apache-arrow-11.0.0", + "version": "apache-arrow-15.0.1", "wheel_build" : true, "default_branch": "main", "package_dir": "p/pyarrow", - "build_script": "pyarrow_ubi_9.3.sh", + "build_script": "pyarrow_15.0.1_ubi_9.3.sh", "docker_build": false, "validate_build_script": true, "use_non_root_user": false, "*": { - "build_script": "pyarrow_ubi_9.3.sh" + "build_script": "pyarrow_15.0.1_ubi_9.3.sh" } } diff --git a/p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh b/p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh new file mode 100644 index 0000000000..a531c4d9e1 --- /dev/null +++ b/p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh @@ -0,0 +1,136 @@ +#!/bin/bash -e +# ----------------------------------------------------------------------------- +# +# Package : pyarrow +# Version : 15.0.1 +# Source repo : https://github.com/apache/arrow.git +# Tested on : UBI:9.3 +# Language : Python +# Travis-Check : True +# Script License: Apache License, Version 2 or later +# Maintainer : Sai Kiran Nukala +# +# Disclaimer: This script has been tested in root mode on the given +# platform using the mentioned version of the package. +# It may not work as expected with newer versions of the +# package and/or distribution. In such a case, please +# contact the "Maintainer" of this script. +# +# ----------------------------------------------------------------------------- + +# Variables +PACKAGE_NAME=pyarrow +PACKAGE_VERSION=${1:-apache-arrow-15.0.1} +PACKAGE_URL=https://github.com/apache/arrow.git +PACKAGE_DIR=arrow + +echo "Installing dependencies..." +yum install -y git wget gcc gcc-c++ python python3-devel python3 python3-pip openssl-devel cmake +echo "Dependencies installed." + +mkdir dist + +# Set environment variables +export CXX=g++ +export CC=gcc +export ARROW_HOME=$(pwd)/dist +export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=$ARROW_HOME:$CMAKE_PREFIX_PATH + +echo "Environment variables set:" +echo "CXX=$CXX" +echo "CC=$CC" +echo "ARROW_HOME=$ARROW_HOME" +echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" +echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" + +echo "Cloning the repository..." +git clone $PACKAGE_URL +cd $PACKAGE_DIR +git checkout $PACKAGE_VERSION +git submodule update --init +echo "Repository cloned and checked out to version $PACKAGE_VERSION." + +echo "Setting test data paths..." +export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" +export ARROW_TEST_DATA="${PWD}/testing/data" +echo "PARQUET_TEST_DATA=$PARQUET_TEST_DATA" +echo "ARROW_TEST_DATA=$ARROW_TEST_DATA" + +echo "Applying fixes for nogil placement and rvalue issues..." +sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/error.pxi +sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/includes/libarrow.pxd +sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/lib.pxd +sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/includes/libarrow_fs.pxd +sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/types.pxi +sed -i -E 's/\&\&/\&/g' python/pyarrow/error.pxi +sed -i -E 's/\&\&/\&/g' python/pyarrow/includes/libarrow.pxd +sed -i -E 's/\&\&/\&/g' python/pyarrow/lib.pxd +sed -i -E 's/\&\&/\&/g' python/pyarrow/includes/libarrow_fs.pxd +sed -i '/cdef object alloc_c_schema(ArrowSchema\*\* c_schema)/s/ noexcept//' python/pyarrow/types.pxi +sed -i '/cdef object alloc_c_array(ArrowArray\*\* c_array)/s/ noexcept//' python/pyarrow/types.pxi +sed -i '/cdef object alloc_c_stream(ArrowArrayStream\*\* c_stream)/s/ noexcept//' python/pyarrow/types.pxi +echo "Fixes applied." + +echo "Installing Python dependencies..." +pip install -r python/requirements-build.txt +pip install cython wheel numpy==1.21.2 +echo "Python dependencies installed." + +echo "Preparing for build..." +mkdir cpp/build +cd cpp/build +echo "Current directory: $(pwd)" + +cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_BUILD_TYPE=Release \ + -DARROW_BUILD_TESTS=ON \ + -DARROW_COMPUTE=ON \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ + -DARROW_FILESYSTEM=ON \ + -DARROW_HDFS=ON \ + -DARROW_JSON=ON \ + -DARROW_PARQUET=ON \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DPARQUET_REQUIRE_ENCRYPTION=ON \ + .. + +echo "Starting build..." +make -j$(nproc) +make install +echo "Build completed successfully." + +cd ../.. +cd python +echo "Current directory: $(pwd)" + +export PYARROW_WITH_PARQUET=1 +export PYARROW_WITH_DATASET=1 +export PYARROW_PARALLEL=4 +export PYARROW_BUILD_TYPE="release" + +echo "PYARROW_WITH_PARQUET=$PYARROW_WITH_PARQUET" +echo "PYARROW_WITH_DATASET=$PYARROW_WITH_DATASET" +echo "PYARROW_PARALLEL=$PYARROW_PARALLEL" +echo "PYARROW_BUILD_TYPE=$PYARROW_BUILD_TYPE" + +if ! python3 setup.py bdist_wheel ; then + echo "------------------$PACKAGE_NAME:wheel_built_fails---------------------" + echo "$PACKAGE_VERSION $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_VERSION | $(uname -o) | GitHub | Fail | wheel_built_fails" + exit 1 +else + echo "------------------$PACKAGE_NAME:wheel_built_success-------------------------" + echo "$PACKAGE_VERSION $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_VERSION | $(uname -o) | GitHub | Pass | Wheel_built_success" + exit 0 +fi + +# Skipping the tests as most of the tests are parity with x86 also \ No newline at end of file From 896d80632996e7636ad94c70e2376fbfe346f9f3 Mon Sep 17 00:00:00 2001 From: kiranNukal Date: Wed, 12 Feb 2025 11:19:28 +0530 Subject: [PATCH 2/4] Update pyarrow_15.0.1_ubi_9.3.sh --- p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh | 70 +++++++++++------------------ 1 file changed, 27 insertions(+), 43 deletions(-) diff --git a/p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh b/p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh index a531c4d9e1..712c774381 100644 --- a/p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh +++ b/p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh @@ -8,7 +8,7 @@ # Language : Python # Travis-Check : True # Script License: Apache License, Version 2 or later -# Maintainer : Sai Kiran Nukala +# Maintainer : Sai Kiran Nukala # # Disclaimer: This script has been tested in root mode on the given # platform using the mentioned version of the package. @@ -22,31 +22,25 @@ PACKAGE_NAME=pyarrow PACKAGE_VERSION=${1:-apache-arrow-15.0.1} PACKAGE_URL=https://github.com/apache/arrow.git -PACKAGE_DIR=arrow +PACKAGE_DIR=./arrow/python +CURRENT_DIR="${PWD}" + +# Install necessary dependencies +yum install -y git wget gcc gcc-c++ python python3-devel python3 python3-pip openssl-devel cmake unzip -echo "Installing dependencies..." -yum install -y git wget gcc gcc-c++ python python3-devel python3 python3-pip openssl-devel cmake echo "Dependencies installed." mkdir dist - -# Set environment variables export CXX=g++ export CC=gcc export ARROW_HOME=$(pwd)/dist -export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH +export PYARROW_BUNDLE_ARROW_CPP=1 +export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH export CMAKE_PREFIX_PATH=$ARROW_HOME:$CMAKE_PREFIX_PATH -echo "Environment variables set:" -echo "CXX=$CXX" -echo "CC=$CC" -echo "ARROW_HOME=$ARROW_HOME" -echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" -echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" - echo "Cloning the repository..." git clone $PACKAGE_URL -cd $PACKAGE_DIR +cd arrow git checkout $PACKAGE_VERSION git submodule update --init echo "Repository cloned and checked out to version $PACKAGE_VERSION." @@ -54,15 +48,12 @@ echo "Repository cloned and checked out to version $PACKAGE_VERSION." echo "Setting test data paths..." export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" export ARROW_TEST_DATA="${PWD}/testing/data" -echo "PARQUET_TEST_DATA=$PARQUET_TEST_DATA" -echo "ARROW_TEST_DATA=$ARROW_TEST_DATA" echo "Applying fixes for nogil placement and rvalue issues..." sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/error.pxi sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/includes/libarrow.pxd sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/lib.pxd sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/includes/libarrow_fs.pxd -sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/types.pxi sed -i -E 's/\&\&/\&/g' python/pyarrow/error.pxi sed -i -E 's/\&\&/\&/g' python/pyarrow/includes/libarrow.pxd sed -i -E 's/\&\&/\&/g' python/pyarrow/lib.pxd @@ -72,18 +63,16 @@ sed -i '/cdef object alloc_c_array(ArrowArray\*\* c_array)/s/ noexcept//' python sed -i '/cdef object alloc_c_stream(ArrowArrayStream\*\* c_stream)/s/ noexcept//' python/pyarrow/types.pxi echo "Fixes applied." -echo "Installing Python dependencies..." pip install -r python/requirements-build.txt pip install cython wheel numpy==1.21.2 -echo "Python dependencies installed." echo "Preparing for build..." + mkdir cpp/build cd cpp/build -echo "Current directory: $(pwd)" cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_INSTALL_LIBDIR=lib64 \ -DCMAKE_BUILD_TYPE=Release \ -DARROW_BUILD_TESTS=ON \ -DARROW_COMPUTE=ON \ @@ -100,37 +89,32 @@ cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ -DPARQUET_REQUIRE_ENCRYPTION=ON \ + -DBUILD_SHARED_LIBS=ON \ .. - -echo "Starting build..." make -j$(nproc) make install -echo "Build completed successfully." - -cd ../.. -cd python -echo "Current directory: $(pwd)" +cd ../../.. +cd $PACKAGE_DIR export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_DATASET=1 export PYARROW_PARALLEL=4 export PYARROW_BUILD_TYPE="release" +export PYARROW_BUNDLE_ARROW_CPP_HEADERS=1 -echo "PYARROW_WITH_PARQUET=$PYARROW_WITH_PARQUET" -echo "PYARROW_WITH_DATASET=$PYARROW_WITH_DATASET" -echo "PYARROW_PARALLEL=$PYARROW_PARALLEL" -echo "PYARROW_BUILD_TYPE=$PYARROW_BUILD_TYPE" -if ! python3 setup.py bdist_wheel ; then - echo "------------------$PACKAGE_NAME:wheel_built_fails---------------------" - echo "$PACKAGE_VERSION $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_VERSION | $(uname -o) | GitHub | Fail | wheel_built_fails" - exit 1 +if ! python3 setup.py bdist_wheel --dist-dir="$CURRENT_DIR/" ; then + echo "------------------$PACKAGE_NAME:wheel_built_fails---------------------" + echo "$PACKAGE_VERSION $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_VERSION | $OS_NAME | GitHub | Fail | wheel_built_fails" + exit 1 else - echo "------------------$PACKAGE_NAME:wheel_built_success-------------------------" - echo "$PACKAGE_VERSION $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_VERSION | $(uname -o) | GitHub | Pass | Wheel_built_success" - exit 0 + echo "------------------$PACKAGE_NAME:wheel_built_success-------------------------" + echo "$PACKAGE_VERSION $PACKAGE_NAME" + echo "$PACKAGE_NAME | $PACKAGE_VERSION | $OS_NAME | GitHub | Pass | Wheel_built_success" + echo "Checking contents of the generated wheel..." + unzip -l "$CURRENT_DIR/pyarrow-*.whl" | grep libarrow + exit 0 fi -# Skipping the tests as most of the tests are parity with x86 also \ No newline at end of file +# Skipping the tests as most of the tests are parity with x86 also From 6b9d8ee43ecefe83a79c7a166993f212337c7031 Mon Sep 17 00:00:00 2001 From: kiranNukal Date: Fri, 14 Feb 2025 14:19:06 +0530 Subject: [PATCH 3/4] Update build_info.json --- p/pyarrow/build_info.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/p/pyarrow/build_info.json b/p/pyarrow/build_info.json index a69a98bfad..1987ace535 100644 --- a/p/pyarrow/build_info.json +++ b/p/pyarrow/build_info.json @@ -10,6 +10,9 @@ "docker_build": false, "validate_build_script": true, "use_non_root_user": false, + "*11.0.0"{ + "build_script": "pyarrow_ubi_9.3.sh" + }, "*": { "build_script": "pyarrow_15.0.1_ubi_9.3.sh" } From af44c4d4c29d7d0c0d4a45102ef03ad97b7dfac1 Mon Sep 17 00:00:00 2001 From: kiranNukal Date: Fri, 14 Feb 2025 14:19:53 +0530 Subject: [PATCH 4/4] Update pyarrow_15.0.1_ubi_9.3.sh --- p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh b/p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh index 712c774381..e6c649afc1 100644 --- a/p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh +++ b/p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh @@ -26,7 +26,7 @@ PACKAGE_DIR=./arrow/python CURRENT_DIR="${PWD}" # Install necessary dependencies -yum install -y git wget gcc gcc-c++ python python3-devel python3 python3-pip openssl-devel cmake unzip +yum install -y git wget gcc gcc-c++ python python3-devel python3 python3-pip openssl-devel cmake echo "Dependencies installed." @@ -112,8 +112,6 @@ else echo "------------------$PACKAGE_NAME:wheel_built_success-------------------------" echo "$PACKAGE_VERSION $PACKAGE_NAME" echo "$PACKAGE_NAME | $PACKAGE_VERSION | $OS_NAME | GitHub | Pass | Wheel_built_success" - echo "Checking contents of the generated wheel..." - unzip -l "$CURRENT_DIR/pyarrow-*.whl" | grep libarrow exit 0 fi