Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Currency: Updated build_script and build_info.json for pyarrow v15.0.1 #5347

Open
wants to merge 2 commits into
base: python-ecosystem
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions p/pyarrow/build_info.json
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
{
"maintainer": "vinodk99",
"maintainer": "kiranNukal",
"package_name": "pyarrow",
"github_url": "https://github.com/apache/arrow.git",
"version": "apache-arrow-11.0.0",
"version": "apache-arrow-15.0.1",
"wheel_build" : true,
"default_branch": "main",
"package_dir": "p/pyarrow",
"build_script": "pyarrow_ubi_9.3.sh",
"build_script": "pyarrow_15.0.1_ubi_9.3.sh",
"docker_build": false,
"validate_build_script": true,
"use_non_root_user": false,
"*": {
"build_script": "pyarrow_ubi_9.3.sh"
"build_script": "pyarrow_15.0.1_ubi_9.3.sh"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You should be having a new block for older pyarrow versions, instead writing it off.

"*"{
   "build_script": "pyarrow_15.0.1_ubi_9.3.sh"
},
"*11.0.0"{
   "build_script": "pyarrow_ubi_9.3.sh"
},

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

addressed.

}
}
136 changes: 136 additions & 0 deletions p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
#!/bin/bash -e
# -----------------------------------------------------------------------------
#
# Package : pyarrow
# Version : 15.0.1
# Source repo : https://github.com/apache/arrow.git
# Tested on : UBI:9.3
# Language : Python
# Travis-Check : True
# Script License: Apache License, Version 2 or later
# Maintainer : Sai Kiran Nukala <[email protected]>
#
# Disclaimer: This script has been tested in root mode on the given
# platform using the mentioned version of the package.
# It may not work as expected with newer versions of the
# package and/or distribution. In such a case, please
# contact the "Maintainer" of this script.
#
# -----------------------------------------------------------------------------

# Variables
PACKAGE_NAME=pyarrow
PACKAGE_VERSION=${1:-apache-arrow-15.0.1}
PACKAGE_URL=https://github.com/apache/arrow.git
PACKAGE_DIR=arrow

echo "Installing dependencies..."
yum install -y git wget gcc gcc-c++ python python3-devel python3 python3-pip openssl-devel cmake
echo "Dependencies installed."

mkdir dist

# Set environment variables
export CXX=g++
export CC=gcc
export ARROW_HOME=$(pwd)/dist
export LD_LIBRARY_PATH=$ARROW_HOME/lib:$LD_LIBRARY_PATH
export CMAKE_PREFIX_PATH=$ARROW_HOME:$CMAKE_PREFIX_PATH

echo "Environment variables set:"
echo "CXX=$CXX"
echo "CC=$CC"
echo "ARROW_HOME=$ARROW_HOME"
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH"

echo "Cloning the repository..."
git clone $PACKAGE_URL
cd $PACKAGE_DIR
git checkout $PACKAGE_VERSION
git submodule update --init
echo "Repository cloned and checked out to version $PACKAGE_VERSION."

echo "Setting test data paths..."
export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data"
export ARROW_TEST_DATA="${PWD}/testing/data"
echo "PARQUET_TEST_DATA=$PARQUET_TEST_DATA"
echo "ARROW_TEST_DATA=$ARROW_TEST_DATA"

echo "Applying fixes for nogil placement and rvalue issues..."
sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/error.pxi
sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/includes/libarrow.pxd
sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/lib.pxd
sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/includes/libarrow_fs.pxd
sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/types.pxi
sed -i -E 's/\&\&/\&/g' python/pyarrow/error.pxi
sed -i -E 's/\&\&/\&/g' python/pyarrow/includes/libarrow.pxd
sed -i -E 's/\&\&/\&/g' python/pyarrow/lib.pxd
sed -i -E 's/\&\&/\&/g' python/pyarrow/includes/libarrow_fs.pxd
sed -i '/cdef object alloc_c_schema(ArrowSchema\*\* c_schema)/s/ noexcept//' python/pyarrow/types.pxi
shubham-dayma-ibm marked this conversation as resolved.
Show resolved Hide resolved
sed -i '/cdef object alloc_c_array(ArrowArray\*\* c_array)/s/ noexcept//' python/pyarrow/types.pxi
sed -i '/cdef object alloc_c_stream(ArrowArrayStream\*\* c_stream)/s/ noexcept//' python/pyarrow/types.pxi
echo "Fixes applied."

echo "Installing Python dependencies..."
pip install -r python/requirements-build.txt
pip install cython wheel numpy==1.21.2
echo "Python dependencies installed."

echo "Preparing for build..."
mkdir cpp/build
cd cpp/build
echo "Current directory: $(pwd)"

cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-DCMAKE_INSTALL_LIBDIR=lib \
-DCMAKE_BUILD_TYPE=Release \
-DARROW_BUILD_TESTS=ON \
-DARROW_COMPUTE=ON \
-DARROW_CSV=ON \
-DARROW_DATASET=ON \
-DARROW_FILESYSTEM=ON \
-DARROW_HDFS=ON \
-DARROW_JSON=ON \
-DARROW_PARQUET=ON \
-DARROW_WITH_BROTLI=ON \
-DARROW_WITH_BZ2=ON \
-DARROW_WITH_LZ4=ON \
-DARROW_WITH_SNAPPY=ON \
-DARROW_WITH_ZLIB=ON \
-DARROW_WITH_ZSTD=ON \
-DPARQUET_REQUIRE_ENCRYPTION=ON \
..

echo "Starting build..."
make -j$(nproc)
make install
echo "Build completed successfully."

cd ../..
cd python
echo "Current directory: $(pwd)"

export PYARROW_WITH_PARQUET=1
export PYARROW_WITH_DATASET=1
export PYARROW_PARALLEL=4
export PYARROW_BUILD_TYPE="release"

echo "PYARROW_WITH_PARQUET=$PYARROW_WITH_PARQUET"
echo "PYARROW_WITH_DATASET=$PYARROW_WITH_DATASET"
echo "PYARROW_PARALLEL=$PYARROW_PARALLEL"
echo "PYARROW_BUILD_TYPE=$PYARROW_BUILD_TYPE"

if ! python3 setup.py bdist_wheel ; then
shubham-dayma-ibm marked this conversation as resolved.
Show resolved Hide resolved
echo "------------------$PACKAGE_NAME:wheel_built_fails---------------------"
echo "$PACKAGE_VERSION $PACKAGE_NAME"
echo "$PACKAGE_NAME | $PACKAGE_VERSION | $(uname -o) | GitHub | Fail | wheel_built_fails"
exit 1
else
echo "------------------$PACKAGE_NAME:wheel_built_success-------------------------"
echo "$PACKAGE_VERSION $PACKAGE_NAME"
echo "$PACKAGE_NAME | $PACKAGE_VERSION | $(uname -o) | GitHub | Pass | Wheel_built_success"
exit 0
fi

# Skipping the tests as most of the tests are parity with x86 also