Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Currency: Updated build_script and build_info.json for pyarrow v15.0.1 #5347

Open
wants to merge 2 commits into
base: python-ecosystem
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions p/pyarrow/build_info.json
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
{
"maintainer": "vinodk99",
"maintainer": "kiranNukal",
"package_name": "pyarrow",
"github_url": "https://github.com/apache/arrow.git",
"version": "apache-arrow-11.0.0",
"version": "apache-arrow-15.0.1",
"wheel_build" : true,
"default_branch": "main",
"package_dir": "p/pyarrow",
"build_script": "pyarrow_ubi_9.3.sh",
"build_script": "pyarrow_15.0.1_ubi_9.3.sh",
"docker_build": false,
"validate_build_script": true,
"use_non_root_user": false,
"*": {
"build_script": "pyarrow_ubi_9.3.sh"
"build_script": "pyarrow_15.0.1_ubi_9.3.sh"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You should be having a new block for older pyarrow versions, instead writing it off.

"*"{
   "build_script": "pyarrow_15.0.1_ubi_9.3.sh"
},
"*11.0.0"{
   "build_script": "pyarrow_ubi_9.3.sh"
},

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

addressed.

}
}
120 changes: 120 additions & 0 deletions p/pyarrow/pyarrow_15.0.1_ubi_9.3.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
#!/bin/bash -e
# -----------------------------------------------------------------------------
#
# Package : pyarrow
# Version : 15.0.1
# Source repo : https://github.com/apache/arrow.git
# Tested on : UBI:9.3
# Language : Python
# Travis-Check : True
# Script License: Apache License, Version 2 or later
# Maintainer : Sai Kiran Nukala <[email protected]>
#
# Disclaimer: This script has been tested in root mode on the given
# platform using the mentioned version of the package.
# It may not work as expected with newer versions of the
# package and/or distribution. In such a case, please
# contact the "Maintainer" of this script.
#
# -----------------------------------------------------------------------------

# Variables
PACKAGE_NAME=pyarrow
PACKAGE_VERSION=${1:-apache-arrow-15.0.1}
PACKAGE_URL=https://github.com/apache/arrow.git
PACKAGE_DIR=./arrow/python
CURRENT_DIR="${PWD}"

# Install necessary dependencies
yum install -y git wget gcc gcc-c++ python python3-devel python3 python3-pip openssl-devel cmake unzip

echo "Dependencies installed."

mkdir dist
export CXX=g++
export CC=gcc
export ARROW_HOME=$(pwd)/dist
export PYARROW_BUNDLE_ARROW_CPP=1
export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH
export CMAKE_PREFIX_PATH=$ARROW_HOME:$CMAKE_PREFIX_PATH

echo "Cloning the repository..."
git clone $PACKAGE_URL
cd arrow
git checkout $PACKAGE_VERSION
git submodule update --init
echo "Repository cloned and checked out to version $PACKAGE_VERSION."

echo "Setting test data paths..."
export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data"
export ARROW_TEST_DATA="${PWD}/testing/data"

echo "Applying fixes for nogil placement and rvalue issues..."
sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/error.pxi
sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/includes/libarrow.pxd
sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/lib.pxd
sed -i -E 's/(nogil)(.*)(except[^:]*)/\2\3 \1/' python/pyarrow/includes/libarrow_fs.pxd
sed -i -E 's/\&\&/\&/g' python/pyarrow/error.pxi
sed -i -E 's/\&\&/\&/g' python/pyarrow/includes/libarrow.pxd
sed -i -E 's/\&\&/\&/g' python/pyarrow/lib.pxd
sed -i -E 's/\&\&/\&/g' python/pyarrow/includes/libarrow_fs.pxd
sed -i '/cdef object alloc_c_schema(ArrowSchema\*\* c_schema)/s/ noexcept//' python/pyarrow/types.pxi
shubham-dayma-ibm marked this conversation as resolved.
Show resolved Hide resolved
sed -i '/cdef object alloc_c_array(ArrowArray\*\* c_array)/s/ noexcept//' python/pyarrow/types.pxi
sed -i '/cdef object alloc_c_stream(ArrowArrayStream\*\* c_stream)/s/ noexcept//' python/pyarrow/types.pxi
echo "Fixes applied."

pip install -r python/requirements-build.txt
pip install cython wheel numpy==1.21.2

echo "Preparing for build..."

mkdir cpp/build
cd cpp/build

cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-DCMAKE_INSTALL_LIBDIR=lib64 \
-DCMAKE_BUILD_TYPE=Release \
-DARROW_BUILD_TESTS=ON \
-DARROW_COMPUTE=ON \
-DARROW_CSV=ON \
-DARROW_DATASET=ON \
-DARROW_FILESYSTEM=ON \
-DARROW_HDFS=ON \
-DARROW_JSON=ON \
-DARROW_PARQUET=ON \
-DARROW_WITH_BROTLI=ON \
-DARROW_WITH_BZ2=ON \
-DARROW_WITH_LZ4=ON \
-DARROW_WITH_SNAPPY=ON \
-DARROW_WITH_ZLIB=ON \
-DARROW_WITH_ZSTD=ON \
-DPARQUET_REQUIRE_ENCRYPTION=ON \
-DBUILD_SHARED_LIBS=ON \
..
make -j$(nproc)
make install
cd ../../..

cd $PACKAGE_DIR
export PYARROW_WITH_PARQUET=1
export PYARROW_WITH_DATASET=1
export PYARROW_PARALLEL=4
export PYARROW_BUILD_TYPE="release"
export PYARROW_BUNDLE_ARROW_CPP_HEADERS=1


if ! python3 setup.py bdist_wheel --dist-dir="$CURRENT_DIR/" ; then
echo "------------------$PACKAGE_NAME:wheel_built_fails---------------------"
echo "$PACKAGE_VERSION $PACKAGE_NAME"
echo "$PACKAGE_NAME | $PACKAGE_VERSION | $OS_NAME | GitHub | Fail | wheel_built_fails"
exit 1
else
echo "------------------$PACKAGE_NAME:wheel_built_success-------------------------"
echo "$PACKAGE_VERSION $PACKAGE_NAME"
echo "$PACKAGE_NAME | $PACKAGE_VERSION | $OS_NAME | GitHub | Pass | Wheel_built_success"
echo "Checking contents of the generated wheel..."
unzip -l "$CURRENT_DIR/pyarrow-*.whl" | grep libarrow
exit 0
fi

# Skipping the tests as most of the tests are parity with x86 also