Skip to content

Commit

Permalink
Add support for targeting musl Linux. (#138)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsirois
jsirois authored Feb 17, 2025
1 parent 35eb871 commit eea2739
Show file tree
Hide file tree
Showing 29 changed files with 861 additions and 214 deletions.
116 changes: 77 additions & 39 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,19 +38,28 @@ jobs:
# non-deprecated ARM Mac runner.
include:
- os: ubuntu-24.04
name: Linux x86-64
name: Linux x86-64 (musl)
docker-image: python:3.12-alpine
docker-platform: linux/amd64
- os: ubuntu-24.04
name: Linux x86-64 (glibc)
docker-image: python:3.12-bookworm
docker-platform: linux/amd64
- os: ubuntu-24.04
name: Linux aarch64
docker-image: python:3.12-bookworm
docker-platform: linux/arm64
- os: ubuntu-24.04
name: Linux armv7l
docker-image: python:3.12-bookworm
docker-platform: linux/arm/v7
- os: ubuntu-24.04
name: Linux s390x
docker-image: python:3.12-bookworm
docker-platform: linux/s390x
- os: ubuntu-24.04
name: Linux powerpc64le
docker-image: python:3.12-bookworm
docker-platform: linux/ppc64le
- os: macos-13
name: macOS x86-64
Expand All @@ -65,10 +74,10 @@ jobs:
SCIENCE_AUTH_API_GITHUB_COM_BEARER: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: Install the latest version of uv
if: matrix.docker-platform == '' && matrix.os != 'windows-arm64'
if: matrix.docker-image == '' && matrix.os != 'windows-arm64'
uses: astral-sh/setup-uv@v5
- name: Setup uv
if: matrix.docker-platform == '' && matrix.os != 'windows-arm64'
if: matrix.docker-image == '' && matrix.os != 'windows-arm64'
run: |
export UV="$(which uv)"
"${UV}" -V
Expand Down Expand Up @@ -98,7 +107,7 @@ jobs:
"${UV}" python install ${UV_PYTHON_VERSION}
echo UV_PYTHON="${UV_PYTHON_VERSION}" >> ${GITHUB_ENV}
- name: Installing emulators
if: matrix.docker-platform != ''
if: matrix.docker-image != ''
run: docker run --privileged --rm tonistiigi/binfmt --install all
- name: Checkout Lift
uses: actions/checkout@v4
Expand All @@ -112,25 +121,32 @@ jobs:
key: ${{ matrix.docker-platform || format('{0}-{1}', matrix.os, runner.arch) }}-a-scie-lift-mypy-v1-${{ github.run_id }}
restore-keys: ${{ matrix.docker-platform || format('{0}-{1}', matrix.os, runner.arch) }}-a-scie-lift-mypy-v1
- name: Check Formatting & Lints
if: matrix.docker-platform == ''
if: matrix.docker-image == ''
run: |
"${UV}" run dev-cmd ci --skip test
- name: Check Formatting & Lints
if: matrix.docker-platform != ''
if: matrix.docker-image != ''
run: |
cat << EOF > _fmt_lint_check.sh
pip install --root-user-action ignore uv
addgroup --gid $(id -g) build
if [ "${{ matrix.docker-image }}" = "python:3.12-alpine" ]; then
adduser -D -g '' -G build -u $(id -u) build
apk add gcc git linux-headers musl-dev python3-dev
else
adduser --disabled-password --gecos '' --gid $(id -g) --uid $(id -u) build
fi
su build -c 'uv run dev-cmd ci --skip test'
EOF
docker run --rm \
-e FORCE_COLOR \
-e SCIENCE_AUTH_API_GITHUB_COM_BEARER \
-v $PWD:/code \
-w /code \
--platform ${{ matrix.docker-platform }} \
python:3.12-bookworm \
bash -c "
pip install --root-user-action ignore uv &&
addgroup --gid $(id -g) build &&
adduser --disabled-password --gecos '' --gid $(id -g) --uid $(id -u) build &&
su build -c 'uv run dev-cmd ci --skip test'
"
${{ matrix.docker-image }} sh -eu _fmt_lint_check.sh
- name: Cache MyPy
uses: actions/cache/save@v4
if: github.ref == 'refs/heads/main'
Expand All @@ -144,26 +160,34 @@ jobs:
echo PYTEST_ADDOPTS="--basetemp C:/tmp/gha/pytest" >> ${GITHUB_ENV}
echo SCIE_BASE=C:/tmp/gha/nce >> ${GITHUB_ENV}
- name: Unit Tests
if: matrix.docker-platform == ''
if: matrix.docker-image == ''
run: |
"${UV}" run dev-cmd test -- -vvs
- name: Unit Tests
if: matrix.docker-platform != ''
if: matrix.docker-image != ''
run: |
cat << EOF > _ci_test.sh
set -euo pipefail
if [[ "${{ matrix.docker-platform }}" == "linux/s390x" ]]; then
cat << EOF > _test.sh
if [ "${{ matrix.docker-platform }}" = "linux/s390x" ]; then
# This hack gets the PyPy provider tests working on this image. The old PyPy s390x
# distributions dynamically link libffi at an older version than I've been able to
# find a multi-platform image with s390x support for.
ln -s /usr/lib/s390x-linux-gnu/libffi.so.8 /usr/lib/s390x-linux-gnu/libffi.so.6
elif [[ "${{ matrix.docker-platform }}" == "linux/ppc64le" ]]; then
elif [ "${{ matrix.docker-platform }}" = "linux/ppc64le" ]; then
echo "Skipping tests on ppc64le."
exit 0
fi
pip install --root-user-action ignore uv
addgroup --gid $(id -g) build
adduser --disabled-password --gecos '' --gid $(id -g) --uid $(id -u) build
if [ "${{ matrix.docker-image }}" = "python:3.12-alpine" ]; then
adduser -D -g '' -G build -u $(id -u) build
# N.B.: The bash and curl packages are additional needs for tests. The rest just
# supports building the psutil wheel needed for the science Python distribution.
apk add bash curl gcc git linux-headers musl-dev python3-dev
else
adduser --disabled-password --gecos '' --gid $(id -g) --uid $(id -u) build
fi
su build -c "uv run dev-cmd test -- -vvs"
EOF
Expand All @@ -173,47 +197,61 @@ jobs:
-v $PWD:/code \
-w /code \
--platform ${{ matrix.docker-platform }} \
python:3.12-bookworm bash _ci_test.sh
${{ matrix.docker-image }} sh -eu _test.sh
- name: Build & Package
if: matrix.docker-platform == ''
if: matrix.docker-image == ''
run: |
"${UV}" run dev-cmd package
- name: Build & Package
if: matrix.docker-platform != ''
if: matrix.docker-image != ''
run: |
cat << EOF > _package.sh
pip install --root-user-action ignore uv
addgroup --gid $(id -g) build
if [ "${{ matrix.docker-image }}" = "python:3.12-alpine" ]; then
adduser -D -g '' -G build -u $(id -u) build
apk add gcc git linux-headers musl-dev python3-dev
else
adduser --disabled-password --gecos '' --gid $(id -g) --uid $(id -u) build
fi
su build -c 'uv run dev-cmd package'
EOF
docker run --rm \
-e FORCE_COLOR \
-e SCIENCE_AUTH_API_GITHUB_COM_BEARER \
-v $PWD:/code \
-w /code \
--platform ${{ matrix.docker-platform }} \
python:3.12-bookworm \
bash -c "
pip install --root-user-action ignore uv &&
addgroup --gid $(id -g) build &&
adduser --disabled-password --gecos '' --gid $(id -g) --uid $(id -u) build &&
su build -c 'uv run dev-cmd package'
"
${{ matrix.docker-image }} sh -eu _package.sh
- name: Generate Doc Site
if: matrix.docker-platform == ''
if: matrix.docker-image == ''
run: |
"${UV}" run dev-cmd doc linkcheck
- name: Generate Doc Site
if: matrix.docker-platform != ''
if: matrix.docker-image != ''
run: |
cat << EOF > _doc_linkcheck.sh
pip install --root-user-action ignore uv
addgroup --gid $(id -g) build
if [ "${{ matrix.docker-image }}" = "python:3.12-alpine" ]; then
adduser -D -g '' -G build -u $(id -u) build
apk add gcc git linux-headers musl-dev python3-dev
else
adduser --disabled-password --gecos '' --gid $(id -g) --uid $(id -u) build
fi
su build -c 'uv run dev-cmd doc linkcheck'
EOF
docker run --rm \
-e FORCE_COLOR \
-e SCIENCE_AUTH_API_GITHUB_COM_BEARER \
-v $PWD:/code \
-w /code \
--platform ${{ matrix.docker-platform }} \
python:3.12-bookworm \
bash -c "
pip install --root-user-action ignore uv &&
addgroup --gid $(id -g) build &&
adduser --disabled-password --gecos '' --gid $(id -g) --uid $(id -u) build &&
su build -c 'uv run dev-cmd doc linkcheck'
"
${{ matrix.docker-image }} sh -eu _doc_linkcheck.sh
- name: Cleanup
if: always()
run: |
Expand Down
42 changes: 29 additions & 13 deletions .github/workflows/release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,19 +55,28 @@ jobs:
# non-deprecated ARM Mac runner.
include:
- os: ubuntu-24.04
name: Linux x86-64
name: Linux x86-64 (musl)
docker-image: python:3.12-alpine
docker-platform: linux/amd64
- os: ubuntu-24.04
name: Linux x86-64 (glibc)
docker-image: python:3.12-bookworm
docker-platform: linux/amd64
- os: ubuntu-24.04
name: Linux aarch64
docker-image: python:3.12-bookworm
docker-platform: linux/arm64
- os: ubuntu-24.04
name: Linux armv7l
docker-image: python:3.12-bookworm
docker-platform: linux/arm/v7
- os: ubuntu-24.04
name: Linux s390x
docker-image: python:3.12-bookworm
docker-platform: linux/s390x
- os: ubuntu-24.04
name: Linux powerpc64le
docker-image: python:3.12-bookworm
docker-platform: linux/ppc64le
- os: macos-13
name: macOS x86-64
Expand All @@ -88,10 +97,10 @@ jobs:
discussions: write
steps:
- name: Install the latest version of uv
if: matrix.docker-platform == '' && matrix.os != 'windows-arm64'
if: matrix.docker-image == '' && matrix.os != 'windows-arm64'
uses: astral-sh/setup-uv@v5
- name: Setup uv
if: matrix.docker-platform == '' && matrix.os != 'windows-arm64'
if: matrix.docker-image == '' && matrix.os != 'windows-arm64'
run: |
export UV="$(which uv)"
"${UV}" -V
Expand Down Expand Up @@ -121,32 +130,39 @@ jobs:
"${UV}" python install ${UV_PYTHON_VERSION}
echo UV_PYTHON_ARGS="--python ${UV_PYTHON_VERSION}" >> ${GITHUB_ENV}
- name: Installing emulators
if: matrix.docker-platform != ''
if: matrix.docker-image != ''
run: docker run --privileged --rm tonistiigi/binfmt --install all
- name: Checkout lift ${{ needs.determine-tag.outputs.release-tag }}
uses: actions/checkout@v4
with:
ref: ${{ needs.determine-tag.outputs.release-tag }}
- name: Package science ${{ needs.determine-tag.outputs.release-tag }} binary
if: matrix.docker-platform == ''
if: matrix.docker-image == ''
run: |
"${UV}" run ${UV_PYTHON_ARGS} dev-cmd package
- name: Package science ${{ needs.determine-tag.outputs.release-tag }} binary
if: matrix.docker-platform != ''
if: matrix.docker-image != ''
run: |
cat << EOF > _package.sh
pip install --root-user-action ignore uv
addgroup --gid $(id -g) build
if [ "${{ matrix.docker-image }}" = "python:3.12-alpine" ]; then
adduser -D -g '' -G build -u $(id -u) build
apk add gcc git linux-headers musl-dev python3-dev
else
adduser --disabled-password --gecos '' --gid $(id -g) --uid $(id -u) build
fi
su build -c 'uv run dev-cmd package'
EOF
docker run --rm \
-e FORCE_COLOR \
-e SCIENCE_AUTH_API_GITHUB_COM_BEARER \
-v $PWD:/code \
-w /code \
--platform ${{ matrix.docker-platform }} \
python:3.12-bookworm \
bash -c "
pip install --root-user-action ignore uv &&
addgroup --gid $(id -g) build &&
adduser --disabled-password --gecos '' --gid $(id -g) --uid $(id -u) build &&
su build -c 'uv run dev-cmd package'
"
${{ matrix.docker-image }} sh -eu _package.sh
- name: Generate science ${{ needs.determine-tag.outputs.release-tag }} artifact attestations
uses: actions/attest-build-provenance@v1
with:
Expand Down
5 changes: 5 additions & 0 deletions CHANGES.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# Release Notes

## 0.12.0

This release adds support for targeting musl libc systems and dogfoods this to ship a `science`
scie for 64 bit musl Linux.

## 0.11.3

Upgrade the science internal Python distribution to [PBS][PBS] CPython 3.12.9.
Expand Down
26 changes: 14 additions & 12 deletions docs/_ext/sphinx_science/toml.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,13 @@
class TOMLType:
label: str

def render_value(self, value: Any) -> str:
def render_value(self, value: Any) -> Any:
return repr(value)


@dataclass(frozen=True)
class PrimitiveType(TOMLType):
def render_value(self, value: Any) -> str:
def render_value(self, value: Any) -> Any:
if isinstance(value, bool):
return "true" if value else "false"
return repr(value)
Expand All @@ -51,8 +51,8 @@ def create(cls, item_type: TOMLType) -> Self:

item_type: TOMLType

def render_value(self, value: Any) -> str:
return repr([self.item_type.render_value(item) for item in value])
def render_value(self, value: Any) -> Any:
return [self.item_type.render_value(item) for item in value]


@dataclass(frozen=True)
Expand All @@ -63,10 +63,10 @@ def create(cls, value_type: TOMLType) -> Self:

value_type: TOMLType | None = None

def render_value(self, value: Any) -> str:
def render_value(self, value: Any) -> Any:
if self.value_type:
return repr({key: self.value_type.render_value(val) for key, val in value.items()})
return repr(value)
return {key: self.value_type.render_value(val) for key, val in value.items()}
return value


@dataclass(frozen=True)
Expand All @@ -84,9 +84,9 @@ def for_enum(
renderer=lambda value: enum_type(value).value,
)

renderer: Callable[[Any], str]
renderer: Callable[[Any], Any]

def render_value(self, value: Any) -> str:
def render_value(self, value: Any) -> Any:
return self.renderer(value)


Expand All @@ -105,9 +105,9 @@ def for_type_info(
renderer=lambda value: toml_type_factory(type(value)).render_value(value),
)

renderer: Callable[[Any], str]
renderer: Callable[[Any], Any]

def render_value(self, value: Any) -> str:
def render_value(self, value: Any) -> Any:
return self.renderer(value)


Expand Down Expand Up @@ -270,7 +270,9 @@ def render_dataclass(self, data_type: type[Dataclass]) -> Iterator[nodes.Node]:
fields.extendleft(dataclass_info(field_dataclass_type).field_info)
continue

field_section = dataclass_section.create_subsection(title=field.name, name=field.name)
field_section = dataclass_section.create_subsection(
title=field.display_name, name=field.display_name
)
field_section.extend(self.render_field(field, owner=class_info.type))

if self._recurse_tables:
Expand Down
Loading

0 comments on commit eea2739

Please sign in to comment.