Skip to content

Commit

Permalink
Pypi packaging (#46)
Browse files Browse the repository at this point in the history
* improve some metadata for packaging purposes

* add some comments about the keyvault secrets configs

* remove reference to an old file

* comments

* Bump version: 0.1.0 → 0.2.0

* improvements on readme

* Bump version: 0.2.0 → 0.2.1

* initial attempts at publishing pypi packages

* fixups for when matplotlib is not installed

* Bump version: 0.2.1 → 0.2.2

* fixups

* Revert "Bump version: 0.2.1 → 0.2.2"

This reverts commit 3b40ded.

* tweak

* Bump version: 0.2.1 → 0.2.2

* fixups

* Add rules for publishing packages

* need the extra file for the container build

* pylint fixups

* don't require the readme to rebuild the devcontainer - breaks the cache

* remove explicit version numbers from the readme

* update readme

* fixups

* tweaks

* link fixup

* fixups

* also tag devcontainers

* cosmetic

* FIXME: increase rebuild timeout

* testing
  • Loading branch information
bpkroth authored Jan 9, 2024
1 parent 564df0e commit 2507ccb
Show file tree
Hide file tree
Showing 12 changed files with 240 additions and 55 deletions.
6 changes: 1 addition & 5 deletions .bumpversion.cfg
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
[bumpversion]
current_version = 0.1.0
current_version = 0.2.2
commit = True
tag = True

[bumpversion:file:README.md]

[bumpversion:file:doc/source/installation.rst]

[bumpversion:file:doc/source/conf.py]

[bumpversion:file:mlos_core/_version.py]
Expand Down
57 changes: 48 additions & 9 deletions .github/workflows/devcontainer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@ on:
description: Disable caching?
default: false
required: false
release:
types: [published]
push:
tags: ["v*"]
branches: [ main ]
pull_request:
branches: [ main ]
Expand Down Expand Up @@ -144,7 +147,9 @@ jobs:
docker exec --user vscode --env USER=vscode mlos-devcontainer make CONDA_INFO_LEVEL=-v dist dist-test
- name: Test rebuilding the devcontainer in the devcontainer
timeout-minutes: 3
# FIXME:
# timeout-minutes: 3
timeout-minutes: 10
run: |
set -x
git --no-pager diff --exit-code
Expand All @@ -157,6 +162,24 @@ jobs:
# Make sure we can publish the coverage report.
rm -f doc/build/html/htmlcov/.gitignore
- name: Publish package to Test PyPi
if: github.event == 'release' || github.ref_type == 'tag'
run: |
if [ -n "${{ secrets.PYPI_TEST_USERNAME }}" ]; then
docker exec --user vscode --env USER=vscode --env MAKEFLAGS=-Oline \
--env TWINE_USERNAME=${{ secrets.PYPI_TEST_USERNAME }} --env TWINE_PASSWORD=${{ secrets.PYPI_TEST_PASSWORD }} \
mlos-devcontainer make CONDA_INFO_LEVEL=-v publish-test-pypi
fi
- name: Publish package to PyPi
if: github.repository == 'microsoft/mlos' && (github.event == 'release' || github.ref_type == 'tag')
run: |
if [ -n "${{ secrets.PYPI_USERNAME }}" ]; then
docker exec --user vscode --env USER=vscode --env MAKEFLAGS=-Oline \
--env TWINE_USERNAME=${{ secrets.PYPI_USERNAME }} --env TWINE_PASSWORD=${{ secrets.PYPI_PASSWORD }} \
mlos-devcontainer make CONDA_INFO_LEVEL=-v publish-pypi
fi
- name: Deploy to GitHub pages
if: github.ref == 'refs/heads/main'
uses: JamesIves/github-pages-deploy-action@v4
Expand All @@ -172,20 +195,36 @@ jobs:
docker rm --force mlos-devcontainer || true
- name: Container Registry Login
if: github.repository == 'microsoft/mlos' && github.ref == 'refs/heads/main'
# FIXME: testing
if: (github.repository == 'microsoft/mlos' || github.repository == 'bpkroth/mlos') && (github.ref == 'refs/heads/main' || github.event == 'release' || github.ref_type == 'tag')
uses: docker/login-action@v3
with:
# This is the URL of the container registry, which is configured in Github
# Settings and currently corresponds to the mlos-core ACR.
registry: ${{ secrets.ACR_LOGINURL }}
username: ${{ secrets.ACR_USERNAME }}
# This secret is configured in Github Settings.
# It can also be obtained in a keyvault in the Azure portal alongside the
# other resources used.
password: ${{ secrets.ACR_PASSWORD }}
- name: Publish the container images
if: github.repository == 'microsoft/mlos' && github.ref == 'refs/heads/main'
# FIXME: testing
if: (github.repository == 'microsoft/mlos' || github.repository == 'bpkroth/mlos') && (github.ref == 'refs/heads/main' || github.event == 'release' || github.ref_type == 'tag')
timeout-minutes: 15
# We only push to the :latest tag, to avoid needing to cleanup the
# registry manually (there's currently no API for that).
run: |
set -x
docker tag devcontainer-cli:latest ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:latest
docker push ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:latest
docker tag mlos-devcontainer:latest ${{ secrets.ACR_LOGINURL }}/mlos-devcontainer:latest
docker push ${{ secrets.ACR_LOGINURL }}/mlos-devcontainer:latest
image_tag=''
if [ "${{ github.ref }}" == 'refs/heads/main' ]; then
image_tag='latest'
elif [ "${{ github.event }}" == 'release' ] || [ "${{ github.ref_type }}" == 'tag' ]; then
image_tag="${{ github.ref_name }}"
fi
if [ -z "$image_tag" ]; then
echo "ERROR: Unhandled event condition or ref: event=${{ github.event}}, ref=${{ github.ref }}, ref_type=${{ github.ref_type }}"
exit 1
fi
docker tag devcontainer-cli:latest ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:$image_tag
docker push ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:$image_tag
docker tag mlos-devcontainer:latest ${{ secrets.ACR_LOGINURL }}/mlos-devcontainer:$image_tag
docker push ${{ secrets.ACR_LOGINURL }}/mlos-devcontainer:$image_tag
27 changes: 26 additions & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ Simply open the project in VSCode and follow the prompts to build and open the d
conda activate mlos
```

## Details
### Details

[`main`](https://github.com/microsoft/MLOS/tree/main) is considered the primary development branch.

Expand All @@ -74,6 +74,31 @@ We expect development to follow a typical "forking" style workflow:
4. PRs are associated with [Github Issues](https://github.com/microsoft/MLOS/issues) and need [MLOS-committers](https://github.com/orgs/microsoft/teams/MLOS-committers) to sign-off (in addition to other CI pipeline checks like tests and lint checks to pass).
5. Once approved, the PR can be completed using a squash merge in order to keep a nice linear history.

## Distributing

You can also locally build and install from wheels like so:

1. Build the *wheel* file(s)

```sh
make dist
```

2. Install it.

```sh
# this will install just the optimizer component with SMAC support:
pip install "dist/tmp/mlos_core-latest-py3-none-any.whl[smac]"
```

```sh
# this will install both the optimizer and the experiment runner:
pip install "dist/mlos_bench-latest-py3-none-any.whl[azure]"
```

> Note: exact versions may differ due to automatic versioning so the `-latest-` part is a symlink.
> If distributing elsewhere, adjust for the current version number in the module's `dist` directory.
### See Also
- <https://docs.github.com/en/get-started/quickstart/fork-a-repo>
Expand Down
28 changes: 27 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,8 @@ mlos_bench/dist/tmp/mlos-bench-latest.tar: PACKAGE_NAME := mlos-bench
# Check to make sure the mlos_bench module has the config directory.
[ "$(MODULE_NAME)" != "mlos_bench" ] || unzip -t $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl | grep -m1 mlos_bench/config/
cd $(MODULE_NAME)/dist/tmp && ln -s ../$(MODULE_NAME)-*-py3-none-any.whl $(MODULE_NAME)-latest-py3-none-any.whl
# Check to make sure the README contents made it into the package metadata.
unzip -p $(MODULE_NAME)/dist/tmp/$(MODULE_NAME)-latest-py3-none-any.whl */METADATA | egrep -v '^[A-Z][a-zA-z-]+:' | grep -q -i '^# mlos'

.PHONY: dist-test-env-clean
dist-test-env-clean:
Expand Down Expand Up @@ -279,6 +281,27 @@ dist-test-clean: dist-test-env-clean
rm -f build/dist-test-env.$(PYTHON_VERSION).build-stamp


.PHONY: publish
publish: publish-pypi

.PHONY:
publish-pypi-deps: build/publish-pypi-deps.build-stamp

build/publish-pypi-deps.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp
conda run -n ${CONDA_ENV_NAME} pip install -U twine
touch $@

build/publish.%.py.build-stamp: build/publish-pypi-deps.${CONDA_ENV_NAME}.build-stamp build/pytest.${CONDA_ENV_NAME}.build-stamp build/dist-test.$(PYTHON_VERSION).build-stamp build/check-doc.build-stamp build/linklint-doc.build-stamp
rm -f mlos_*/dist/*.tar.gz
ls mlos_*/dist/*.tar | xargs -I% gzip -k %
repo_name=`echo "$@" | sed -e 's|build/publish\.||' -e 's|\.py\.build-stamp||'` \
&& conda run -n ${CONDA_ENV_NAME} python3 -m twine upload --repository $$repo_name \
mlos_*/dist/mlos*-*.tar.gz mlos_*/dist/mlos*-*.whl
touch $@

publish-pypi: build/publish.pypi.py.build-stamp
publish-test-pypi: build/publish.testpypi.py.build-stamp

build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp
build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp: doc/requirements.txt
conda run -n ${CONDA_ENV_NAME} pip install -U -r doc/requirements.txt
Expand Down Expand Up @@ -340,7 +363,10 @@ doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/copy-source-
# See check-doc

.PHONY: doc
doc: doc/build/html/.nojekyll build/check-doc.build-stamp build/linklint-doc.build-stamp
doc: doc/build/html/.nojekyll doc-test

.PHONY: doc-test
doc-test: build/check-doc.build-stamp build/linklint-doc.build-stamp

doc/build/html/htmlcov/index.html: doc/build/html/index.html
# Make the codecov html report available for the site.
Expand Down
39 changes: 17 additions & 22 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ MLOS is a project to enable autotuning for systems.
- [Usage Examples](#usage-examples)
- [mlos-core](#mlos-core)
- [mlos-bench](#mlos-bench)
- [Distributing](#distributing)
- [Installation](#installation)
- [See Also](#see-also)
- [Examples](#examples)

Expand Down Expand Up @@ -127,35 +127,30 @@ See Also:
- [mlos_bench/config](./mlos_bench/mlos_bench/config/) for additional configuration details.
- [sqlite-autotuning](https://github.com/Microsoft-CISL/sqlite-autotuning) for a complete external example of using MLOS to tune `sqlite`.
## Distributing
## Installation
MLOS is not [*yet*](https://github.com/microsoft/MLOS/issues/547) published on `pypi`, so until them here are some instructions for installation for usage in production or other environments.
The MLOS modules are published to [pypi](https://pypi.org) when new tags/releases are made.
1. Build the *wheel* file(s)
To install the latest release, simply run:
```sh
make dist
```
```sh
# this will install just the optimizer component with SMAC support:
pip install -U mlos-core[smac]
2. Install it (e.g. after copying it somewhere else).
```sh
# this will install just the optimizer component with SMAC support:
pip install dist/mlos_core-0.1.0-py3-none-any.whl[smac]
# this will install just the optimizer component with flaml support:
pip install -U "mlos-core[flaml]"
# this will install just the optimizer component with flaml support:
pip install dist/mlos_core-0.1.0-py3-none-any.whl[flaml]
# this will install just the optimizer component with smac and flaml support:
pip install -U "mlos-core[smac,flaml]"
# this will install just the optimizer component with smac and flaml support:
pip install dist/mlos_core-0.1.0-py3-none-any.whl[smac,flaml]
```
# this will install both the flaml optimizer and the experiment runner with azure support:
pip install -U "mlos-bench[flaml,azure]"
```sh
# this will install both the optimizer and the experiment runner:
pip install dist/mlos_bench-0.1.0-py3-none-any.whl
```
# this will install both the smac optimizer and the experiment runner with ssh support:
pip install -U "mlos-bench[smac,ssh]"
```
> Note: exact versions may differ due to automatic versioning.
Details on using a local version from git are available in [CONTRIBUTING.md](./CONTRIBUTING.md).
## See Also
Expand Down
20 changes: 12 additions & 8 deletions conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,15 @@ def pytest_configure(config: pytest.Config) -> None:
"""
# Workaround some issues loading emukit in certain environments.
if os.environ.get('DISPLAY', None):
import matplotlib # pylint: disable=import-outside-toplevel
matplotlib.rcParams['backend'] = 'agg'
if is_master(config) or dict(getattr(config, 'workerinput', {}))['workerid'] == 'gw0':
# Only warn once.
warn(UserWarning('DISPLAY environment variable is set, which can cause problems in some setups (e.g. WSL). '
+ f'Adjusting matplotlib backend to "{matplotlib.rcParams["backend"]}" to compensate.'))
try:
import matplotlib # pylint: disable=import-outside-toplevel
matplotlib.rcParams['backend'] = 'agg'
if is_master(config) or dict(getattr(config, 'workerinput', {}))['workerid'] == 'gw0':
# Only warn once.
warn(UserWarning('DISPLAY environment variable is set, which can cause problems in some setups (e.g. WSL). '
+ f'Adjusting matplotlib backend to "{matplotlib.rcParams["backend"]}" to compensate.'))
except ImportError:
pass

# Create a temporary directory for sharing files between master and worker nodes.
if is_master(config):
Expand Down Expand Up @@ -72,8 +75,9 @@ def pytest_unconfigure(config: pytest.Config) -> None:
Called after all tests have completed.
"""
if is_master(config):
shared_tmp_dir = str(getattr(config, "shared_temp_dir"))
shutil.rmtree(shared_tmp_dir)
shared_tmp_dir = getattr(config, "shared_temp_dir", None)
if shared_tmp_dir:
shutil.rmtree(str(shared_tmp_dir))


@pytest.fixture(scope="session")
Expand Down
2 changes: 1 addition & 1 deletion doc/source/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
author = 'GSL'

# The full version, including alpha/beta/rc tags
release = '0.1.0'
release = '0.2.2'

try:
from setuptools_scm import get_version
Expand Down
2 changes: 1 addition & 1 deletion mlos_bench/_version.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@
"""

# NOTE: This should be managed by bumpversion.
_VERSION = '0.1.0'
_VERSION = '0.2.2'
56 changes: 53 additions & 3 deletions mlos_bench/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,46 @@
Setup instructions for the mlos_bench package.
"""

# pylint: disable=duplicate-code

from logging import warning
from itertools import chain
from typing import Dict, List

import os
import re

from setuptools import setup, find_packages

from _version import _VERSION # pylint: disable=import-private-name


# A simple routine to read and adjust the README.md for this module into a format
# suitable for packaging.
# See Also: copy-source-tree-docs.sh
# Unfortunately we can't use that directly due to the way packaging happens inside a
# temp directory.
# Similarly, we can't use a utility script outside this module, so this code has to
# be duplicated for now.
def _get_long_desc_from_readme(base_url: str) -> dict:
pkg_dir = os.path.dirname(__file__)
readme_path = os.path.join(pkg_dir, 'README.md')
if not os.path.isfile(readme_path):
return {}
jsonc_re = re.compile(r'```jsonc')
link_re = re.compile(r'\]\(([^:#)]+)(#[a-zA-Z0-9_-]+)?\)')
with open(readme_path, mode='r', encoding='utf-8') as readme_fh:
lines = readme_fh.readlines()
# Tweak the lexers for local expansion by pygments instead of github's.
lines = [link_re.sub(f"]({base_url}" + r'/\1\2)', line) for line in lines]
# Tweak source source code links.
lines = [jsonc_re.sub(r'```json', line) for line in lines]
return {
'long_description': ''.join(lines),
'long_description_content_type': 'text/markdown',
}


try:
from setuptools_scm import get_version
version = get_version(root='..', relative_to=__file__)
Expand Down Expand Up @@ -85,10 +117,28 @@
] + extra_requires['storage-sql-sqlite'], # NOTE: For now sqlite is a fallback storage backend, so we always install it.
extras_require=extra_requires,
author='Microsoft',
license='MIT',
**_get_long_desc_from_readme('https://github.com/microsoft/MLOS/tree/main/mlos_bench'),
author_email='[email protected]',
description=('MLOS Bench Python interface for benchmark automation and optimization.'),
license='MIT',
keywords='',
url='https://aka.ms/mlos-core',
url='https://github.com/microsoft/MLOS',
project_urls={
'Documentation': 'https://microsoft.github.io/MLOS',
'Package Source': 'https://github.com/microsoft/MLOS/tree/main/mlos_bench/',
},
python_requires='>=3.8',
keywords=[
'autotuning',
'benchmarking',
'optimization',
'systems',
],
classifiers=[
"Intended Audience :: Developers",
"Intended Audience :: Science/Research",
"Intended Audience :: System Administrators",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3",
],
)
2 changes: 1 addition & 1 deletion mlos_core/_version.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@
"""

# NOTE: This should be managed by bumpversion.
_VERSION = '0.1.0'
_VERSION = '0.2.2'
Loading

0 comments on commit 2507ccb

Please sign in to comment.