From 2507ccbc771aa566b1e0f0be9d4a8a39d0632a78 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Tue, 9 Jan 2024 13:00:34 -0600 Subject: [PATCH] Pypi packaging (#46) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * improve some metadata for packaging purposes * add some comments about the keyvault secrets configs * remove reference to an old file * comments * Bump version: 0.1.0 → 0.2.0 * improvements on readme * Bump version: 0.2.0 → 0.2.1 * initial attempts at publishing pypi packages * fixups for when matplotlib is not installed * Bump version: 0.2.1 → 0.2.2 * fixups * Revert "Bump version: 0.2.1 → 0.2.2" This reverts commit 3b40ded1fc0f09055db9e1a85811cf239ce59548. * tweak * Bump version: 0.2.1 → 0.2.2 * fixups * Add rules for publishing packages * need the extra file for the container build * pylint fixups * don't require the readme to rebuild the devcontainer - breaks the cache * remove explicit version numbers from the readme * update readme * fixups * tweaks * link fixup * fixups * also tag devcontainers * cosmetic * FIXME: increase rebuild timeout * testing --- .bumpversion.cfg | 6 +--- .github/workflows/devcontainer.yml | 57 +++++++++++++++++++++++++----- CONTRIBUTING.md | 27 +++++++++++++- Makefile | 28 ++++++++++++++- README.md | 39 +++++++++----------- conftest.py | 20 ++++++----- doc/source/conf.py | 2 +- mlos_bench/_version.py | 2 +- mlos_bench/setup.py | 56 +++++++++++++++++++++++++++-- mlos_core/_version.py | 2 +- mlos_core/setup.py | 55 ++++++++++++++++++++++++++-- scripts/update-version.sh | 1 + 12 files changed, 240 insertions(+), 55 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 135eeaf62da..7861c45b260 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,12 +1,8 @@ [bumpversion] -current_version = 0.1.0 +current_version = 0.2.2 commit = True tag = True -[bumpversion:file:README.md] - -[bumpversion:file:doc/source/installation.rst] - [bumpversion:file:doc/source/conf.py] [bumpversion:file:mlos_core/_version.py] diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml index 1262b37d1c5..f8666523322 100644 --- a/.github/workflows/devcontainer.yml +++ b/.github/workflows/devcontainer.yml @@ -10,7 +10,10 @@ on: description: Disable caching? default: false required: false + release: + types: [published] push: + tags: ["v*"] branches: [ main ] pull_request: branches: [ main ] @@ -144,7 +147,9 @@ jobs: docker exec --user vscode --env USER=vscode mlos-devcontainer make CONDA_INFO_LEVEL=-v dist dist-test - name: Test rebuilding the devcontainer in the devcontainer - timeout-minutes: 3 + # FIXME: + # timeout-minutes: 3 + timeout-minutes: 10 run: | set -x git --no-pager diff --exit-code @@ -157,6 +162,24 @@ jobs: # Make sure we can publish the coverage report. rm -f doc/build/html/htmlcov/.gitignore + - name: Publish package to Test PyPi + if: github.event == 'release' || github.ref_type == 'tag' + run: | + if [ -n "${{ secrets.PYPI_TEST_USERNAME }}" ]; then + docker exec --user vscode --env USER=vscode --env MAKEFLAGS=-Oline \ + --env TWINE_USERNAME=${{ secrets.PYPI_TEST_USERNAME }} --env TWINE_PASSWORD=${{ secrets.PYPI_TEST_PASSWORD }} \ + mlos-devcontainer make CONDA_INFO_LEVEL=-v publish-test-pypi + fi + + - name: Publish package to PyPi + if: github.repository == 'microsoft/mlos' && (github.event == 'release' || github.ref_type == 'tag') + run: | + if [ -n "${{ secrets.PYPI_USERNAME }}" ]; then + docker exec --user vscode --env USER=vscode --env MAKEFLAGS=-Oline \ + --env TWINE_USERNAME=${{ secrets.PYPI_USERNAME }} --env TWINE_PASSWORD=${{ secrets.PYPI_PASSWORD }} \ + mlos-devcontainer make CONDA_INFO_LEVEL=-v publish-pypi + fi + - name: Deploy to GitHub pages if: github.ref == 'refs/heads/main' uses: JamesIves/github-pages-deploy-action@v4 @@ -172,20 +195,36 @@ jobs: docker rm --force mlos-devcontainer || true - name: Container Registry Login - if: github.repository == 'microsoft/mlos' && github.ref == 'refs/heads/main' + # FIXME: testing + if: (github.repository == 'microsoft/mlos' || github.repository == 'bpkroth/mlos') && (github.ref == 'refs/heads/main' || github.event == 'release' || github.ref_type == 'tag') uses: docker/login-action@v3 with: + # This is the URL of the container registry, which is configured in Github + # Settings and currently corresponds to the mlos-core ACR. registry: ${{ secrets.ACR_LOGINURL }} username: ${{ secrets.ACR_USERNAME }} + # This secret is configured in Github Settings. + # It can also be obtained in a keyvault in the Azure portal alongside the + # other resources used. password: ${{ secrets.ACR_PASSWORD }} - name: Publish the container images - if: github.repository == 'microsoft/mlos' && github.ref == 'refs/heads/main' + # FIXME: testing + if: (github.repository == 'microsoft/mlos' || github.repository == 'bpkroth/mlos') && (github.ref == 'refs/heads/main' || github.event == 'release' || github.ref_type == 'tag') timeout-minutes: 15 - # We only push to the :latest tag, to avoid needing to cleanup the - # registry manually (there's currently no API for that). run: | set -x - docker tag devcontainer-cli:latest ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:latest - docker push ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:latest - docker tag mlos-devcontainer:latest ${{ secrets.ACR_LOGINURL }}/mlos-devcontainer:latest - docker push ${{ secrets.ACR_LOGINURL }}/mlos-devcontainer:latest + image_tag='' + if [ "${{ github.ref }}" == 'refs/heads/main' ]; then + image_tag='latest' + elif [ "${{ github.event }}" == 'release' ] || [ "${{ github.ref_type }}" == 'tag' ]; then + image_tag="${{ github.ref_name }}" + fi + if [ -z "$image_tag" ]; then + echo "ERROR: Unhandled event condition or ref: event=${{ github.event}}, ref=${{ github.ref }}, ref_type=${{ github.ref_type }}" + exit 1 + fi + + docker tag devcontainer-cli:latest ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:$image_tag + docker push ${{ secrets.ACR_LOGINURL }}/devcontainer-cli:$image_tag + docker tag mlos-devcontainer:latest ${{ secrets.ACR_LOGINURL }}/mlos-devcontainer:$image_tag + docker push ${{ secrets.ACR_LOGINURL }}/mlos-devcontainer:$image_tag diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a4a13c5b86..d6f0377a01e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ Simply open the project in VSCode and follow the prompts to build and open the d conda activate mlos ``` -## Details +### Details [`main`](https://github.com/microsoft/MLOS/tree/main) is considered the primary development branch. @@ -74,6 +74,31 @@ We expect development to follow a typical "forking" style workflow: 4. PRs are associated with [Github Issues](https://github.com/microsoft/MLOS/issues) and need [MLOS-committers](https://github.com/orgs/microsoft/teams/MLOS-committers) to sign-off (in addition to other CI pipeline checks like tests and lint checks to pass). 5. Once approved, the PR can be completed using a squash merge in order to keep a nice linear history. +## Distributing + +You can also locally build and install from wheels like so: + +1. Build the *wheel* file(s) + + ```sh + make dist + ``` + +2. Install it. + + ```sh + # this will install just the optimizer component with SMAC support: + pip install "dist/tmp/mlos_core-latest-py3-none-any.whl[smac]" + ``` + + ```sh + # this will install both the optimizer and the experiment runner: + pip install "dist/mlos_bench-latest-py3-none-any.whl[azure]" + ``` + + > Note: exact versions may differ due to automatic versioning so the `-latest-` part is a symlink. + > If distributing elsewhere, adjust for the current version number in the module's `dist` directory. + ### See Also - diff --git a/Makefile b/Makefile index e0ab825d246..19894d670fa 100644 --- a/Makefile +++ b/Makefile @@ -232,6 +232,8 @@ mlos_bench/dist/tmp/mlos-bench-latest.tar: PACKAGE_NAME := mlos-bench # Check to make sure the mlos_bench module has the config directory. [ "$(MODULE_NAME)" != "mlos_bench" ] || unzip -t $(MODULE_NAME)/dist/$(MODULE_NAME)-*-py3-none-any.whl | grep -m1 mlos_bench/config/ cd $(MODULE_NAME)/dist/tmp && ln -s ../$(MODULE_NAME)-*-py3-none-any.whl $(MODULE_NAME)-latest-py3-none-any.whl + # Check to make sure the README contents made it into the package metadata. + unzip -p $(MODULE_NAME)/dist/tmp/$(MODULE_NAME)-latest-py3-none-any.whl */METADATA | egrep -v '^[A-Z][a-zA-z-]+:' | grep -q -i '^# mlos' .PHONY: dist-test-env-clean dist-test-env-clean: @@ -279,6 +281,27 @@ dist-test-clean: dist-test-env-clean rm -f build/dist-test-env.$(PYTHON_VERSION).build-stamp +.PHONY: publish +publish: publish-pypi + +.PHONY: +publish-pypi-deps: build/publish-pypi-deps.build-stamp + +build/publish-pypi-deps.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp + conda run -n ${CONDA_ENV_NAME} pip install -U twine + touch $@ + +build/publish.%.py.build-stamp: build/publish-pypi-deps.${CONDA_ENV_NAME}.build-stamp build/pytest.${CONDA_ENV_NAME}.build-stamp build/dist-test.$(PYTHON_VERSION).build-stamp build/check-doc.build-stamp build/linklint-doc.build-stamp + rm -f mlos_*/dist/*.tar.gz + ls mlos_*/dist/*.tar | xargs -I% gzip -k % + repo_name=`echo "$@" | sed -e 's|build/publish\.||' -e 's|\.py\.build-stamp||'` \ + && conda run -n ${CONDA_ENV_NAME} python3 -m twine upload --repository $$repo_name \ + mlos_*/dist/mlos*-*.tar.gz mlos_*/dist/mlos*-*.whl + touch $@ + +publish-pypi: build/publish.pypi.py.build-stamp +publish-test-pypi: build/publish.testpypi.py.build-stamp + build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp: build/conda-env.${CONDA_ENV_NAME}.build-stamp build/doc-prereqs.${CONDA_ENV_NAME}.build-stamp: doc/requirements.txt conda run -n ${CONDA_ENV_NAME} pip install -U -r doc/requirements.txt @@ -340,7 +363,10 @@ doc/build/html/index.html: $(SPHINX_API_RST_FILES) doc/Makefile doc/copy-source- # See check-doc .PHONY: doc -doc: doc/build/html/.nojekyll build/check-doc.build-stamp build/linklint-doc.build-stamp +doc: doc/build/html/.nojekyll doc-test + +.PHONY: doc-test +doc-test: build/check-doc.build-stamp build/linklint-doc.build-stamp doc/build/html/htmlcov/index.html: doc/build/html/index.html # Make the codecov html report available for the site. diff --git a/README.md b/README.md index 07920eeec46..fb1a1ebab33 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ MLOS is a project to enable autotuning for systems. - [Usage Examples](#usage-examples) - [mlos-core](#mlos-core) - [mlos-bench](#mlos-bench) - - [Distributing](#distributing) + - [Installation](#installation) - [See Also](#see-also) - [Examples](#examples) @@ -127,35 +127,30 @@ See Also: - [mlos_bench/config](./mlos_bench/mlos_bench/config/) for additional configuration details. - [sqlite-autotuning](https://github.com/Microsoft-CISL/sqlite-autotuning) for a complete external example of using MLOS to tune `sqlite`. -## Distributing +## Installation -MLOS is not [*yet*](https://github.com/microsoft/MLOS/issues/547) published on `pypi`, so until them here are some instructions for installation for usage in production or other environments. +The MLOS modules are published to [pypi](https://pypi.org) when new tags/releases are made. -1. Build the *wheel* file(s) +To install the latest release, simply run: - ```sh - make dist - ``` +```sh +# this will install just the optimizer component with SMAC support: +pip install -U mlos-core[smac] -2. Install it (e.g. after copying it somewhere else). - - ```sh - # this will install just the optimizer component with SMAC support: - pip install dist/mlos_core-0.1.0-py3-none-any.whl[smac] +# this will install just the optimizer component with flaml support: +pip install -U "mlos-core[flaml]" - # this will install just the optimizer component with flaml support: - pip install dist/mlos_core-0.1.0-py3-none-any.whl[flaml] +# this will install just the optimizer component with smac and flaml support: +pip install -U "mlos-core[smac,flaml]" - # this will install just the optimizer component with smac and flaml support: - pip install dist/mlos_core-0.1.0-py3-none-any.whl[smac,flaml] - ``` +# this will install both the flaml optimizer and the experiment runner with azure support: +pip install -U "mlos-bench[flaml,azure]" - ```sh - # this will install both the optimizer and the experiment runner: - pip install dist/mlos_bench-0.1.0-py3-none-any.whl - ``` +# this will install both the smac optimizer and the experiment runner with ssh support: +pip install -U "mlos-bench[smac,ssh]" +``` - > Note: exact versions may differ due to automatic versioning. +Details on using a local version from git are available in [CONTRIBUTING.md](./CONTRIBUTING.md). ## See Also diff --git a/conftest.py b/conftest.py index 2dc448d6f8f..e22395f82f0 100644 --- a/conftest.py +++ b/conftest.py @@ -33,12 +33,15 @@ def pytest_configure(config: pytest.Config) -> None: """ # Workaround some issues loading emukit in certain environments. if os.environ.get('DISPLAY', None): - import matplotlib # pylint: disable=import-outside-toplevel - matplotlib.rcParams['backend'] = 'agg' - if is_master(config) or dict(getattr(config, 'workerinput', {}))['workerid'] == 'gw0': - # Only warn once. - warn(UserWarning('DISPLAY environment variable is set, which can cause problems in some setups (e.g. WSL). ' - + f'Adjusting matplotlib backend to "{matplotlib.rcParams["backend"]}" to compensate.')) + try: + import matplotlib # pylint: disable=import-outside-toplevel + matplotlib.rcParams['backend'] = 'agg' + if is_master(config) or dict(getattr(config, 'workerinput', {}))['workerid'] == 'gw0': + # Only warn once. + warn(UserWarning('DISPLAY environment variable is set, which can cause problems in some setups (e.g. WSL). ' + + f'Adjusting matplotlib backend to "{matplotlib.rcParams["backend"]}" to compensate.')) + except ImportError: + pass # Create a temporary directory for sharing files between master and worker nodes. if is_master(config): @@ -72,8 +75,9 @@ def pytest_unconfigure(config: pytest.Config) -> None: Called after all tests have completed. """ if is_master(config): - shared_tmp_dir = str(getattr(config, "shared_temp_dir")) - shutil.rmtree(shared_tmp_dir) + shared_tmp_dir = getattr(config, "shared_temp_dir", None) + if shared_tmp_dir: + shutil.rmtree(str(shared_tmp_dir)) @pytest.fixture(scope="session") diff --git a/doc/source/conf.py b/doc/source/conf.py index 272da718014..fbbc2283c8a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -35,7 +35,7 @@ author = 'GSL' # The full version, including alpha/beta/rc tags -release = '0.1.0' +release = '0.2.2' try: from setuptools_scm import get_version diff --git a/mlos_bench/_version.py b/mlos_bench/_version.py index c89ae10815f..eb36ec54896 100644 --- a/mlos_bench/_version.py +++ b/mlos_bench/_version.py @@ -7,4 +7,4 @@ """ # NOTE: This should be managed by bumpversion. -_VERSION = '0.1.0' +_VERSION = '0.2.2' diff --git a/mlos_bench/setup.py b/mlos_bench/setup.py index d5b9d9f73db..27f065149f7 100644 --- a/mlos_bench/setup.py +++ b/mlos_bench/setup.py @@ -6,14 +6,46 @@ Setup instructions for the mlos_bench package. """ +# pylint: disable=duplicate-code + from logging import warning from itertools import chain from typing import Dict, List +import os +import re + from setuptools import setup, find_packages from _version import _VERSION # pylint: disable=import-private-name + +# A simple routine to read and adjust the README.md for this module into a format +# suitable for packaging. +# See Also: copy-source-tree-docs.sh +# Unfortunately we can't use that directly due to the way packaging happens inside a +# temp directory. +# Similarly, we can't use a utility script outside this module, so this code has to +# be duplicated for now. +def _get_long_desc_from_readme(base_url: str) -> dict: + pkg_dir = os.path.dirname(__file__) + readme_path = os.path.join(pkg_dir, 'README.md') + if not os.path.isfile(readme_path): + return {} + jsonc_re = re.compile(r'```jsonc') + link_re = re.compile(r'\]\(([^:#)]+)(#[a-zA-Z0-9_-]+)?\)') + with open(readme_path, mode='r', encoding='utf-8') as readme_fh: + lines = readme_fh.readlines() + # Tweak the lexers for local expansion by pygments instead of github's. + lines = [link_re.sub(f"]({base_url}" + r'/\1\2)', line) for line in lines] + # Tweak source source code links. + lines = [jsonc_re.sub(r'```json', line) for line in lines] + return { + 'long_description': ''.join(lines), + 'long_description_content_type': 'text/markdown', + } + + try: from setuptools_scm import get_version version = get_version(root='..', relative_to=__file__) @@ -85,10 +117,28 @@ ] + extra_requires['storage-sql-sqlite'], # NOTE: For now sqlite is a fallback storage backend, so we always install it. extras_require=extra_requires, author='Microsoft', + license='MIT', + **_get_long_desc_from_readme('https://github.com/microsoft/MLOS/tree/main/mlos_bench'), author_email='mlos-maintainers@service.microsoft.com', description=('MLOS Bench Python interface for benchmark automation and optimization.'), - license='MIT', - keywords='', - url='https://aka.ms/mlos-core', + url='https://github.com/microsoft/MLOS', + project_urls={ + 'Documentation': 'https://microsoft.github.io/MLOS', + 'Package Source': 'https://github.com/microsoft/MLOS/tree/main/mlos_bench/', + }, python_requires='>=3.8', + keywords=[ + 'autotuning', + 'benchmarking', + 'optimization', + 'systems', + ], + classifiers=[ + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + ], ) diff --git a/mlos_core/_version.py b/mlos_core/_version.py index c89ae10815f..eb36ec54896 100644 --- a/mlos_core/_version.py +++ b/mlos_core/_version.py @@ -7,4 +7,4 @@ """ # NOTE: This should be managed by bumpversion. -_VERSION = '0.1.0' +_VERSION = '0.2.2' diff --git a/mlos_core/setup.py b/mlos_core/setup.py index efab570d17e..2a0804baec7 100644 --- a/mlos_core/setup.py +++ b/mlos_core/setup.py @@ -6,10 +6,15 @@ Setup instructions for the mlos_core package. """ +# pylint: disable=duplicate-code + from itertools import chain from logging import warning from typing import Dict, List +import os +import re + from setuptools import setup, find_packages from _version import _VERSION # pylint: disable=import-private-name @@ -25,6 +30,34 @@ warning(f"setuptools_scm failed to find git version, using version from _version.py: {e}") +# A simple routine to read and adjust the README.md for this module into a format +# suitable for packaging. +# See Also: copy-source-tree-docs.sh +# Unfortunately we can't use that directly due to the way packaging happens inside a +# temp directory. +# Similarly, we can't use a utility script outside this module, so this code has to +# be duplicated for now. +# Also, to avoid caching issues when calculating dependencies for the devcontainer, +# we return nothing when the file is not available. +def _get_long_desc_from_readme(base_url: str) -> dict: + pkg_dir = os.path.dirname(__file__) + readme_path = os.path.join(pkg_dir, 'README.md') + if not os.path.isfile(readme_path): + return {} + jsonc_re = re.compile(r'```jsonc') + link_re = re.compile(r'\]\(([^:#)]+)(#[a-zA-Z0-9_-]+)?\)') + with open(readme_path, mode='r', encoding='utf-8') as readme_fh: + lines = readme_fh.readlines() + # Tweak the lexers for local expansion by pygments instead of github's. + lines = [link_re.sub(f"]({base_url}" + r'/\1\2)', line) for line in lines] + # Tweak source source code links. + lines = [jsonc_re.sub(r'```json', line) for line in lines] + return { + 'long_description': ''.join(lines), + 'long_description_content_type': 'text/markdown', + } + + extra_requires: Dict[str, List[str]] = { # pylint: disable=consider-using-namedtuple-or-dataclass 'flaml': ['flaml[blendsearch]'], 'smac': ['smac>=2.0.0'], # NOTE: Major refactoring on SMAC starting from v2.0.0 @@ -62,9 +95,25 @@ extras_require=extra_requires, author='Microsoft', author_email='mlos-maintainers@service.microsoft.com', - description=('MLOS Core Python interface for parameter optimization.'), license='MIT', - keywords='', - url='https://aka.ms/mlos-core', + **_get_long_desc_from_readme('https://github.com/microsoft/MLOS/tree/main/mlos_core'), + description=('MLOS Core Python interface for parameter optimization.'), + url='https://github.com/microsoft/MLOS', + project_urls={ + 'Documentation': 'https://microsoft.github.io/MLOS', + 'Package Source': 'https://github.com/microsoft/MLOS/tree/main/mlos_core/', + }, python_requires='>=3.8', + keywords=[ + 'autotuning', + 'optimization', + ], + classifiers=[ + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + ], ) diff --git a/scripts/update-version.sh b/scripts/update-version.sh index b34f84158ca..c5b61496afb 100755 --- a/scripts/update-version.sh +++ b/scripts/update-version.sh @@ -14,4 +14,5 @@ cd "$scriptdir/.." set -x # Example usage: "./update-version.sh --dry-run patch" to bump v0.0.4 -> v0.0.5, for instance. +# Example usage: "./update-version.sh --dry-run minor" to bump v0.0.4 -> v0.1.0, for instance. conda run -n ${CONDA_ENV_NAME:-mlos} bumpversion --verbose $*