From 98b5098253ac822404fd2a386285e46a60df65e4 Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Tue, 15 Aug 2023 16:48:28 +0200 Subject: [PATCH 01/15] Add bootstrap script to template --- .github/workflows/main.yml | 106 ++++++++++++------ README.md | 20 +++- copier.yml | 18 ++- template/README.md.jinja | 8 +- template/bootstrap.py.jinja | 91 +++++++++++++++ template/scripts/pre-commit | 5 + ...'conda' %}environment.yml{% endif %}.jinja | 11 ++ 7 files changed, 209 insertions(+), 50 deletions(-) create mode 100755 template/bootstrap.py.jinja create mode 100755 template/scripts/pre-commit create mode 100644 template/{% if venv == 'conda' %}environment.yml{% endif %}.jinja diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7154636..8b70c93 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,13 +45,11 @@ jobs: run: | set -eux mkdir ${NAME} - python -m copier copy -l -d author_name="My Name" -d labextension_name="${NAME}" -d python_name="${PYNAME}" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . ${NAME} + python -m copier copy -l -d author_name="My Name" -d labextension_name="${NAME}" -d python_name="${PYNAME}" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . ${NAME} pushd ${NAME} python -m pip install "jupyterlab>=4.0.0,<5" - YARN_ENABLE_IMMUTABLE_INSTALLS=false jlpm - jlpm lint:check - python -m pip install -e . - jupyter labextension develop . --overwrite + YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py + ./scripts/pre-commit jupyter labextension list jupyter labextension list 2>&1 | grep -ie "${NAME}.*OK" python -m jupyterlab.browser_check @@ -89,13 +87,12 @@ jobs: run: | set -eux mkdir myextension - python -m copier copy -l -d author_name="My Name" -d test=n -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . myextension + python -m copier copy -l -d author_name="My Name" -d test=n -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension pushd myextension + YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py pip install "jupyterlab>=4.0.0,<5" - YARN_ENABLE_IMMUTABLE_INSTALLS=false jlpm - jlpm lint:check - pip install -e . - jupyter labextension develop . --overwrite + ./scripts/pre-commit + jupyter labextension list jupyter labextension list 2>&1 | grep -ie "myextension.*OK" python -m jupyterlab.browser_check @@ -133,15 +130,14 @@ jobs: run: | set -eux mkdir myextension - python -m copier copy -l -d author_name="My Name" -d has_settings=y -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . myextension + python -m copier copy -l -d author_name="My Name" -d has_settings=y -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension pushd myextension pip install "jupyterlab>=4.0.0,<5" - YARN_ENABLE_IMMUTABLE_INSTALLS=false jlpm + YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py # It is not easily possible to get this version compatible with linter rules jlpm lint - jlpm lint:check - pip install -e . - jupyter labextension develop . --overwrite + ./scripts/pre-commit + jupyter labextension list jupyter labextension list 2>&1 | grep -ie "myextension.*OK" python -m jupyterlab.browser_check @@ -182,13 +178,11 @@ jobs: YARN_ENABLE_IMMUTABLE_INSTALLS: false run: | mkdir myextension - python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . myextension + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension cd myextension cat pyproject.toml pip install . pip install "jupyterlab>=4.0.0,<5" - jlpm - jlpm lint:check - name: Check pip install method run: | @@ -210,12 +204,10 @@ jobs: YARN_ENABLE_IMMUTABLE_INSTALLS: false run: | mkdir myextension - python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . myextension + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension cd myextension - python -m pip install -e .[test] + python bootstrap.py python -m pip install "jupyterlab>=4.0.0,<5" - jupyter labextension develop . --overwrite - jupyter server extension enable myextension # Check unit tests are passing python -m pytest -vv -r ap --cov myextension @@ -232,9 +224,14 @@ jobs: python -m jupyterlab.browser_check shell: bash + - name: Check rerunning bootstrap.py + run: | + set -eux + python bootstrap.py + shell: bash + - name: Build server extension in develop mode run: | - jupyter labextension develop ./myextension --overwrite jupyter labextension build ./myextension jupyter labextension uninstall myextension @@ -250,7 +247,7 @@ jobs: YARN_ENABLE_IMMUTABLE_INSTALLS: false run: | mkdir myextension - python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . myextension + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension cd myextension python -m pip install "jupyterlab>=4.0.0,<5" jupyter lab clean --all @@ -351,13 +348,12 @@ jobs: run: | set -eux mkdir mytheme - python -m copier copy -l -d kind=theme -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . mytheme + python -m copier copy -l -d kind=theme -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . mytheme pushd mytheme python -m pip install "jupyterlab>=4.0.0,<5" - YARN_ENABLE_IMMUTABLE_INSTALLS=false jlpm - jlpm lint:check - python -m pip install -e . - jupyter labextension develop . --overwrite + YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py + ./scripts/pre-commit + jupyter labextension list jupyter labextension list 2>&1 | grep -ie "mytheme.*OK" python -m jupyterlab.browser_check @@ -395,13 +391,13 @@ jobs: run: | set -eux mkdir myextension - python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . myextension + python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension pushd myextension + YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py python -m pip install "jupyterlab>=4.0.0,<5" - YARN_ENABLE_IMMUTABLE_INSTALLS=false jlpm - jlpm lint:check - python -m pip install -e . - jupyter labextension develop . --overwrite + + ./scripts/pre-commit + jupyter labextension list jupyter labextension list 2>&1 | grep -ie "myextension.*OK" python -m jupyterlab.browser_check @@ -412,6 +408,48 @@ jobs: popd rm -rf myextension + conda_boostrap: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-activate-base: true + activate-environment: "" + + - name: Install dependencies + run: | + python -m pip install "copier>=8.0.0" jinja2-time "pydantic<2.0.0" + + - name: Setup Git + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Create server extension with conda + run: | + mkdir myextension + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . myextension + + - name: Check conda environment + run: | + set -eux + conda env list | grep -ie "myextension_dev" + conda activate myextension_dev + jupyter server extension list + jupyter server extension list 2>&1 | grep -ie "myextension.*OK" + jupyter labextension list + jupyter labextension list 2>&1 | grep -ie "myextension.*OK" + + # This test should be made outside the extension folder + python -m jupyterlab.browser_check + + pip uninstall -y myextension jupyterlab + rm -rf myextension + shell: bash + pnpm_linker: runs-on: ubuntu-latest strategy: diff --git a/README.md b/README.md index 7ab49cf..4fdc736 100644 --- a/README.md +++ b/README.md @@ -34,16 +34,13 @@ cd myextension 3. Use copier to generate an extension, following the prompts to fill all required information. -``` +```sh copier copy https://github.com/jupyterlab/extension-template . ``` > If you use copier v8+, you will need to pass the flag `--UNSAFE` (see [documentation](https://copier.readthedocs.io/en/stable/configuring/#unsafe)). -> If you are using Visual Studio Code, you may be interested in the -> [configuration template](https://github.com/jupyterlab/vscode-config-template) for JupyterLab extension. - ---- +
How to use a template specific version If you'd like to generate an extension for a older release, use the `--vcs-ref` option and give a tag or commit from this repository. @@ -53,6 +50,17 @@ copier --vcs-ref v4.0.0 copy https://github.com/jupyterlab/extension-template . > If you use copier v8+, you will need to pass the flag `--UNSAFE` (see [documentation](https://copier.readthedocs.io/en/stable/configuring/#unsafe)). +--- + +
+ +4. What's next + +At this point, you repository has been [initialize with git](https://www.atlassian.com/git/tutorials/setting-up-a-repository) as code versioning tool and a development environment has been set up if you picked `conda` or `venv`. You should be all set. Open your preferred code editor and start coding. + +> If you are using Visual Studio Code, you may be interested in the +> [configuration template](https://github.com/jupyterlab/vscode-config-template) for JupyterLab extension. + > If you are looking for a template compatible with JupyterLab version prior to 4.0.0, look at > the [cookiecutter template](https://github.com/jupyterlab/extension-cookiecutter-ts) or the > [mimerenderer template](https://github.com/jupyterlab/mimerender-cookiecutter-ts). @@ -61,7 +69,7 @@ copier --vcs-ref v4.0.0 copy https://github.com/jupyterlab/extension-template . > This only works with an older version of the _copier_ template. It does not work > with an extension generated using the cookiecutter template. In that case, you -> could try the script `python -m jupyterlab.upgrade_extension`. +> could try to execute the script `python -m jupyterlab.upgrade_extension .` first. Extension generated from the copier template can be [updated](https://copier.readthedocs.io/en/stable/updating/) with a newer version of the template by executing the command: diff --git a/copier.yml b/copier.yml index 6cb54ca..4fc96d8 100644 --- a/copier.yml +++ b/copier.yml @@ -2,16 +2,18 @@ _min_copier_version: "7.1.0" _subdirectory: template _jinja_extensions: - jinja2_time.TimeExtension +_tasks: + - ["{{ _copier_python }}", bootstrap.py] kind: type: str help: What is your extension kind? default: frontend choices: - - frontend - - mimerenderer - - server - - theme + Pure frontend: frontend + Mimetype renderer: mimerenderer + Frontend and backend: server + Theme: theme author_name: type: str @@ -64,6 +66,14 @@ repository: help: Git remote repository URL placeholder: https://github.com/github_username/my-extension +venv: + type: str + help: What tool are you using to handle virtual environments? + default: conda + choices: + Conda environment (https://docs.conda.io/projects/conda/en/latest/index.html): "conda" + None (nothing set up): "none" + viewer_name: when: "{{ kind == 'mimerenderer' }}" type: str diff --git a/template/README.md.jinja b/template/README.md.jinja index a199eed..4da5e40 100644 --- a/template/README.md.jinja +++ b/template/README.md.jinja @@ -60,12 +60,8 @@ The `jlpm` command is JupyterLab's pinned version of ```bash # Clone the repo to your local environment # Change directory to the {{ python_name }} directory -# Install package in development mode -pip install -e ".{% if test and kind.lower() == 'server' %}[test]{% endif %}" -# Link your development version of the extension with JupyterLab -jupyter labextension develop . --overwrite{% if kind.lower() == 'server' %} -# Server extension must be manually installed in develop mode -jupyter server extension enable {{ python_name }}{% endif %} +python bootstrap.py + # Rebuild extension Typescript source after making changes jlpm build ``` diff --git a/template/bootstrap.py.jinja b/template/bootstrap.py.jinja new file mode 100755 index 0000000..7d9bebf --- /dev/null +++ b/template/bootstrap.py.jinja @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# This script initialize your repository and development environment. +# You can re-execute it at any time + +import json +import os +from pathlib import Path +from shutil import copy, which +from subprocess import CalledProcessError, PIPE, check_call, check_output, run + + +NOT_GIT_REPO = "fatal: not a git repository (or any of the parent directories): .git" +ENVIRONMENT_NAME = os.environ.get("CONDA_ENVIRONMENT_NAME", "{{ python_name }}_dev") +ENVIRONMENT_FILE = os.environ.get("CONDA_ENVIRONMENT_FILE", "environment.yml") + + +def initialize_git_repository(): + """Initialize git repository""" + git_status = run(["git", "status"], stdout=PIPE, stderr=PIPE, encoding="utf-8") + if git_status.returncode > 0 and git_status.stderr.strip() == NOT_GIT_REPO: + print("\nInitializing git...") + try: + check_call(["git", "init"]) + check_call(["git", "branch", "-m", "main"]) + except CalledProcessError as e: + print(f"Failed to initialize git repository:\n{e!s}\n") + else: + print("Git initialized for the extension\n") + + # Copy pre-commit hook + copy("scripts/pre-commit", ".git/hooks") + +{% if venv == "conda" %} +def create_conda_environment(): + mamba = which("mamba") + conda_exe = "mamba" if mamba is not None else "conda" + + print("\nCreating conda environment...") + try: + output = check_output([conda_exe, "env", "list", "--json"], encoding="utf-8") + envs = json.loads(output)["envs"] + # if environment exists + command = "update" if conda_exe == "mamba" or any(map(lambda e: Path(e).name == ENVIRONMENT_NAME, envs)) else "create" + check_call([conda_exe, "env", command, "--name", ENVIRONMENT_NAME, "--file", ENVIRONMENT_FILE]) + except CalledProcessError as e: + print(f"Fail to create development environment using '{conda_exe}':\n{e!s}\n") + else: + print(f"Conda environment '{ENVIRONMENT_NAME}' {command}d. Execute '{conda_exe} activate {ENVIRONMENT_NAME}' to use it.\n") + +{% endif %} +def install_extension():{% if venv == "conda" %} + mamba = which("mamba") + conda_exe = "mamba" if mamba is not None else "conda" + pre_command = [conda_exe, "run", "--name", ENVIRONMENT_NAME] +{% else %} + pre_command = [] + # Checking nodeJS is available + try: + output = check_output(["node", "--version"], encoding="utf-8") + except CalledProcessError as e: + print("You must install nodeJS LTS to develop Jupyter extension; see https://nodejs.org") + return + + if "CI" not in os.environ: + # Don't prompt on CI + answer = input("Do you want to install the extension in development mode?\ny/N: ") + if not answer.lower().startswith('y'): + return +{% endif %} + print("\nInstalling the extension in development mode...") + + # Force uninstalling the extension to avoid trouble when installing in dev mode + run(pre_command + ["pip", "uninstall", "--yes", "{{ python_name }}"], stdout=PIPE, stderr=PIPE) + + try: + # Install package in development mode + check_output(pre_command + ["pip", "install", "-e", ".{% if test and kind.lower() == 'server' %}[test]{% endif %}"]) + # Link your development version of the extension with JupyterLab + check_output(pre_command + ["jupyter", "labextension", "develop", ".", "--overwrite"]){% if kind.lower() == 'server' %} + # Server extension must be manually installed in develop mode + check_output(pre_command + ["jupyter", "server", "extension", "enable", "{{ python_name }}"]){% endif %} + except CalledProcessError as e: + print(f"Fail to install the extension in development mode:\n{e!s}\n") + else: + print("Extension installed in development mode.\n") + + +if __name__ == "__main__": + initialize_git_repository(){% if venv == "conda" %} + create_conda_environment(){% endif %} + install_extension() diff --git a/template/scripts/pre-commit b/template/scripts/pre-commit new file mode 100755 index 0000000..59da0fe --- /dev/null +++ b/template/scripts/pre-commit @@ -0,0 +1,5 @@ +#!/bin/sh +# This script should be installed as pre-commit git hook (see https://www.atlassian.com/git/tutorials/git-hooks) + +# Check frontend files are linted +jlpm lint:check diff --git a/template/{% if venv == 'conda' %}environment.yml{% endif %}.jinja b/template/{% if venv == 'conda' %}environment.yml{% endif %}.jinja new file mode 100644 index 0000000..561894b --- /dev/null +++ b/template/{% if venv == 'conda' %}environment.yml{% endif %}.jinja @@ -0,0 +1,11 @@ +# Extension development environment +channels: + - conda-forge + +dependencies: + # runtime dependencies + - python + - jupyterlab >=4.0.0,<5 + # labextension build dependencies + - nodejs >=18,<19 + - pip From 58875092bfb76b3c996649552cfd8d7b811d282d Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Tue, 15 Aug 2023 17:18:50 +0200 Subject: [PATCH 02/15] Create extension in temporary folder --- .github/workflows/main.yml | 74 +++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8b70c93..079c736 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -86,9 +86,10 @@ jobs: - name: Create pure frontend extension run: | set -eux - mkdir myextension + tmpfolder=$(mktemp -d) + mkdir $tmpfolder python -m copier copy -l -d author_name="My Name" -d test=n -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension - pushd myextension + pushd $tmpfolder YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py pip install "jupyterlab>=4.0.0,<5" ./scripts/pre-commit @@ -101,7 +102,7 @@ jobs: pip uninstall -y myextension jupyterlab popd - rm -rf myextension + rm -rf $tmpfolder settings: runs-on: ubuntu-latest @@ -129,9 +130,10 @@ jobs: - name: Create pure frontend extension run: | set -eux - mkdir myextension - python -m copier copy -l -d author_name="My Name" -d has_settings=y -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension - pushd myextension + tmpfolder=$(mktemp -d) + mkdir $tmpfolder + python -m copier copy -l -d author_name="My Name" -d has_settings=y -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder + pushd $tmpfolder pip install "jupyterlab>=4.0.0,<5" YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py # It is not easily possible to get this version compatible with linter rules @@ -146,7 +148,7 @@ jobs: pip uninstall -y myextension jupyterlab popd - rm -rf myextension + rm -rf $tmpfolder server: runs-on: ${{ matrix.os }} @@ -177,9 +179,10 @@ jobs: env: YARN_ENABLE_IMMUTABLE_INSTALLS: false run: | - mkdir myextension - python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension - cd myextension + tmpfolder=$(mktemp -d) + mkdir $tmpfolder + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder + cd $tmpfolder cat pyproject.toml pip install . pip install "jupyterlab>=4.0.0,<5" @@ -196,16 +199,17 @@ jobs: python -m jupyterlab.browser_check pip uninstall -y myextension jupyterlab - rm -rf myextension + rm -rf $tmpfolder shell: bash - name: Create server extension pip develop env: YARN_ENABLE_IMMUTABLE_INSTALLS: false run: | - mkdir myextension - python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension - cd myextension + tmpfolder=$(mktemp -d) + mkdir $tmpfolder + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder + cd $tmpfolder python bootstrap.py python -m pip install "jupyterlab>=4.0.0,<5" @@ -232,23 +236,24 @@ jobs: - name: Build server extension in develop mode run: | - jupyter labextension build ./myextension + jupyter labextension build $tmpfolder jupyter labextension uninstall myextension python -m pip uninstall -y myextension jupyterlab - run: | set -eux - rm -rf myextension + rm -rf $tmpfolder shell: bash - name: Install server extension from a tarball env: YARN_ENABLE_IMMUTABLE_INSTALLS: false run: | - mkdir myextension - python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension - cd myextension + tmpfolder=$(mktemp -d) + mkdir $tmpfolder + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder + cd $tmpfolder python -m pip install "jupyterlab>=4.0.0,<5" jupyter lab clean --all python -m build @@ -267,10 +272,10 @@ jobs: python -m jupyterlab.browser_check - cp myextension/dist/*.tar.gz myextension.tar.gz - cp myextension/dist/*.whl myextension.whl + cp $tmpfolder/dist/*.tar.gz myextension.tar.gz + cp $tmpfolder/dist/*.whl myextension.whl python -m pip uninstall -y myextension jupyterlab - rm -rf myextension + rm -rf $tmpfolder shell: bash - uses: actions/upload-artifact@v2 @@ -390,9 +395,10 @@ jobs: - name: Create pure frontend extension run: | set -eux - mkdir myextension - python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension - pushd myextension + tmpfolder=$(mktemp -d) + mkdir $tmpfolder + python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder + pushd $tmpfolder YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py python -m pip install "jupyterlab>=4.0.0,<5" @@ -406,7 +412,7 @@ jobs: python -m pip uninstall -y myextension jupyterlab popd - rm -rf myextension + rm -rf $tmpfolder conda_boostrap: runs-on: ubuntu-latest @@ -430,8 +436,9 @@ jobs: - name: Create server extension with conda run: | - mkdir myextension - python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . myextension + tmpfolder=$(mktemp -d) + mkdir $tmpfolder + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . $tmpfolder - name: Check conda environment run: | @@ -447,7 +454,7 @@ jobs: python -m jupyterlab.browser_check pip uninstall -y myextension jupyterlab - rm -rf myextension + rm -rf $tmpfolder shell: bash pnpm_linker: @@ -477,13 +484,14 @@ jobs: KIND: ${{ matrix.kind }} run: | set -eux - mkdir myextension - python -m copier copy -l -d kind=${KIND} -d author_name="My Name" -d labextension_name=myextension -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . myextension - pushd myextension + tmpfolder=$(mktemp -d) + mkdir $tmpfolder + python -m copier copy -l -d kind=${KIND} -d author_name="My Name" -d labextension_name=myextension -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . $tmpfolder + pushd $tmpfolder sed -i 's/^\(nodeLinker:\s\).*$/\1pnpm/' .yarnrc.yml python -m pip install "jupyterlab>=4.0.0,<5" YARN_ENABLE_IMMUTABLE_INSTALLS=false jlpm if [ ! -d node_modules/.store ] ; then echo 'nodes_module directory should contain a .store directory when using pnpm nodeLinker'; exit 1; fi; jlpm build popd - rm -rf myextension + rm -rf $tmpfolder From 5882cb7d8af76627afdd65de25bd5fa454a7a1a3 Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Tue, 15 Aug 2023 17:22:44 +0200 Subject: [PATCH 03/15] Don't remake the temp folder --- .github/workflows/main.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 079c736..bfe5bcd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,9 +44,10 @@ jobs: PYNAME: ${{ matrix.pyname }} run: | set -eux - mkdir ${NAME} - python -m copier copy -l -d author_name="My Name" -d labextension_name="${NAME}" -d python_name="${PYNAME}" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . ${NAME} - pushd ${NAME} + tmpfolder=$(mktemp -d) + + python -m copier copy -l -d author_name="My Name" -d labextension_name="${NAME}" -d python_name="${PYNAME}" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder + pushd $tmpfolder python -m pip install "jupyterlab>=4.0.0,<5" YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py ./scripts/pre-commit @@ -58,7 +59,7 @@ jobs: python -m pip uninstall -y ${NAME} jupyterlab popd - rm -rf ${NAME} + rm -rf $tmpfolder no-tests: runs-on: ubuntu-latest @@ -87,7 +88,7 @@ jobs: run: | set -eux tmpfolder=$(mktemp -d) - mkdir $tmpfolder + python -m copier copy -l -d author_name="My Name" -d test=n -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension pushd $tmpfolder YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py @@ -131,7 +132,7 @@ jobs: run: | set -eux tmpfolder=$(mktemp -d) - mkdir $tmpfolder + python -m copier copy -l -d author_name="My Name" -d has_settings=y -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder pushd $tmpfolder pip install "jupyterlab>=4.0.0,<5" @@ -180,7 +181,7 @@ jobs: YARN_ENABLE_IMMUTABLE_INSTALLS: false run: | tmpfolder=$(mktemp -d) - mkdir $tmpfolder + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder cd $tmpfolder cat pyproject.toml @@ -207,7 +208,7 @@ jobs: YARN_ENABLE_IMMUTABLE_INSTALLS: false run: | tmpfolder=$(mktemp -d) - mkdir $tmpfolder + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder cd $tmpfolder python bootstrap.py @@ -251,7 +252,7 @@ jobs: YARN_ENABLE_IMMUTABLE_INSTALLS: false run: | tmpfolder=$(mktemp -d) - mkdir $tmpfolder + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder cd $tmpfolder python -m pip install "jupyterlab>=4.0.0,<5" @@ -396,7 +397,7 @@ jobs: run: | set -eux tmpfolder=$(mktemp -d) - mkdir $tmpfolder + python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder pushd $tmpfolder YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py @@ -437,7 +438,7 @@ jobs: - name: Create server extension with conda run: | tmpfolder=$(mktemp -d) - mkdir $tmpfolder + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . $tmpfolder - name: Check conda environment @@ -485,7 +486,7 @@ jobs: run: | set -eux tmpfolder=$(mktemp -d) - mkdir $tmpfolder + python -m copier copy -l -d kind=${KIND} -d author_name="My Name" -d labextension_name=myextension -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . $tmpfolder pushd $tmpfolder sed -i 's/^\(nodeLinker:\s\).*$/\1pnpm/' .yarnrc.yml From ab99ea7eca8eb9962e05c3852a97afb0635b6434 Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Tue, 15 Aug 2023 17:43:11 +0200 Subject: [PATCH 04/15] Add jupyterlab as dev extra --- .github/workflows/main.yml | 6 ------ template/bootstrap.py.jinja | 2 +- template/pyproject.toml.jinja | 9 ++++++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bfe5bcd..bf9e186 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,7 +48,6 @@ jobs: python -m copier copy -l -d author_name="My Name" -d labextension_name="${NAME}" -d python_name="${PYNAME}" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder pushd $tmpfolder - python -m pip install "jupyterlab>=4.0.0,<5" YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py ./scripts/pre-commit jupyter labextension list @@ -92,7 +91,6 @@ jobs: python -m copier copy -l -d author_name="My Name" -d test=n -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension pushd $tmpfolder YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py - pip install "jupyterlab>=4.0.0,<5" ./scripts/pre-commit jupyter labextension list @@ -135,7 +133,6 @@ jobs: python -m copier copy -l -d author_name="My Name" -d has_settings=y -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder pushd $tmpfolder - pip install "jupyterlab>=4.0.0,<5" YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py # It is not easily possible to get this version compatible with linter rules jlpm lint @@ -212,7 +209,6 @@ jobs: python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder cd $tmpfolder python bootstrap.py - python -m pip install "jupyterlab>=4.0.0,<5" # Check unit tests are passing python -m pytest -vv -r ap --cov myextension @@ -356,7 +352,6 @@ jobs: mkdir mytheme python -m copier copy -l -d kind=theme -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . mytheme pushd mytheme - python -m pip install "jupyterlab>=4.0.0,<5" YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py ./scripts/pre-commit @@ -401,7 +396,6 @@ jobs: python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder pushd $tmpfolder YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py - python -m pip install "jupyterlab>=4.0.0,<5" ./scripts/pre-commit diff --git a/template/bootstrap.py.jinja b/template/bootstrap.py.jinja index 7d9bebf..3384035 100755 --- a/template/bootstrap.py.jinja +++ b/template/bootstrap.py.jinja @@ -74,7 +74,7 @@ def install_extension():{% if venv == "conda" %} try: # Install package in development mode - check_output(pre_command + ["pip", "install", "-e", ".{% if test and kind.lower() == 'server' %}[test]{% endif %}"]) + check_output(pre_command + ["pip", "install", "-e", ".[dev{% if test and kind.lower() == 'server' %}, test{% endif %}]"]) # Link your development version of the extension with JupyterLab check_output(pre_command + ["jupyter", "labextension", "develop", ".", "--overwrite"]){% if kind.lower() == 'server' %} # Server extension must be manually installed in develop mode diff --git a/template/pyproject.toml.jinja b/template/pyproject.toml.jinja index 1891b05..faa4ff4 100644 --- a/template/pyproject.toml.jinja +++ b/template/pyproject.toml.jinja @@ -25,16 +25,19 @@ dependencies = [{% if kind.lower() == "server" %} "jupyter_server>=2.0.1,<3"{% endif %} ] dynamic = ["version", "description", "authors", "urls", "keywords"] -{% if test and kind.lower() == 'server' %} + [project.optional-dependencies] +dev = [ + "jupyterlab>=4.0.0,<5" +]{% if test and kind.lower() == 'server' %} test = [ "coverage", "pytest", "pytest-asyncio", "pytest-cov", "pytest-jupyter[server]>=0.6.0" -] -{% endif %} +]{% endif %} + [tool.hatch.version] source = "nodejs" From 77de15f04df046b555fc5ae107bdd5dd148d548a Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Tue, 15 Aug 2023 17:55:20 +0200 Subject: [PATCH 05/15] More CI fixes --- .github/workflows/check-tests.yml | 44 +++++++++++++++++-------------- .github/workflows/main.yml | 2 +- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/.github/workflows/check-tests.yml b/.github/workflows/check-tests.yml index 7da16ca..5fc0671 100644 --- a/.github/workflows/check-tests.yml +++ b/.github/workflows/check-tests.yml @@ -23,21 +23,23 @@ jobs: run: python -m pip install -U "copier>=8.0.0" jinja2-time "jupyterlab>=4.0.0,<5" "pydantic<2.0.0" - name: Create the extension + id: create-extension run: | set -eux - mkdir myextension - python -m copier copy -l -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . myextension - cat myextension/pyproject.toml + tmpfolder=$(mktemp -d) + echo "extension_folder=${tmpfolder}" >> "$GITHUB_OUTPUT" + python -m copier copy -l -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder + cat $tmpfolder/pyproject.toml - name: Test the extension - working-directory: myextension + working-directory: ${{ steps.create-extension.outputs.extension_folder }} run: | set -eux YARN_ENABLE_IMMUTABLE_INSTALLS=false jlpm jlpm test - name: Install the extension - working-directory: myextension + working-directory: ${{ steps.create-extension.outputs.extension_folder }} run: | set -eux python -m pip install -v . @@ -48,7 +50,7 @@ jobs: jupyter labextension list - name: Install UI tests dependencies - working-directory: myextension/ui-tests + working-directory: ${{ steps.create-extension.outputs.extension_folder }}/ui-tests env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 YARN_ENABLE_IMMUTABLE_INSTALLS: false @@ -59,14 +61,14 @@ jobs: with: path: | ${{ github.workspace }}/pw-browsers - key: ${{ runner.os }}-${{ hashFiles('myextension/ui-tests/yarn.lock') }} + key: ${{ runner.os }}-${{ hashFiles('${{ steps.create-extension.outputs.extension_folder }}/ui-tests/yarn.lock') }} - name: Install browser run: jlpm playwright install chromium - working-directory: myextension/ui-tests + working-directory: ${{ steps.create-extension.outputs.extension_folder }}/ui-tests - name: Execute integration tests - working-directory: myextension/ui-tests + working-directory: ${{ steps.create-extension.outputs.extension_folder }}/ui-tests run: | jlpm playwright test @@ -76,8 +78,8 @@ jobs: with: name: extension-playwright-tests path: | - myextension/ui-tests/test-results - myextension/ui-tests/playwright-report + ${{ steps.create-extension.outputs.extension_folder }}/ui-tests/test-results + ${{ steps.create-extension.outputs.extension_folder }}/ui-tests/playwright-report test-mimerenderer: runs-on: ubuntu-latest @@ -92,13 +94,15 @@ jobs: run: python -m pip install -U "copier>=8.0.0" jinja2-time "jupyterlab>=4.0.0,<5" "pydantic<2.0.0" - name: Create the extension + id: create-extension run: | set -eux - mkdir myextension - python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . myextension + tmpfolder=$(mktemp -d) + python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension + echo "extension_folder=${tmpfolder}" >> "$GITHUB_OUTPUT" - name: Install the extension - working-directory: myextension + working-directory: ${{ steps.create-extension.outputs.extension_folder }} run: | set -eux YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m pip install -v . @@ -108,7 +112,7 @@ jobs: jupyter labextension list - name: Install UI tests dependencies - working-directory: myextension/ui-tests + working-directory: ${{ steps.create-extension.outputs.extension_folder }}/ui-tests env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 YARN_ENABLE_IMMUTABLE_INSTALLS: false @@ -119,14 +123,14 @@ jobs: with: path: | ${{ github.workspace }}/pw-browsers - key: ${{ runner.os }}-${{ hashFiles('myextension/ui-tests/yarn.lock') }} + key: ${{ runner.os }}-${{ hashFiles('${{ steps.create-extension.outputs.extension_folder }}/ui-tests/yarn.lock') }} - name: Install browser run: jlpm playwright install chromium - working-directory: myextension/ui-tests + working-directory: ${{ steps.create-extension.outputs.extension_folder }}/ui-tests - name: Execute integration tests - working-directory: myextension/ui-tests + working-directory: ${{ steps.create-extension.outputs.extension_folder }}/ui-tests run: | # Generate reference snapshot first jlpm playwright test -u @@ -138,5 +142,5 @@ jobs: with: name: mimerenderer-playwright-tests path: | - myextension/ui-tests/test-results - myextension/ui-tests/playwright-report \ No newline at end of file + ${{ steps.create-extension.outputs.extension_folder }}/ui-tests/test-results + ${{ steps.create-extension.outputs.extension_folder }}/ui-tests/playwright-report \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bf9e186..5f912ce 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -253,7 +253,7 @@ jobs: cd $tmpfolder python -m pip install "jupyterlab>=4.0.0,<5" jupyter lab clean --all - python -m build + python -m build $tmpfolder cd dist python -m pip install myextension-0.1.0.tar.gz From 64cc22539149517c851b59ed96a42cf89bc39576 Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Sun, 20 Aug 2023 12:33:39 +0200 Subject: [PATCH 06/15] Improve bootstrap script --- template/bootstrap.py.jinja | 26 ++++++++++++++++++++------ template/scripts/pre-commit | 4 +++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/template/bootstrap.py.jinja b/template/bootstrap.py.jinja index 3384035..4551f0f 100755 --- a/template/bootstrap.py.jinja +++ b/template/bootstrap.py.jinja @@ -5,17 +5,23 @@ import json import os from pathlib import Path -from shutil import copy, which +from shutil import which from subprocess import CalledProcessError, PIPE, check_call, check_output, run +# Git error looked for to determined if the extension folder is already using Git NOT_GIT_REPO = "fatal: not a git repository (or any of the parent directories): .git" +# Conda development environment name; defined by the environment variable CONDA_ENVIRONMENT_NAME and default to the 'python package name_dev' ENVIRONMENT_NAME = os.environ.get("CONDA_ENVIRONMENT_NAME", "{{ python_name }}_dev") +# Conda development environment specification; defined the environment variable CONDA_ENVIRONMENT_FILE and default to 'environment.yml' ENVIRONMENT_FILE = os.environ.get("CONDA_ENVIRONMENT_FILE", "environment.yml") def initialize_git_repository(): - """Initialize git repository""" + """Initialize git repository. + + The git repository is initialized with the default branch 'main' + """ git_status = run(["git", "status"], stdout=PIPE, stderr=PIPE, encoding="utf-8") if git_status.returncode > 0 and git_status.stderr.strip() == NOT_GIT_REPO: print("\nInitializing git...") @@ -25,13 +31,18 @@ def initialize_git_repository(): except CalledProcessError as e: print(f"Failed to initialize git repository:\n{e!s}\n") else: - print("Git initialized for the extension\n") - - # Copy pre-commit hook - copy("scripts/pre-commit", ".git/hooks") + print("Git initialized with default branch 'main' for the extension\n") {% if venv == "conda" %} def create_conda_environment(): + """Create a development Conda environment. + + The default environment name is defined by the constant ENVIRONMENT_NAME + The environment specification are defined in ENVIRONMENT_FILE (see the + documentation for more information https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-from-an-environment-yml-file) + + It will use 'mamba' instead of 'conda' if it is found. + """ mamba = which("mamba") conda_exe = "mamba" if mamba is not None else "conda" @@ -49,10 +60,13 @@ def create_conda_environment(): {% endif %} def install_extension():{% if venv == "conda" %} + """Install the extension in development mode{% if venv == "conda" %} in the conda environment. + """ mamba = which("mamba") conda_exe = "mamba" if mamba is not None else "conda" pre_command = [conda_exe, "run", "--name", ENVIRONMENT_NAME] {% else %} + """ pre_command = [] # Checking nodeJS is available try: diff --git a/template/scripts/pre-commit b/template/scripts/pre-commit index 59da0fe..262b582 100755 --- a/template/scripts/pre-commit +++ b/template/scripts/pre-commit @@ -1,5 +1,7 @@ #!/bin/sh -# This script should be installed as pre-commit git hook (see https://www.atlassian.com/git/tutorials/git-hooks) +# This script can be installed as pre-commit git hook (see https://www.atlassian.com/git/tutorials/git-hooks) +# by copying it in '.git/hooks' folder. It will allow to catch +# linter error early. # Check frontend files are linted jlpm lint:check From 594998d32055d71ed89e7c6ba85a6423a9cb864f Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Sun, 20 Aug 2023 12:46:01 +0200 Subject: [PATCH 07/15] Improve the README --- README.md | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4fdc736..db6777d 100644 --- a/README.md +++ b/README.md @@ -11,18 +11,38 @@ a JupyterLab extension. Four kinds of extension are supported: ## Use the template to create extension +0. Pre-requisites + +We strongly advice you to use conda package manager. +To do so, you can install [Miniconda](https://docs.conda.io/en/latest/miniconda.html). + +Once you have it installed, open a _Anaconda Prompt_ (on Windows) or any shell +(on Linux and macOS) and execute the following commands: + +```sh +conda config --add channels conda-forge +conda install -yc conda-forge mamba +``` + +The first one will give you access to the community driven package repository +[conda-forge](https://conda-forge.org/docs/user/introduction.html). And the +second will install a faster package manager than conda, [mamba](https://mamba.readthedocs.io). +It is fully compatible with _conda_ and uses the same command syntax. + 1. Install copier and some plugins. -With `pip`: +With `conda` / `mamba` (the two commands have the same API): ```sh -pip install "copier~=7.2" jinja2-time "pydantic<2.0.0" +mamba install -c conda-forge "copier>=7,<8" jinja2-time ``` -Or with `conda` / `mamba`: +or + +With `pip`: ```sh -conda install -c conda-forge "copier>=7,<8" jinja2-time +pip install "copier~=7.2" jinja2-time "pydantic<2.0.0" ``` 2. Create an extension directory and go to it. From 0bc5182eb9a95ebb17f593e72327fa028d3cd3ca Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Sun, 20 Aug 2023 12:54:34 +0200 Subject: [PATCH 08/15] Try fixing the CI --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5f912ce..a1ce117 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -421,21 +421,25 @@ jobs: activate-environment: "" - name: Install dependencies + shell: bash -el {0} run: | python -m pip install "copier>=8.0.0" jinja2-time "pydantic<2.0.0" - name: Setup Git + shell: bash -el {0} run: | git config --global user.name "github-actions[bot]" git config --global user.email "github-actions[bot]@users.noreply.github.com" - name: Create server extension with conda + shell: bash -el {0} run: | tmpfolder=$(mktemp -d) python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . $tmpfolder - name: Check conda environment + shell: bash -el {0} run: | set -eux conda env list | grep -ie "myextension_dev" @@ -450,7 +454,6 @@ jobs: pip uninstall -y myextension jupyterlab rm -rf $tmpfolder - shell: bash pnpm_linker: runs-on: ubuntu-latest From 05aa8a61626b88bf102fd12c510ef7487f289bb5 Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Sun, 20 Aug 2023 14:52:20 +0200 Subject: [PATCH 09/15] Fix jinja template --- template/bootstrap.py.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/bootstrap.py.jinja b/template/bootstrap.py.jinja index 4551f0f..4552b39 100755 --- a/template/bootstrap.py.jinja +++ b/template/bootstrap.py.jinja @@ -59,7 +59,7 @@ def create_conda_environment(): print(f"Conda environment '{ENVIRONMENT_NAME}' {command}d. Execute '{conda_exe} activate {ENVIRONMENT_NAME}' to use it.\n") {% endif %} -def install_extension():{% if venv == "conda" %} +def install_extension(): """Install the extension in development mode{% if venv == "conda" %} in the conda environment. """ mamba = which("mamba") From c7de99dde473f3cc46bebb75e4e1fc700c431260 Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Sun, 20 Aug 2023 15:11:28 +0200 Subject: [PATCH 10/15] Try to be smarter --- .github/workflows/main.yml | 34 +++++++++++++++------------------- template/bootstrap.py.jinja | 4 ++++ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a1ce117..d7a1175 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,9 +46,9 @@ jobs: set -eux tmpfolder=$(mktemp -d) - python -m copier copy -l -d author_name="My Name" -d labextension_name="${NAME}" -d python_name="${PYNAME}" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder + YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m copier copy -l -d author_name="My Name" -d labextension_name="${NAME}" -d python_name="${PYNAME}" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder pushd $tmpfolder - YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py + ./scripts/pre-commit jupyter labextension list jupyter labextension list 2>&1 | grep -ie "${NAME}.*OK" @@ -88,9 +88,9 @@ jobs: set -eux tmpfolder=$(mktemp -d) - python -m copier copy -l -d author_name="My Name" -d test=n -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension + YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m copier copy -l -d author_name="My Name" -d test=n -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder pushd $tmpfolder - YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py + ./scripts/pre-commit jupyter labextension list @@ -131,9 +131,9 @@ jobs: set -eux tmpfolder=$(mktemp -d) - python -m copier copy -l -d author_name="My Name" -d has_settings=y -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder + YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m copier copy -l -d author_name="My Name" -d has_settings=y -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder pushd $tmpfolder - YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py + # It is not easily possible to get this version compatible with linter rules jlpm lint ./scripts/pre-commit @@ -175,14 +175,14 @@ jobs: - name: Create server extension pip install env: - YARN_ENABLE_IMMUTABLE_INSTALLS: false + COPIER_SKIP_INSTALL_TASK: '1' run: | tmpfolder=$(mktemp -d) python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder cd $tmpfolder cat pyproject.toml - pip install . + YARN_ENABLE_IMMUTABLE_INSTALLS=false pip install . pip install "jupyterlab>=4.0.0,<5" - name: Check pip install method @@ -201,14 +201,11 @@ jobs: shell: bash - name: Create server extension pip develop - env: - YARN_ENABLE_IMMUTABLE_INSTALLS: false run: | tmpfolder=$(mktemp -d) - python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder + YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder cd $tmpfolder - python bootstrap.py # Check unit tests are passing python -m pytest -vv -r ap --cov myextension @@ -245,7 +242,7 @@ jobs: - name: Install server extension from a tarball env: - YARN_ENABLE_IMMUTABLE_INSTALLS: false + COPIER_SKIP_INSTALL_TASK: '1' run: | tmpfolder=$(mktemp -d) @@ -253,7 +250,7 @@ jobs: cd $tmpfolder python -m pip install "jupyterlab>=4.0.0,<5" jupyter lab clean --all - python -m build $tmpfolder + YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m build $tmpfolder cd dist python -m pip install myextension-0.1.0.tar.gz @@ -350,9 +347,8 @@ jobs: run: | set -eux mkdir mytheme - python -m copier copy -l -d kind=theme -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . mytheme + YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m copier copy -l -d kind=theme -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . mytheme pushd mytheme - YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py ./scripts/pre-commit jupyter labextension list @@ -393,9 +389,8 @@ jobs: set -eux tmpfolder=$(mktemp -d) - python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder + YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder pushd $tmpfolder - YARN_ENABLE_IMMUTABLE_INSTALLS=false python bootstrap.py ./scripts/pre-commit @@ -436,7 +431,7 @@ jobs: run: | tmpfolder=$(mktemp -d) - python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . $tmpfolder + YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . $tmpfolder - name: Check conda environment shell: bash -el {0} @@ -480,6 +475,7 @@ jobs: - name: Create pure frontend extension env: KIND: ${{ matrix.kind }} + COPIER_SKIP_INSTALL_TASK: '1' run: | set -eux tmpfolder=$(mktemp -d) diff --git a/template/bootstrap.py.jinja b/template/bootstrap.py.jinja index 4552b39..83cd21f 100755 --- a/template/bootstrap.py.jinja +++ b/template/bootstrap.py.jinja @@ -81,6 +81,10 @@ def install_extension(): if not answer.lower().startswith('y'): return {% endif %} + if os.environ("COPIER_SKIP_INSTALL_TASK", "0") != "0": + print("Skipping installation task") + return + print("\nInstalling the extension in development mode...") # Force uninstalling the extension to avoid trouble when installing in dev mode From cff86d99746ce41f1b1592439c429af9a2f0b90b Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Sun, 20 Aug 2023 15:13:14 +0200 Subject: [PATCH 11/15] Fix environment get syntax --- template/bootstrap.py.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/bootstrap.py.jinja b/template/bootstrap.py.jinja index 83cd21f..418cc93 100755 --- a/template/bootstrap.py.jinja +++ b/template/bootstrap.py.jinja @@ -81,7 +81,7 @@ def install_extension(): if not answer.lower().startswith('y'): return {% endif %} - if os.environ("COPIER_SKIP_INSTALL_TASK", "0") != "0": + if os.environ.get("COPIER_SKIP_INSTALL_TASK", "0") != "0": print("Skipping installation task") return From 121880b194f16e4ee7082adf49f675cead485749 Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Sun, 20 Aug 2023 15:25:45 +0200 Subject: [PATCH 12/15] More CI fixes --- .github/workflows/check-tests.yml | 10 +++++++--- .github/workflows/main.yml | 28 ++++++++++++++++++---------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.github/workflows/check-tests.yml b/.github/workflows/check-tests.yml index 5fc0671..d707a60 100644 --- a/.github/workflows/check-tests.yml +++ b/.github/workflows/check-tests.yml @@ -24,12 +24,14 @@ jobs: - name: Create the extension id: create-extension + env: + COPIER_SKIP_INSTALL_TASK: '1' run: | set -eux tmpfolder=$(mktemp -d) echo "extension_folder=${tmpfolder}" >> "$GITHUB_OUTPUT" - python -m copier copy -l -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder - cat $tmpfolder/pyproject.toml + python -m copier copy -l -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . ${tmpfolder} + cat ${tmpfolder}/pyproject.toml - name: Test the extension working-directory: ${{ steps.create-extension.outputs.extension_folder }} @@ -95,11 +97,13 @@ jobs: - name: Create the extension id: create-extension + env: + COPIER_SKIP_INSTALL_TASK: '1' run: | set -eux tmpfolder=$(mktemp -d) - python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . myextension echo "extension_folder=${tmpfolder}" >> "$GITHUB_OUTPUT" + python -m copier copy -l -d kind=mimerenderer -d viewer_name="My Viewer" -d mimetype="application/vnd.my_org.my_type" -d mimetype_name="my_type" -d file_extension=".my_type" -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . ${tmpfolder} - name: Install the extension working-directory: ${{ steps.create-extension.outputs.extension_folder }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d7a1175..903d39c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -174,10 +174,12 @@ jobs: git config --global user.email "github-actions[bot]@users.noreply.github.com" - name: Create server extension pip install + id: create-extension-1 env: COPIER_SKIP_INSTALL_TASK: '1' run: | tmpfolder=$(mktemp -d) + echo "extension_folder=${tmpfolder}" >> "$GITHUB_OUTPUT" python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder cd $tmpfolder @@ -197,15 +199,17 @@ jobs: python -m jupyterlab.browser_check pip uninstall -y myextension jupyterlab - rm -rf $tmpfolder + rm -rf "${{ steps.create-extension-1.outputs.extension_folder }}" shell: bash - name: Create server extension pip develop + id: create-extension-2 run: | tmpfolder=$(mktemp -d) + echo "extension_folder=${tmpfolder}" >> "$GITHUB_OUTPUT" YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder - cd $tmpfolder + cd ${tmpfolder} # Check unit tests are passing python -m pytest -vv -r ap --cov myextension @@ -230,27 +234,29 @@ jobs: - name: Build server extension in develop mode run: | - jupyter labextension build $tmpfolder + jupyter labextension build "${{ steps.create-extension-2.outputs.extension_folder }}" jupyter labextension uninstall myextension python -m pip uninstall -y myextension jupyterlab - run: | set -eux - rm -rf $tmpfolder + rm -rf "${{ steps.create-extension-2.outputs.extension_folder }}" shell: bash - name: Install server extension from a tarball + id: create-extension-3 env: COPIER_SKIP_INSTALL_TASK: '1' run: | tmpfolder=$(mktemp -d) + echo "extension_folder=${tmpfolder}" >> "$GITHUB_OUTPUT" python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder - cd $tmpfolder + cd ${tmpfolder} python -m pip install "jupyterlab>=4.0.0,<5" jupyter lab clean --all - YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m build $tmpfolder + YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m build ${tmpfolder} cd dist python -m pip install myextension-0.1.0.tar.gz @@ -266,10 +272,10 @@ jobs: python -m jupyterlab.browser_check - cp $tmpfolder/dist/*.tar.gz myextension.tar.gz - cp $tmpfolder/dist/*.whl myextension.whl + cp "${{ steps.create-extension-3.outputs.extension_folder }}"/dist/*.tar.gz myextension.tar.gz + cp "${{ steps.create-extension-3.outputs.extension_folder }}"/dist/*.whl myextension.whl python -m pip uninstall -y myextension jupyterlab - rm -rf $tmpfolder + rm -rf "${{ steps.create-extension-3.outputs.extension_folder }}" shell: bash - uses: actions/upload-artifact@v2 @@ -427,9 +433,11 @@ jobs: git config --global user.email "github-actions[bot]@users.noreply.github.com" - name: Create server extension with conda + id: create-extension shell: bash -el {0} run: | tmpfolder=$(mktemp -d) + echo "extension_folder=${tmpfolder}" >> "$GITHUB_OUTPUT" YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" --vcs-ref HEAD --UNSAFE . $tmpfolder @@ -448,7 +456,7 @@ jobs: python -m jupyterlab.browser_check pip uninstall -y myextension jupyterlab - rm -rf $tmpfolder + rm -rf "${{ steps.create-extension.outputs.extension_folder }}" pnpm_linker: runs-on: ubuntu-latest From bbae958a1c1837897f863426270ae8e07b1cba34 Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Sun, 20 Aug 2023 15:43:47 +0200 Subject: [PATCH 13/15] Fix rerunning bootstrap --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 903d39c..e01d60c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -231,6 +231,7 @@ jobs: set -eux python bootstrap.py shell: bash + working-directory: ${{ steps.create-extension-2.outputs.extension_folder }} - name: Build server extension in develop mode run: | From 0ad86665e8fb67e47d37478046f0fd26433afb9e Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Sat, 9 Sep 2023 13:34:13 +0200 Subject: [PATCH 14/15] Debug CI --- .github/workflows/main.yml | 21 +++++++++++---------- template/bootstrap.py.jinja | 10 +++++----- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e01d60c..946592a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -253,13 +253,17 @@ jobs: tmpfolder=$(mktemp -d) echo "extension_folder=${tmpfolder}" >> "$GITHUB_OUTPUT" - python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . $tmpfolder - cd ${tmpfolder} + python -m copier copy -l -d kind=server -d author_name="My Name" -d repository="https://github.com/test/lab-extension" -d venv="none" --vcs-ref HEAD --UNSAFE . ${tmpfolder} + python -m pip install "jupyterlab>=4.0.0,<5" jupyter lab clean --all YARN_ENABLE_IMMUTABLE_INSTALLS=false python -m build ${tmpfolder} - cd dist - python -m pip install myextension-0.1.0.tar.gz + + cp ${tmpfolder}/dist/*.tar.gz myextension.tar.gz + cp ${tmpfolder}/dist/*.whl myextension.whl + + tar list -f ${tmpfolder}/dist/myextension-0.1.0.tar.gz + python -m pip install ${tmpfolder}/dist/myextension-0.1.0.tar.gz - name: Check install tarball method run: | @@ -272,15 +276,12 @@ jobs: jupyter lab build --dev-build --no-minimize python -m jupyterlab.browser_check - - cp "${{ steps.create-extension-3.outputs.extension_folder }}"/dist/*.tar.gz myextension.tar.gz - cp "${{ steps.create-extension-3.outputs.extension_folder }}"/dist/*.whl myextension.whl python -m pip uninstall -y myextension jupyterlab rm -rf "${{ steps.create-extension-3.outputs.extension_folder }}" shell: bash - - uses: actions/upload-artifact@v2 - if: startsWith(runner.os, 'Linux') + - uses: actions/upload-artifact@v3 + if: startsWith(runner.os, 'Linux') && always() with: name: myextension-sdist path: | @@ -310,7 +311,7 @@ jobs: restore-keys: | pip-3.7- pip- - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: myextension-sdist - name: Install and Test diff --git a/template/bootstrap.py.jinja b/template/bootstrap.py.jinja index 418cc93..863f778 100755 --- a/template/bootstrap.py.jinja +++ b/template/bootstrap.py.jinja @@ -60,13 +60,16 @@ def create_conda_environment(): {% endif %} def install_extension(): - """Install the extension in development mode{% if venv == "conda" %} in the conda environment. + """Install the extension in development mode{% if venv == "conda" %} in the conda environment.{% endif %} """ + if os.environ.get("COPIER_SKIP_INSTALL_TASK", "0") != "0": + print("Skipping installation task") + return +{% if venv == "conda" %} mamba = which("mamba") conda_exe = "mamba" if mamba is not None else "conda" pre_command = [conda_exe, "run", "--name", ENVIRONMENT_NAME] {% else %} - """ pre_command = [] # Checking nodeJS is available try: @@ -81,9 +84,6 @@ def install_extension(): if not answer.lower().startswith('y'): return {% endif %} - if os.environ.get("COPIER_SKIP_INSTALL_TASK", "0") != "0": - print("Skipping installation task") - return print("\nInstalling the extension in development mode...") From d8b9a97c0f10fd7c97e62b55abd1966cad98a611 Mon Sep 17 00:00:00 2001 From: Frederic COLLONVAL Date: Sat, 9 Sep 2023 18:09:54 +0200 Subject: [PATCH 15/15] Restore removed lint command --- .github/workflows/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 946592a..445df7c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -186,6 +186,7 @@ jobs: cat pyproject.toml YARN_ENABLE_IMMUTABLE_INSTALLS=false pip install . pip install "jupyterlab>=4.0.0,<5" + jlpm lint:check - name: Check pip install method run: | @@ -262,7 +263,9 @@ jobs: cp ${tmpfolder}/dist/*.tar.gz myextension.tar.gz cp ${tmpfolder}/dist/*.whl myextension.whl - tar list -f ${tmpfolder}/dist/myextension-0.1.0.tar.gz + tar -tf ${tmpfolder}/dist/myextension-0.1.0.tar.gz + + python -m pip uninstall -y myextension || true python -m pip install ${tmpfolder}/dist/myextension-0.1.0.tar.gz - name: Check install tarball method