Major changes in dependency management, docs, linting, and checks

.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.{py,toml}]
+indent_style = space
+indent_size = 4
+
+[*.yml,yaml,json]
+indent_style = space
+indent_size = 2

.github/actions/setup-env/action.yml

@@ -0,0 +1,42 @@
+name: "setup-env"
+description: "Set up a Python environment with Poetry"
+
+inputs:
+  python-version:
+    required: false
+    description: "The version of Python to use with Poetry"
+    default: "3.11"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Install Poetry
+      env:
+        POETRY_VERSION: "1.8.4"
+      run: curl -sSL https://install.python-poetry.org | python - -y
+      shell: bash
+
+    - name: Add Poetry to Path
+      run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+      shell: bash
+
+    - name: Configure Poetry virtual environment in project
+      run: poetry config virtualenvs.in-project true
+      shell: bash
+
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v4
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('poetry.lock') }}
+
+    - name: Install dependencies
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      run: poetry install --no-interaction
+      shell: bash

.github/workflows/documentation.yml

@@ -20,24 +20,25 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.11.7' 
+        python-version: '3.11.7'
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+    - name: Set up Poetry
+    - uses: ./.github/actions/setup-env
     - name: Install dependencies
       run: |
-          pip install -r docs/requirements.txt
+        poetry install --with doc --no-interaction
     - name: List directories
       run: |
           ls -la
-    - name: Build Sphinx Documentation
+    - name: Build Documentation
       run: |
-          cd docs
-          make html
+          mkdocs build -f docs/mkdocs.yml -d docs/_build
     - name: Add files
       run: |
           echo "nebula.enriquetomasmb.com" > docs/_build/CNAME

.github/workflows/main.yml

@@ -0,0 +1,38 @@
+name: Main
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+jobs:
+  quality:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out
+        uses: actions/checkout@v4
+
+      - uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
+
+      - name: Set up the environment
+        uses: ./.github/actions/setup-env
+
+      - name: Run checks
+        run: make check
+
+  check-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out
+        uses: actions/checkout@v4
+
+      - name: Set up the environment
+        uses: ./.github/actions/setup-env
+
+      - name: Check if documentation can be built
+        run: poetry run mkdocs build -s

.github/workflows/on-release.yml

@@ -0,0 +1,39 @@
+name: Release
+
+on:
+  release:
+    types: [published]
+    branches: [main]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out
+        uses: actions/checkout@v4
+
+      - name: Set up the environment
+        uses: ./.github/actions/setup-env
+
+      - name: Export tag
+        id: vars
+        run: echo tag=${GITHUB_REF#refs/*/} >> $GITHUB_OUTPUT
+
+      - name: Build and publish
+        run: |
+          source .venv/bin/activate
+          poetry version $RELEASE_VERSION
+          make build-and-publish
+        env:
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+          RELEASE_VERSION: ${{ steps.vars.outputs.tag }}
+  deploy-docs:
+    needs: publish
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger documentation workflow
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: documentation.yml
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: main

.gitignore

@@ -118,14 +118,23 @@ venv.bak/
 # Rope project settings
 .ropeproject
 
+# poetry
+poetry.lock
+
 # mkdocs documentation
 /site
+_prebuilt/
+_build/
 
 # mypy
 .mypy_cache/
 .dmypy.json
 dmypy.json
 
+# ruff
+.ruff/
+.ruff_cache/
+
 # Pyre type checker
 .pyre/
 
@@ -142,4 +151,4 @@ data/
 .requirements.txt
 data-analysis/
 network-analysis/
-docker-compose.yml
+docker-compose.yml

.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: "v5.0.0"
+    hooks:
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-toml
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.7.2
+    hooks:
+      - id: ruff
+        args: [--exit-non-zero-on-fix]
+      - id: ruff-format

CHANGELOG.md

@@ -0,0 +1,2 @@
+# Changelog
+All notable changes to this project will be documented in this file.

CONTRIBUTING.md

@@ -0,0 +1,2 @@
+# Contributing
+We welcome contributions to this project. Please read the following guidelines.

Dockerfile

@@ -16,6 +16,19 @@ RUN apt-get update && apt-get install -y python3.11 python3.11-dev python3.11-di
 # Install curl and network tools
 RUN apt-get install -y curl net-tools iproute2 iputils-ping
 
+# Update alternatives to make Python 3.11 the default
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 2
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+
+# Install pip
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+RUN python3.11 get-pip.py
+
+RUN python3.11 -m pip install --upgrade pip setuptools virtualenv
+
+# Install gcc and git
+RUN apt-get update && apt-get install -y build-essential gcc g++ clang git make cmake
+
 # Install docker
 RUN apt-get install -y ca-certificates curl gnupg
 RUN install -m 0755 -d /etc/apt/keyrings
@@ -29,17 +42,12 @@ RUN apt-get update
 
 RUN apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
 
-# Install pip
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-RUN python3.11 get-pip.py
+RUN curl -sSL https://install.python-poetry.org | python3 -
 
-# Upgrade pip
-RUN python3.11 -m pip install --upgrade pip
+ENV PATH="${PATH}:/root/.local/bin"
 
-# Install gcc and git
-RUN apt-get update && apt-get install -y build-essential gcc g++ clang git make cmake
+ENV POETRY_VIRTUALENVS_CREATE=false
+
+COPY pyproject.toml .
 
-WORKDIR /nebula
-COPY nebula/requirements.txt .
-# Install the required packages
-RUN python3.11 -m pip install --ignore-installed -r requirements.txt
+RUN poetry install --only core --no-root

Makefile

@@ -0,0 +1,63 @@
+.PHONY: install
+install: ## Install the poetry environment and install the pre-commit hooks
+	@echo "📦 Installing dependencies with Poetry"
+	@poetry install
+	@echo "🔧 Installing pre-commit hooks"
+	@poetry run pre-commit install
+	@echo "🐚 Activating virtual environment"
+	@poetry shell
+
+.PHONY: check
+check: ## Run code quality tools.
+	@echo "🛠️ Running code quality checks"
+	@echo "🔍 Checking Poetry lock file consistency"
+	@poetry check --lock
+	@echo "🚨 Linting code with pre-commit"
+	@poetry run pre-commit run -a
+
+.PHONY: check-plus
+check-plus: check ## Run additional code quality tools.
+	@echo "🔍 Checking code formatting with black
+	@poetry run black --check ."
+	@echo "⚙️ Static type checking with mypy"
+	@poetry run mypy
+	@echo "🔎 Checking for obsolete dependencies"
+	@poetry run deptry .
+
+.PHONY: build
+build: clean-build ## Build wheel file using poetry
+	@echo "🚀 Creating wheel file"
+	@poetry build
+
+.PHONY: clean-build
+clean-build: ## clean build artifacts
+	@rm -rf dist
+
+.PHONY: publish
+publish: ## publish a release to pypi.
+	@echo "🚀 Publishing: Dry run."
+	@poetry config pypi-token.pypi $(PYPI_TOKEN)
+	@poetry publish --dry-run
+	@echo "🚀 Publishing."
+	@poetry publish
+
+.PHONY: build-and-publish
+build-and-publish: build publish ## Build and publish.
+
+.PHONY: format
+format: ## Format code with black and isort
+	@echo "🎨 Formatting code"
+	@poetry run black .
+	@poetry run isort .
+
+.PHONY: clean
+clean: clean-build ## Clean up build artifacts and cache files
+	@echo "🧹 Cleaning up build artifacts and caches"
+	@rm -rf __pycache__ */__pycache__ .mypy_cache
+
+.PHONY: help
+help:
+	@echo "Available commands:"
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "💡 \033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+.DEFAULT_GOAL := help

README.md

@@ -149,8 +149,8 @@ We would like to thank the following projects for their contributions which have
 - [PyTorch Lightning](https://github.com/Lightning-AI/pytorch-lightning) for the training loop and model management
 - [Tensorboard](https://github.com/tensorflow/tensorboard) and [Aim](https://github.com/aimhubio/aim) for the visualization tools and monitoring capabilities
 - Different datasets ([nebula/core/datasets](https://github.com/enriquetomasmb/nebula/tree/main/nebula/core/datasets)) and models ([nebula/core/models](https://github.com/enriquetomasmb/nebula/tree/main/nebula/core/models)) for testing and validation purposes
-- [FastAPI](https://github.com/tiangolo/fastapi) for the RESTful API 
+- [FastAPI](https://github.com/tiangolo/fastapi) for the RESTful API
 - [Web3](https://github.com/ethereum/web3.py) for the blockchain integration
 - [Fedstellar](https://github.com/enriquetomasmb/fedstellar) platform and [p2pfl](https://github.com/pguijas/p2pfl/) library
 - [Adversarial Robustness Toolbox (ART)](https://github.com/Trusted-AI/adversarial-robustness-toolbox) for the implementation of adversarial attacks
-- [D3.js](https://github.com/d3/d3-force) for the network visualizations
+- [D3.js](https://github.com/d3/d3-force) for the network visualizations

app/main.py

@@ -38,14 +38,7 @@
     help="Frontend port (default: 6060)",
 )
 
-argparser.add_argument(
-    "-t",
-    "--test",
-    dest="test",
-    action="store_true",
-    default=False,
-    help="Run tests"
-)
+argparser.add_argument("-t", "--test", dest="test", action="store_true", default=False, help="Run tests")
 
 argparser.add_argument(
     "-st",
@@ -127,10 +120,9 @@
 args = argparser.parse_args()
 
 """
-Code for deploying the controller 
+Code for deploying the controller
 """
 if __name__ == "__main__":
-
     if args.stop == "all":
         Controller.stop()
     elif args.stop == "nodes":

docs/Makefile

@@ -1,12 +1,11 @@
-.PHONY: apidoc html clean all
+.PHONY: test
+test: ## Test if documentation can be built without warnings or errors
+	@poetry run mkdocs build -s
 
-apidoc:
-	sphinx-apidoc -f -o . ../nebula
+.PHONY: build
+build: ## Build the documentation
+	@poetry run mkdocs build -d _build
 
-html:
-	sphinx-build -b html . _build
-
-clean:
-	rm -rf _build
-
-all: apidoc html
+.PHONY: serve
+serve: ## Build and serve the documentation
+	@poetry run mkdocs serve