add ! documentation

.github/file-filters.yml

@@ -37,3 +37,6 @@ lint:
   -  *python
   - '.flake8'
   - 'pyproject.toml'
+
+documentation:
+  - added|modified: 'docs/**'

.github/workflows/documentation.yml

@@ -0,0 +1,87 @@
+name: Deploy Jekyll site to Pages
+
+on:
+  push:
+    branches: 
+      - develop
+      - master
+      - staging
+      - release/*
+      - documentation/*
+    paths:
+      - "docs/**"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    timeout-minutes: 1
+    outputs:
+      run_deploy: ${{ steps.changed-files.outputs.documentation }}
+    steps:
+      - name: Checkout code
+        uses: actions/[email protected]
+      - id: changed-files
+        name: Check for documentation files changes
+        uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+  build:
+    name: Build Jekyll site
+    needs: [ changes ]
+    if: (needs.changes.outputs.run_deploy == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    defaults:
+      run:
+        working-directory: docs    
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3' # Not needed with a .ruby-version file
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+          cache-version: 0 # Increment this number if you need to re-download cached gems
+          working-directory: '${{ github.workspace }}/docs'
+      - name: Setup Pages
+        id: pages
+        uses: actions/configure-pages@v5
+      - name: Build with Jekyll
+        # Outputs to the './_site' directory by default
+        run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}"
+        env:
+          JEKYLL_ENV: production
+      - name: Upload artifact
+        # Automatically uploads an artifact from the './_site' directory by default
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: "docs/_site/"
+
+  deploy:
+    name: Deploy to GitHub Pages
+    needs: [ changes,build ]
+    runs-on: ubuntu-latest
+    if: (needs.changes.outputs.run_deploy == 'true')
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
+        with:
+          branch: gh-pages

docs/.gitignore

@@ -0,0 +1,15 @@
+# Not sure what a .gitignore is?
+# See: https://git-scm.com/docs/gitignore
+
+# These are directly copied from Jekyll's first-party docs on `.gitignore` files:
+# https://jekyllrb.com/tutorials/using-jekyll-with-bundler/#commit-to-source-control
+
+# Ignore the default location of the built site, and caches and metadata generated by Jekyll
+_site/
+.sass-cache/
+.jekyll-cache/
+.jekyll-metadata
+
+# Ignore folders generated by Bundler
+.bundle/
+vendor/

docs/404.html

@@ -0,0 +1,11 @@
+---
+layout: default
+title: 404
+permalink: /404
+nav_exclude: true
+search_exclude: true
+---
+
+<h1>Page not found</h1>
+
+<p>The page you requested could not be found. Try using the navigation {% if site.search_enabled != false %}or search {% endif %}to find what you're looking for or go to this <a href="{{ '/' | relative_url }}">site's home page</a>.</p>

docs/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gem "jekyll", "~> 4.3.3" # installed by `gem jekyll`
+# gem "webrick"        # required when using Ruby >= 3 and Jekyll <= 4.2.2
+
+gem "just-the-docs", "0.8.2" # pinned to the current release
+# gem "just-the-docs"        # always download the latest release

docs/Gemfile.lock

@@ -0,0 +1,92 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.6)
+      public_suffix (>= 2.0.2, < 6.0)
+    colorator (1.1.0)
+    concurrent-ruby (1.2.3)
+    em-websocket (0.5.3)
+      eventmachine (>= 0.12.9)
+      http_parser.rb (~> 0)
+    eventmachine (1.2.7)
+    ffi (1.16.3)
+    forwardable-extended (2.6.0)
+    google-protobuf (4.26.1)
+      rake (>= 13)
+    google-protobuf (4.26.1-arm64-darwin)
+      rake (>= 13)
+    google-protobuf (4.26.1-x86_64-linux)
+      rake (>= 13)
+    http_parser.rb (0.8.0)
+    i18n (1.14.4)
+      concurrent-ruby (~> 1.0)
+    jekyll (4.3.3)
+      addressable (~> 2.4)
+      colorator (~> 1.0)
+      em-websocket (~> 0.5)
+      i18n (~> 1.0)
+      jekyll-sass-converter (>= 2.0, < 4.0)
+      jekyll-watch (~> 2.0)
+      kramdown (~> 2.3, >= 2.3.1)
+      kramdown-parser-gfm (~> 1.0)
+      liquid (~> 4.0)
+      mercenary (>= 0.3.6, < 0.5)
+      pathutil (~> 0.9)
+      rouge (>= 3.0, < 5.0)
+      safe_yaml (~> 1.0)
+      terminal-table (>= 1.8, < 4.0)
+      webrick (~> 1.7)
+    jekyll-include-cache (0.2.1)
+      jekyll (>= 3.7, < 5.0)
+    jekyll-sass-converter (3.0.0)
+      sass-embedded (~> 1.54)
+    jekyll-seo-tag (2.8.0)
+      jekyll (>= 3.8, < 5.0)
+    jekyll-watch (2.2.1)
+      listen (~> 3.0)
+    just-the-docs (0.8.2)
+      jekyll (>= 3.8.5)
+      jekyll-include-cache
+      jekyll-seo-tag (>= 2.0)
+      rake (>= 12.3.1)
+    kramdown (2.4.0)
+      rexml
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
+    liquid (4.0.4)
+    listen (3.9.0)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
+    mercenary (0.4.0)
+    pathutil (0.16.2)
+      forwardable-extended (~> 2.6)
+    public_suffix (5.0.5)
+    rake (13.2.1)
+    rb-fsevent (0.11.2)
+    rb-inotify (0.10.1)
+      ffi (~> 1.0)
+    rexml (3.2.6)
+    rouge (4.2.1)
+    safe_yaml (1.0.5)
+    sass-embedded (1.75.0)
+      google-protobuf (>= 3.25, < 5.0)
+      rake (>= 13.0.0)
+    sass-embedded (1.75.0-arm64-darwin)
+      google-protobuf (>= 3.25, < 5.0)
+    sass-embedded (1.75.0-x86_64-linux-gnu)
+      google-protobuf (>= 3.25, < 5.0)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
+    unicode-display_width (2.5.0)
+    webrick (1.8.1)
+
+PLATFORMS
+  arm64-darwin
+  x86_64-linux-gnu
+
+DEPENDENCIES
+  jekyll (~> 4.3.3)
+  just-the-docs (= 0.8.2)
+
+BUNDLED WITH
+   2.5.9

docs/_config.yml

@@ -0,0 +1,44 @@
+title: "HOPE-dD Engine"
+description: "HOPE deduplication engine documentaion"
+theme: "just-the-docs"
+favicon_ico: "/assets/images/favicon.ico"
+
+url: https://vitali-yanushchyk-valor.github.io/hde-doc/
+
+aux_links:
+  HOPE Repository: https://github.com/unicef/hct-mis
+  HOPE-DE Repository: https://github.com/unicef/hope-dedup-engine
+
+# Pick an available version from https://cdn.jsdelivr.net/npm/mermaid/
+mermaid:
+  version: "10.9.1"
+
+# Enable or disable the side/mobile menu globally
+# Nav menu can also be selectively enabled or disabled using page variables or the minimal layout
+nav_enabled: true
+
+# Heading anchor links appear on hover over h1-h6 tags in page content
+# allowing users to deep link to a particular heading on a page.
+#
+# Supports true (default) or false
+heading_anchors: true
+
+# Color scheme supports "light" (default) and "dark"
+# color_scheme: dark
+
+
+# Google Analytics Tracking (optional)
+# Supports a CSV of tracking ID strings (eg. "UA-1234567-89,G-1AB234CDE5")
+# ga_tracking: 
+# ga_tracking_anonymize_ip: true # Use GDPR compliant Google Analytics settings (true/nil by default)
+
+
+
+# Footer content
+# appears at the bottom of every page's main content
+# Note: The footer_content option is deprecated and will be removed in a future major release. Please use `_includes/footer_custom.html` for more robust markup / liquid-based content.
+footer_content: "Copyright © 2024-2024. Distributed by ..."
+
+# Footer last edited timestamp
+# last_edit_timestamp: true # show or hide edit time - page must have `last_modified_date` defined in the frontmatter
+# last_edit_time_format: "%b %e %Y at %I:%M %p" # uses ruby's time format: https://ruby-doc.org/stdlib-2.7.0/libdoc/time/rdoc/Time.html

docs/compose.yml

@@ -0,0 +1,9 @@
+version: '3.8'
+
+services:
+  hde-doc-build:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+    ports:
+      - 4000:4000

docs/docker/Dockerfile

@@ -0,0 +1,11 @@
+FROM ruby:3.3.4
+
+RUN gem install bundler:2.5.9
+
+WORKDIR /usr/src/app
+COPY . .
+
+RUN bundle install \
+    && bundle exec jekyll build --baseurl '/hde-doc'
+
+CMD ["jekyll",  "serve",  "--host",  "0.0.0.0"]

docs/handbook/API-Documentation.md

@@ -0,0 +1,18 @@
+---
+layout: default
+title: API
+nav_order: 2
+---
+
+
+The application provides comprehensive API documentation to facilitate ease of use and integration. The API documentation is accessible through two main interfaces: Swagger UI and Redoc.
+
+**Swagger UI**: An interactive interface that allows users to explore and test the API endpoints. It provides detailed information about the available endpoints, their parameters, and response formats. Users can input data and execute requests directly from the interface.
+
+URL: `//<your-domain.com>/api/rest/swagger/`
+
+**Redoc**: A static, beautifully rendered documentation interface that offers a more structured and user-friendly presentation of the API. It includes comprehensive details about each endpoint, including descriptions, parameters, and example requests and responses.
+
+URL: `//<your-domain.com>/api/rest/redoc/`
+
+These interfaces ensure that developers have all the necessary information to effectively utilize the API, enabling seamless integration and interaction with the application’s features.

docs/handbook/deduplication.md

@@ -0,0 +1,61 @@
+---
+layout: default
+title: Deduplication
+nav_order: 2
+---
+
+# Deduplication Worlflow
+{: .no_toc }
+
+<details open markdown="block">
+  <summary>
+    Table of contents
+  </summary>
+  {: .text-delta }
+- TOC
+{:toc}
+</details>
+
+---
+
+### Inference Mode Operation
+
+This application operates in inference mode, meaning it utilizes a pre-trained model for face recognition rather than training the model itself. The pre-trained model is stored in Azure Blob Storage and is downloaded by the application upon startup. Additionally, the model can be manually updated via the admin panel.
+
+### Model Details
+
+The face recognition functionality is powered by the [OpenCV](https://github.com/opencv/opencv) library.
+
+Currently, the application uses an open-source model. The model consists of two files: **deploy.prototxt** and **res10_300x300_ssd_iter_140000.caffemodel**. The model was trained using the Caffe deep learning framework and employs the Res10 architecture with a resolution of 300x300 and 140,000 iterations. This model is based on the SSD (Single Shot MultiBox Detector) methodology and is optimized for face detection tasks.
+
+### Worklow Diagram
+```mermaid
+flowchart LR
+  subgraph DNNManager[DNN Manager]
+      direction TB
+      load_model[Load Model] --> set_preferences[Set Preferences]
+  end
+
+  subgraph ImageProcessing[Image Processing]
+      direction LR
+      
+      subgraph FaceDetection[Face Detection]
+          direction TB
+          load_image[Load Image] -- decoded image as 3D numpy array\n(height, width, channels of BlueGreeRed color space) --> prepare_image[Prepare Image] -- blob 4D tensor\n(normalized size, use scale factor and means) --> run_model[Run Model] -- shape (1, 1, N, 7),\n1 image\nN is the number of detected faces\neach face is described by the 7 detection values--> filter_results[Filter Results] -- confidence is above the minimum threshold\nNMS to suppress overlapping bounding boxes --> return_detections[Return Detections]
+      end
+      
+      subgraph FaceRecognition[Face Recognition]
+          direction TB
+          load_image_[Load Image] --> detect_faces[Detect Faces] -- detected face regions --> generate_encodings[Generate Encodings] -- numerical representations of the facial features\n(face's geometry and appearance) --> save_encodings[Save Encodings]
+      end
+  end
+
+  subgraph DuplicateFinder[Duplicate Finder]
+      direction TB
+      load_encodings[Load Encodings] --> compare_encodings[Compare Encodings] -- face distance less then threshold--> return_duplicates[Return Duplicates]
+  end
+
+  DNNManager --> ImageProcessing --> DuplicateFinder
+  FaceDetection --> FaceRecognition
+
+```

docs/handbook/demo.md

@@ -0,0 +1,26 @@
+---
+layout: default
+title: Demo
+nav_order: 2
+---
+
+### Instructions to Run the Demo Application
+
+To run the demo application locally using Docker Compose, follow these steps:
+
+- Navigate to the root directory of the repository.
+
+- Execute the following command:
+
+    ```docker compose -f tests/extras/demoapp/compose.yml up```
+
+### Accessing the HDE application
+
+- **Admin Panel**: http://localhost:8000/ (username: `[email protected]`, password: `adm`)
+
+- **API Documentation**:
+
+Swagger UI: http://localhost:8000/api/rest/swagger/
+
+
+Redoc: http://localhost:8000/api/rest/redoc/