tests: Add automated notification tests. (#789)

This adds a suite of automated notification tests that simply compare a
screenshot of before and after a notification has been sent. It uses
docker, pytest and selenium, and generate an XML report.

Using the `makefile` we can pass in which servers we want to use: `make
ENV="dev" notification-test`

Closes: https://mozilla-hub.atlassian.net/browse/SYNC-4371

Base Screenshot: 

![base_screenshot](https://github.com/user-attachments/assets/950dae81-4802-4be6-bd5a-77811ce43ce1)

Screenshot with notification: 

![screenshot](https://github.com/user-attachments/assets/66a36083-33ab-47e3-8177-2e462bfd40de)

Makefile

@@ -4,6 +4,7 @@ TESTS_DIR := tests
 TEST_RESULTS_DIR ?= workspace/test-results
 PYTEST_ARGS ?= $(if $(SKIP_SENTRY),-m "not sentry") $(if $(TEST_STUB),,-m "not stub") # Stub tests do not work in CI
 INTEGRATION_TEST_FILE := $(TESTS_DIR)/integration/test_integration_all_rust.py
+NOTIFICATION_TEST_DIR := $(TESTS_DIR)/notification
 LOAD_TEST_DIR := $(TESTS_DIR)/load
 POETRY := poetry --directory $(TESTS_DIR)
 DOCKER_COMPOSE := docker compose
@@ -29,18 +30,27 @@ upgrade:
 
 integration-test-legacy:
 	$(POETRY) -V
-	$(POETRY) install --without dev,load
+	$(POETRY) install --without dev,load,notification --no-root
 	$(POETRY) run pytest $(INTEGRATION_TEST_FILE) \
 		--junit-xml=$(TEST_RESULTS_DIR)/integration_test_legacy_results.xml \
 		-v $(PYTEST_ARGS)
 
 integration-test:
 	$(POETRY) -V
-	$(POETRY) install --without dev,load
-	$(POETRY) run pytest $(INTEGRATION_TEST_FILE) \
+	$(POETRY) install --without dev,load,notification --no-root
+		$(POETRY) run pytest $(INTEGRATION_TEST_FILE) \
 		--junit-xml=$(TEST_RESULTS_DIR)/integration_test_results.xml \
 		-v $(PYTEST_ARGS)
 
+notification-test:
+	$(DOCKER_COMPOSE) -f $(NOTIFICATION_TEST_DIR)/docker-compose.yml build
+	$(DOCKER_COMPOSE) -f $(NOTIFICATION_TEST_DIR)/docker-compose.yml up -d server
+	$(DOCKER_COMPOSE) -f $(NOTIFICATION_TEST_DIR)/docker-compose.yml run -e NOTIFICATION_TEST_ENV=$(NOTIFICATION_TEST_ENV) --remove-orphans -it --name notification-tests tests
+	docker cp notification-tests:/code/notification-tests.xml $(NOTIFICATION_TEST_DIR)
+
+notification-test-clean:
+	docker rm notification-tests
+
 .PHONY: format
 format: $(INSTALL_STAMP)  ##  Sort imports and reformats code
 	$(POETRY) run isort $(TESTS_DIR)

tests/load/Dockerfile

@@ -24,7 +24,7 @@ ENV POETRY_VIRTUALENVS_CREATE=false \
   POETRY_VERSION=1.7.0
 RUN python -m pip install --no-cache-dir --quiet poetry
 COPY ./tests/pyproject.toml ./tests/poetry.lock ./
-RUN poetry install --without dev,integration --no-interaction --no-ansi
+RUN poetry install --without dev,integration,notification --no-interaction --no-ansi
 
 RUN useradd --create-home locust
 WORKDIR /home/locust

tests/notification/Dockerfile

@@ -0,0 +1,65 @@
+FROM python:3-slim
+
+ENV POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_NO_INTERACTION=1 \
+    GECKODRIVER="0.35.0"
+
+ENV PATH="$POETRY_HOME/bin:$PATH"
+
+ENV NOTIFICATION_TEST_ENV ="dev"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    musl-dev \
+    xvfb \
+    xauth \
+    curl \
+    wget \
+    gnupg2 \
+    libgtk-3-0 \
+    libdbus-glib-1-2 \
+    libxt6 \
+    libx11-xcb1 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxfixes3 \
+    libxrender1 \
+    libxext6 \
+    libxrandr2 \
+    libasound2 \
+    libpango-1.0-0 \
+    libpangocairo-1.0-0 \
+    libdrm2 \
+    libgbm1 \
+    libatspi2.0-0 \
+    bzip2 \
+    libglib2.0-0 \
+    libnss3 \
+    libgconf-2-4 \
+    libfontconfig1 \
+    libdbus-glib-1-2 \
+    && apt-get clean -y
+
+# Download and install the latest Firefox release
+RUN wget -O firefox.tar.bz2 "https://download.mozilla.org/?product=firefox-latest&os=linux64&lang=en-US" \
+    && tar -xjf firefox.tar.bz2 -C /opt/ \
+    && rm firefox.tar.bz2 \
+    && ln -s /opt/firefox/firefox /usr/local/bin/firefox
+
+# Install GeckoDriver
+RUN wget https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER}/geckodriver-v${GECKODRIVER}-linux64.tar.gz \
+    && tar -xvzf geckodriver-v${GECKODRIVER}-linux64.tar.gz \
+    && mv geckodriver /usr/local/bin/ \
+    && rm geckodriver-v${GECKODRIVER}-linux64.tar.gz
+
+RUN curl -sSL https://install.python-poetry.org | python3 -
+
+WORKDIR /code
+ADD notification/ /code
+ADD ../poetry.lock /code
+ADD ../pyproject.toml /code
+
+RUN poetry install --only=notification
+
+CMD xvfb-run --auto-servernum --server-args="-screen 0 1920x1080x24" poetry run pytest --driver Firefox --env ${NOTIFICATION_TEST_ENV}

tests/notification/README.md

@@ -0,0 +1,33 @@
+# Autopush Notification Integration Tests
+
+## About
+
+Notifications in Firefox are a crucial part of its functionality. Firefox uses [autopush](https://github.com/mozilla-services/autopush) for this. This directory  contains a set of tests to check the functionality of these notifications.
+
+## Technology
+
+The tests use [Selenium](https://www.selenium.dev/), [pytest](https://docs.pytest.org/en/stable/index.html), [docker](https://www.docker.com/) as well as Firefox.
+
+## Getting Started
+
+Make sure you have installed [docker-compose](https://docs.docker.com/compose/) as well as Docker.
+
+```sh
+docker compose build
+docker compose up server
+NOTIFICATION_TEST_ENV="dev" docker compose run -it tests
+```
+
+You can also use the `Makefile` at the root of the project like so:
+```sh
+NOTIFICATION_TEST_ENV="stage" make notification-test
+```
+
+Be sure to run `make notification-test-clean` between successive test runs.
+
+### Command line options
+
+```NOTIFICATION_TEST_ENV``` : stage, dev, prod. This controls the URL that is set for the push server.
+- stage: wss://autopush.stage.mozaws.net
+- dev: wss://autopush.dev.mozaws.net/
+- prod: wss://push.services.mozilla.com/

tests/notification/conftest.py

@@ -0,0 +1,40 @@
+"""Conftest file for notification tests."""
+
+import logging
+
+import pytest
+from selenium.webdriver.firefox.options import Options as FirefoxOptions
+from selenium.webdriver.firefox.webdriver import WebDriver
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    """CLI Parser options."""
+    parser.addoption("--env", action="store")
+
+
+@pytest.fixture
+def autopush_env(pytestconfig: pytest.Config) -> str:
+    """Autopush websocket URLs."""
+    environment = pytestconfig.getoption("env")
+    urls: dict[str, str] = {
+        "dev": "wss://autopush.dev.mozaws.net/",
+        "stage": "wss://autopush.stage.mozaws.net",
+        "prod": "wss://push.services.mozilla.com/",
+    }
+    logging.info(f"Testing ENVIRONMENT: {environment}")
+    return urls.get(environment, "")
+
+
+@pytest.fixture
+def selenium(selenium: WebDriver) -> WebDriver:
+    """Selenium setup fixture."""
+    selenium.maximize_window()
+    return selenium
+
+
+@pytest.fixture
+def firefox_options(firefox_options: FirefoxOptions, autopush_env: str) -> FirefoxOptions:
+    """Selenium Firefox options fixture."""
+    firefox_options.set_preference("dom.push.serverURL", autopush_env)
+    firefox_options.add_argument("-foreground")
+    return firefox_options

tests/notification/docker-compose.yml

@@ -0,0 +1,18 @@
+services:
+  server:
+    image: "firefoxtesteng/autopush-e2e-test"
+    expose:
+      - "8201"
+    network_mode: host
+    platform: linux/amd64
+
+  tests:
+    environment:
+      SERVER_URL: server
+    build:
+      context: ..
+      dockerfile: notification/Dockerfile
+    depends_on: 
+      - server
+    network_mode: host
+    platform: linux/amd64

tests/notification/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+addopts = --junit-xml=notification-tests.xml -vvv
+log_cli = true
+log_cli_level = info

tests/notification/test_notification.py

@@ -0,0 +1,119 @@
+"""Module containing the Notification test files for autopush-rs."""
+
+import logging
+import time
+
+import imgcompare
+import pytest
+from PIL import ImageGrab
+from PIL.Image import Image
+from selenium.webdriver.common.by import By
+from selenium.webdriver.firefox.webdriver import WebDriver
+
+
+@pytest.fixture
+def images_dir(tmpdir: pytest.Testdir) -> object:
+    """Directory to store the screenshots for testing."""
+    return tmpdir.mkdir("images")
+
+
+@pytest.fixture(autouse=True)
+def setup_page(selenium: WebDriver, images_dir: object) -> Image:
+    """Fixture to setup the test page and take the base screenshot."""
+    selenium.get("localhost:8201")
+    selenium.find_element(By.CSS_SELECTOR, ".container").click()
+    time.sleep(5)  # wait a bit to take the base screenshot
+    base_img: Image = ImageGrab.grab()
+    base_img.save(f"{images_dir}/base_screenshot.jpg")
+    logging.info(images_dir)
+    return base_img
+
+
+@pytest.mark.nondestructive
+def test_basic_notification_by_itself(
+    selenium: WebDriver, images_dir: object, setup_page: Image
+) -> None:
+    """Tests a basic notification with no changes."""
+    el = selenium.find_element(
+        By.CSS_SELECTOR, ".container > p:nth-child(5) > button:nth-child(1)"
+    )
+    el.click()
+    # click allow notification
+    with selenium.context(selenium.CONTEXT_CHROME):
+        button = selenium.find_element(By.CSS_SELECTOR, "button.popup-notification-primary-button")
+        button.click()
+    img: Image = ImageGrab.grab()
+    img.save(f"{images_dir}/screenshot.jpg")
+    # compare images
+    diff = imgcompare.image_diff_percent(setup_page, img)
+    assert diff < 2
+
+
+@pytest.mark.nondestructive
+def test_basic_notification_with_altered_title(selenium: WebDriver, images_dir: object):
+    """Tests a basic notification with a different title."""
+    title_box = selenium.find_element(By.CSS_SELECTOR, "#msg_txt")
+    title_box.send_keys(" testing titles")
+    selenium.find_element(By.CSS_SELECTOR, ".container").click()
+    base_img = ImageGrab.grab()
+    base_img.save(f"{images_dir}/base_screenshot_with_altered_title.jpg")
+    el = selenium.find_element(
+        By.CSS_SELECTOR, ".container > p:nth-child(5) > button:nth-child(1)"
+    )
+    el.click()
+    # click allow notification
+    with selenium.context(selenium.CONTEXT_CHROME):
+        button = selenium.find_element(By.CSS_SELECTOR, "button.popup-notification-primary-button")
+        button.click()
+    selenium.find_element(By.CSS_SELECTOR, ".container").click()
+    img: Image = ImageGrab.grab()
+    img.save(f"{images_dir}/screenshot.jpg")
+    # compare images
+    diff = imgcompare.image_diff_percent(base_img, img)
+    assert diff < 2
+
+
+@pytest.mark.nondestructive
+def test_basic_notification_with_altered_body(selenium: WebDriver, images_dir: object):
+    """Tests a basic notification with an altered notification body."""
+    body_box = selenium.find_element(By.CSS_SELECTOR, "#body_txt")
+    body_box.send_keys(" testing body text")
+    base_img = ImageGrab.grab()
+    el = selenium.find_element(
+        By.CSS_SELECTOR, ".container > p:nth-child(5) > button:nth-child(1)"
+    )
+    el.click()
+    # click allow notification
+    with selenium.context(selenium.CONTEXT_CHROME):
+        button = selenium.find_element(By.CSS_SELECTOR, "button.popup-notification-primary-button")
+        button.click()
+    base_img.save(f"{images_dir}/base_screenshot_with_altered_body.jpg")
+    img: Image = ImageGrab.grab()
+    img.save(f"{images_dir}/screenshot_with_altered_body.jpg")
+    diff = imgcompare.image_diff_percent(base_img, img)
+    assert diff < 2
+
+
+@pytest.mark.nondestructive
+def test_basic_notification_close(selenium: WebDriver, images_dir: object, setup_page: Image):
+    """Tests a basic notification with and then closes it."""
+    el = selenium.find_element(
+        By.CSS_SELECTOR, ".container > p:nth-child(5) > button:nth-child(1)"
+    )
+    el.click()
+    # click allow notification
+    with selenium.context(selenium.CONTEXT_CHROME):
+        button = selenium.find_element(By.CSS_SELECTOR, "button.popup-notification-primary-button")
+        button.click()
+    img: Image = ImageGrab.grab()
+    img.save(f"{images_dir}/screenshot.jpg")
+    # compare images
+    diff = imgcompare.image_diff_percent(setup_page, img)
+    assert diff < 2
+    selenium.find_element(
+        By.CSS_SELECTOR, ".container > p:nth-child(6) > button:nth-child(1)"
+    ).click()
+    closed_notification_img = ImageGrab.grab()
+    closed_notification_img.save(f"{images_dir}/screenshot_close.jpg")
+    diff = imgcompare.image_diff_percent(setup_page, closed_notification_img)
+    assert round(diff, 2) <= 0.1  # assert closed page is less than 1% diff from base