Apply the cookiecutter to h

.cookiecutter/cookiecutter.json

@@ -0,0 +1,53 @@
+{
+  "template": "https://github.com/hypothesis/cookiecutters",
+  "checkout": null,
+  "directory": "pyramid-app",
+  "ignore": [
+      ".docker.env",
+      "Dockerfile",
+      "LICENSE",
+      "bin/make_devdata",
+      "h/db.py",
+      "h/migrations/env.py",
+      "h/templates/.gitkeep",
+      "tests/functional/app_test.py"
+  ],
+  "extra_context": {
+    "name": "h",
+    "package_name": "h",
+    "slug": "h",
+    "short_description": "h is the web app that serves most of the https://hypothes.is/ website, including the web annotations API at https://hypothes.is/api/. The [Hypothesis client](https://github.com/hypothesis/client) is a browser-based annotator that is a client for h's API.",
+    "python_version": "3.11.7",
+    "port": "5000",
+    "github_owner": "hypothesis",
+    "visibility": "public",
+    "copyright_holder": "Hypothesis",
+    "dependabot_pip_interval": "monthly",
+    "dependabot_npm_interval": "monthly",
+    "dependabot_docker_interval": "monthly",
+    "devdata": "yes",
+    "frontend": "yes",
+    "__frontend_typechecking": "yes",
+    "postgres": "yes",
+    "docker": "yes",
+    "linter": "pylint",
+    "__postgres_version": "15.6-alpine",
+    "__postgres_port": "5432",
+    "__docker_namespace": "hypothesis",
+    "__docker_network": "h_default",
+    "__github_url": "https://github.com/hypothesis/h",
+    "__copyright_year": "2022",
+    "__parallel_unit_tests": true,
+    "__format_runner_type": "ubuntu-latest",
+    "__lint_runner_type": "ubuntu-latest-32-cores",
+    "__typecheck_runner_type": "ubuntu-latest",
+    "__tests_runner_type": "ubuntu-latest-32-cores",
+    "__coverage_runner_type": "ubuntu-latest",
+    "__functests_runner_type": "ubuntu-latest",
+    "__gunicorn_bind": "unix:/tmp/gunicorn-web.sock",
+    "__scheduled_workflows": ",Report refresh",
+    "__slack_notifications_workflows": "",
+    "__docker_hub_application_name": "hypothesis",
+    "_checkout": null
+  }
+}

.cookiecutter/includes/.github/workflows/ci/services.yml

@@ -0,0 +1,6 @@
+elasticsearch:
+  image: hypothesis/elasticsearch:elasticsearch7.10
+  ports:
+  - 9200:9200
+  env:
+    discovery.type: single-node

.cookiecutter/includes/.github/workflows/ci/services_functests.yml

@@ -0,0 +1,4 @@
+rabbitmq:
+  image: rabbitmq:3.12-management-alpine
+  ports:
+  - 5672:5672

.cookiecutter/includes/README/head.md

@@ -0,0 +1,11 @@
+## Community
+
+Join us on Slack ([request an invite](https://slack.hypothes.is) or [log in once you've created an account](https://hypothesis-open.slack.com/)).
+
+If you'd like to contribute to the project, you should also [subscribe](mailto:[email protected])
+to the [development mailing list](https://groups.google.com/a/list.hypothes.is/forum/#!forum/dev)
+and read our [Contributor's guide](https://h.readthedocs.io/en/latest/developing/).
+Then consider getting started on one of the issues that are ready for work.
+
+Please note that this project is released with a [Contributor Code of Conduct](https://github.com/hypothesis/.github/blob/main/CODE_OF_CONDUCT.md).
+By participating in this project you agree to abide by its terms.

.cookiecutter/includes/conf/gunicorn-dev.conf.py/tail

@@ -0,0 +1,13 @@
+
+
+from os import environ
+
+max_requests = 100
+max_requests_jitter = 10
+
+
+if "GUNICORN_CERTFILE" in environ:
+    certfile = environ["GUNICORN_CERTFILE"]
+
+if "GUNICORN_KEYFILE" in environ:
+    keyfile = environ["GUNICORN_KEYFILE"]

.cookiecutter/includes/conf/gunicorn.conf.py/tail

@@ -0,0 +1,8 @@
+
+
+from os import environ
+
+workers = environ["WEB_NUM_WORKERS"]
+
+max_requests = environ.get("GUNICORN_MAX_REQUESTS", 500_000)
+max_requests_jitter = environ.get("GUNICORN_MAX_REQUESTS_JITTER", 100_000)

.cookiecutter/includes/conf/supervisord-dev.conf.json

@@ -0,0 +1,27 @@
+{
+  "programs": {
+    "init_db": {
+        "command": "python3 -m h.scripts.init_db --create --stamp",
+        "startsecs": 0
+    },
+    "init_elasticsearch": {
+        "command": "python3 -m h.scripts.init_elasticsearch conf/development.ini",
+        "startsecs": 0
+    },
+    "web": {
+      "command": "newrelic-admin run-program gunicorn --paste conf/development.ini --config conf/gunicorn-dev.conf.py",
+      "autostart": "%(ENV_ENABLE_WEB)s"
+    },
+    "websocket": {
+      "command": "gunicorn --paste conf/websocket-dev.ini --config conf/gunicorn-websocket-dev.conf.py",
+      "autostart": "%(ENV_ENABLE_WEBSOCKET)s"
+    },
+    "worker": {
+      "command": "newrelic-admin run-program bin/hypothesis --dev celery worker --loglevel=INFO",
+      "autostart": "%(ENV_ENABLE_WORKER)s"
+    },
+    "assets": {
+      "command": "node_modules/.bin/gulp watch"
+    }
+  }
+}

.cookiecutter/includes/conf/supervisord.conf.json

@@ -0,0 +1,26 @@
+{
+  "programs": {
+    "nginx": {
+      "command": "nginx",
+      "autostart": "%(ENV_ENABLE_NGINX)s"
+
+    },
+    "web": {
+      "command": "newrelic-admin run-program gunicorn --paste conf/production.ini --config conf/gunicorn.conf.py",
+      "autostart": "%(ENV_ENABLE_WEB)s"
+    },
+    "websocket": {
+      "command": "gunicorn --paste conf/websocket.ini --config conf/gunicorn-websocket.conf.py",
+      "autostart": "%(ENV_ENABLE_WEBSOCKET)s"
+    },
+    "websocket-monolithic": {
+      "command": "gunicorn --paste conf/websocket-monolithic.ini --config conf/gunicorn-websocket-monolithic.conf.py",
+      "autostart": "%(ENV_ENABLE_WEBSOCKET_MONOLITHIC)s",
+      "process_name": "websocket"
+    },
+    "worker": {
+      "command": "newrelic-admin run-program hypothesis celery worker --loglevel=INFO",
+      "autostart": "%(ENV_ENABLE_WORKER)s"
+    }
+  }
+}

.cookiecutter/includes/coverage/omit

@@ -0,0 +1,3 @@
+"h/cli/*",
+"h/pshell.py",
+"h/scripts/init_elasticsearch.py",

.cookiecutter/includes/development.ini

@@ -0,0 +1,14 @@
+pyramid.debug_all: True
+pyramid.reload_templates: True
+
+h.bouncer_url: http://localhost:8000
+h.client_rpc_allowed_origins: http://localhost:8001 https://localhost:48001
+h.client_url: {current_scheme}://{current_host}:3001/hypothesis
+h.websocket_url: ws://localhost:5001/ws
+
+h.debug: True
+h.reload_assets: True
+
+es.url: http://localhost:9200
+
+secret_key: notverysecretafterall

.cookiecutter/includes/docker-compose/postgres.yml

@@ -0,0 +1,2 @@
+networks:
+  - dbs

.cookiecutter/includes/docker-compose/services.yml

@@ -0,0 +1,15 @@
+elasticsearch:
+  image: hypothesis/elasticsearch:elasticsearch7.10
+  ports:
+    - '127.0.0.1:9200:9200'
+  healthcheck:
+      test: curl --fail --silent http://localhost:9200 >/dev/null
+      interval: 3s
+      start_period: 1m
+  environment:
+    - discovery.type=single-node
+rabbit:
+  image: rabbitmq:3.12-management-alpine
+  ports:
+    - '127.0.0.1:5672:5672'
+    - '127.0.0.1:15672:15672'

.cookiecutter/includes/docker-compose/tail.yml

@@ -0,0 +1,6 @@
+networks:
+  # This external network allows FDW connections between H, LMS and report DBs.
+  # To avoid having unnecessary dependencies between the projects
+  # the network is created with `docker network crate dbs` in each project's Makefile (make services)
+  dbs:
+    external: true

.cookiecutter/includes/gitignore

@@ -0,0 +1,19 @@
+.hypothesis/
+
+# Pyramid debug mailer
+mail/
+
+# development TLS cert/key
+.tlscert.pem
+.tlscsr.pem
+.tlskey.pem
+
+# See https://yarnpkg.com/getting-started/qa#which-files-should-be-gitignored.
+# This is the "If you're not using Zero-Installs" list.
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions

.cookiecutter/includes/hacking/setting_up.md

@@ -0,0 +1 @@
+See the [Contributor's guide](https://h.readthedocs.io/en/latest/developing/) for further instructions on setting up a development environment and contributing to h.

.cookiecutter/includes/mypy/ignore_errors_modules/tail

@@ -0,0 +1,4 @@
+# Disable type-checking in h (by ignoring all files) for now.
+# This is just to get `make typecheck` passing.
+# In future we should remove this and fix h's type-checking errors.
+"h.*",

.cookiecutter/includes/pycodestyle/ignores

@@ -0,0 +1 @@
+E712,

.cookiecutter/includes/pylint/disables

@@ -0,0 +1,6 @@
+"missing-param-doc",
+"fixme",
+
+# This is intermittently causing false-positives with SQLAlchemy's Mapped, for example:
+# h/models/token.py:57:10: E1136: Value 'Mapped' is unsubscriptable (unsubscriptable-object)
+"unsubscriptable-object",

.cookiecutter/includes/pytest/filterwarnings

@@ -0,0 +1,10 @@
+"ignore:^pkg_resources is deprecated as an API:DeprecationWarning:deform",
+"ignore:^The behavior of AcceptLanguageValidHeader\\.__iter__ is currently maintained for backward compatibility, but will change in the future.$:DeprecationWarning:webob.acceptparse",
+"ignore:^Use of \\.\\. or absolute path in a resource path is not allowed and will raise exceptions in a future release\\.$:DeprecationWarning:pkg_resources",
+"ignore:^Use of \\.\\. or absolute path in a resource path is not allowed and will raise exceptions in a future release\\.$:DeprecationWarning:pyramid",
+
+# https://foss.heptapod.net/python-libs/passlib/-/issues/148
+"ignore:^'crypt' is deprecated and slated for removal in Python 3.13:DeprecationWarning:passlib",
+
+# Usage of deprecated method of urllib3 within the elasticsearch library
+"ignore:^HTTPResponse.getheaders\\(\\) is deprecated and will be removed in urllib3 v2.1.0. Instead access HTTPResponse.headers directly.$:DeprecationWarning:elasticsearch",

.cookiecutter/includes/requirements/dev.in

@@ -0,0 +1 @@
+ipdb

.cookiecutter/includes/requirements/functests.in

@@ -0,0 +1 @@
+pytest-reverse

.cookiecutter/includes/requirements/prod.in

@@ -0,0 +1,48 @@
+sentry-sdk
+h-pyramid-sentry
+h-matchers
+h-api
+h-assets
+data_tasks
+pyramid-sanity
+PyJWT
+bcrypt
+bleach
+celery
+certifi  # Required to connect to Elasticsearch over SSL.
+cffi
+click
+deform
+elasticsearch==6.8.2
+elasticsearch-dsl==6.4.0
+gevent >= 1.4.0
+importlib_resources
+itsdangerous
+jsonschema
+kombu
+markdown
+oauthlib
+packaging
+passlib
+psycogreen
+pycryptodomex
+pyparsing >= 2.1.5
+pyramid-exclog
+markupsafe
+jinja2
+pyramid-jinja2
+pyramid-services
+pyramid-mailer
+pyramid-retry
+python-dateutil
+python-slugify < 8.1.0
+pytz
+rfc3339-validator  # Required for date-time format checking in jsonschema
+supervisor >= 4.0.0
+sqlparse
+tabulate
+transaction
+venusian
+wsaccel
+ws4py >= 0.4.0
+zope.interface

.cookiecutter/includes/requirements/tests.in

@@ -0,0 +1,2 @@
+hypothesis
+pytest-mock

.cookiecutter/includes/tests/pylint/disables

@@ -0,0 +1,12 @@
+"fixme",
+"too-many-positional-arguments",
+
+# h-matchers triggers this when called without brackets
+"no-value-for-parameter",
+
+# This is intermittently causing a false-positive:
+# tests/unit/h/models/document/_uri_test.py:17:15: W0143: Comparing against a callable, did you omit the parenthesis? (comparison-with-callable)
+# ...if we add a # pylint:disable comment for it then it starts intermittently causing:
+# tests/unit/h/models/document/_uri_test.py:17:0: I0021: Useless suppression of 'comparison-with-callable' (useless-suppression)
+# Just disable the whole rule globally.
+"comparison-with-callable",

.cookiecutter/includes/tox/commands

@@ -0,0 +1,2 @@
+docs: sphinx-autobuild -qT --open-browser -b dirhtml -d {envdir}/doctrees docs {envdir}/html
+checkdocs: sphinx-build -qTWn -b dirhtml -d {envdir}/doctrees docs {envdir}/html

.cookiecutter/includes/tox/passenv

@@ -0,0 +1,15 @@
+dev: BOUNCER_URL
+dev: CLIENT_OAUTH_ID
+dev: CLIENT_RPC_ALLOWED_ORIGINS
+dev: CLIENT_URL
+dev: GOOGLE_ANALYTICS_MEASUREMENT_ID
+dev: SENTRY_DSN_CLIENT
+dev: SENTRY_DSN_FRONTEND
+dev: SENTRY_ENVIRONMENT
+dev: USE_HTTPS
+dev: NEW_RELIC_APP_NAME
+dev: NODE_ENV
+dev: PROXY_AUTH
+{tests,functests}: DATABASE_URL
+{tests,functests}: ELASTICSEARCH_URL
+functests: BROKER_URL

.cookiecutter/includes/tox/setenv

@@ -1 +1,15 @@
-SQLALCHEMY_SILENCE_UBER_WARNING=1
+REPORT_FDW_USERS=lms-fdw report-fdw
+dev: AUTHORITY = {env:AUTHORITY:localhost}
+dev: HTTP_HOST = {env:HTTP_HOST:localhost:5000}
+dev: APP_URL = {env:APP_URL:http://localhost:5000}
+dev: WEBSOCKET_URL = {env:WEBSOCKET_URL:ws://localhost:5001/ws}
+dev: ENABLE_WEB = {env:ENABLE_WEB:true}
+dev: ENABLE_WEBSOCKET = {env:ENABLE_WEBSOCKET:true}
+dev: ENABLE_WORKER = {env:ENABLE_WORKER:true}
+dev: JWE_SECRET_LMS = {env:JWE_SECRET_LMS:SUPER_SECRET}
+dev: H_API_AUTH_COOKIE_SECRET_KEY = {env:H_API_AUTH_COOKIE_SECRET_KEY:"dev_h_api_auth_cookie_secret_key"}
+dev: H_API_AUTH_COOKIE_SALT = {env:H_API_AUTH_COOKIE_SALT:"dev_h_api_auth_cookie_salt"}
+dev: REPLICA_DATABASE_URL = {env:DATABASE_URL:postgresql://postgres@localhost/postgres}
+tests: ELASTICSEARCH_INDEX = {env:ELASTICSEARCH_INDEX:hypothesis-tests}
+functests: ELASTICSEARCH_INDEX = {env:ELASTICSEARCH_INDEX:hypothesis-functests}
+{tests,functests}: AUTHORITY = {env:AUTHORITY:example.com}

.docker.env

@@ -0,0 +1,18 @@
+APP_URL=http://localhost:5000
+AUTHORITY=localhost
+BROKER_URL=amqp://guest:guest@rabbit:5672//
+CLIENT_URL=http://localhost:3001/hypothesis
+DATABASE_URL=postgresql://postgres@postgres/postgres
+ELASTICSEARCH_URL=http://elasticsearch:9200
+NEW_RELIC_APP_NAME=h (dev)
+SECRET_KEY=notasecret
+ENABLE_NGINX=true
+ENABLE_WEB=true
+WEB_NUM_WORKERS=2
+ENABLE_WEBSOCKET=false
+ENABLE_WEBSOCKET_MONOLITHIC=true
+WEBSOCKET_NUM_WORKERS=2
+WEBSOCKET_URL=ws://localhost:5001/ws
+ENABLE_WORKER=true
+H_API_AUTH_COOKIE_SECRET_KEY=dev_h_api_auth_cookie_secret_key
+H_API_AUTH_COOKIE_SALT=dev_h_api_auth_cookie_salt