Merge pull request #245 from unicef-drp/feat-logto-file

redirect logs to file

Makefile

@@ -29,7 +29,7 @@ test-azure:
 	@echo "------------------------------------------------------------------"
 	@echo "Run Azure production instance"
 	@echo "------------------------------------------------------------------"
-	@docker compose ${ARGS} up -d azurite django auth worker celery_beat nginx
+	@docker compose ${ARGS} up -d azurite django auth worker celery_beat nginx logrotate
 
 # below commands are executed already from django entrypoint+initialize.py
 # @docker compose ${ARGS} exec -T django python manage.py migrate

deployment/.template.env

@@ -39,3 +39,8 @@ TEGOLA_CONCURRENCY=2
 AZURE_STORAGE=
 # Azure container name
 AZURE_STORAGE_CONTAINER=
+
+# log rotate
+LOGROTATE_COPIES=14
+LOGROTATE_SIZE=100M
+LOGROTATE_INTERVAL=daily

deployment/docker-compose.azure.override.devcontainer.yml

@@ -7,6 +7,13 @@ services:
     ports:
       - "5432:5432"
 
+  redis:
+    volumes:
+      # fix redis permission issue because it's using user redis (1001)
+      # sudo chown -R 1001:1001 ./volumes/tmp_data/redis
+      - ./volumes/tmp_data/redis:/tmp
+      - ./redis/redis.conf:/opt/bitnami/redis/mounted-etc/overrides.conf
+
   worker:
     image: kartoza/georepo_dev
     build:
@@ -19,6 +26,7 @@ services:
       - ./volumes/static:/home/web/static
       - ./volumes/media:/home/web/media
       - ./volumes/tegola_config:/opt/tegola_config
+      - ./volumes/tmp_data/worker:/tmp
     links:
       - db
       - azurite
@@ -35,6 +43,7 @@ services:
       - ./volumes/static:/home/web/static
       - ./volumes/media:/home/web/media
       - ./volumes/tegola_config:/opt/tegola_config
+      - ./volumes/tmp_data/celery_beat:/tmp
 
   dev:
     image: kartoza/georepo_dev
@@ -49,6 +58,8 @@ services:
       - ./volumes/static:/home/web/static
       - ./volumes/media:/home/web/media
       - ./volumes/tegola_config:/opt/tegola_config
+      - ./volumes/tmp_data/django:/tmp
+      - ./volumes/tmp_data:/home/web/logs
     links:
       - db
       - worker
@@ -61,3 +72,8 @@ services:
       - ./volumes/azurite:/opt/azurite/data
     ports:
       - 10000:10000
+
+  logrotate:
+    volumes:
+      - ./volumes/tmp_data:/logs
+      - ./volumes/tmp_data/logrotate:/tmp

deployment/docker-compose.override.devcontainer.yml

@@ -7,6 +7,13 @@ services:
     ports:
       - "5432:5432"
 
+  redis:
+    volumes:
+      # fix redis permission issue because it's using user redis (1001)
+      # sudo chown -R 1001:1001 ./volumes/tmp_data/redis
+      - ./volumes/tmp_data/redis:/tmp
+      - ./redis/redis.conf:/opt/bitnami/redis/mounted-etc/overrides.conf
+
   worker:
     image: kartoza/georepo_dev
     build:
@@ -19,6 +26,7 @@ services:
       - ./volumes/media:/home/web/media
       - ./volumes/tegola_config:/opt/tegola_config
       - ./volumes/layer_tiles:/opt/layer_tiles
+      - ./volumes/tmp_data/worker:/tmp
 
   celery_beat:
     image: kartoza/georepo_dev
@@ -32,6 +40,7 @@ services:
       - ./volumes/media:/home/web/media
       - ./volumes/tegola_config:/opt/tegola_config
       - ./volumes/layer_tiles:/opt/layer_tiles
+      - ./volumes/tmp_data/celery_beat:/tmp
 
   dev:
     image: kartoza/georepo_dev
@@ -46,3 +55,10 @@ services:
       - ./volumes/media:/home/web/media
       - ./volumes/tegola_config:/opt/tegola_config
       - ./volumes/layer_tiles:/opt/layer_tiles
+      - ./volumes/tmp_data/django:/tmp
+      - ./volumes/tmp_data:/home/web/logs
+
+  logrotate:
+    volumes:
+      - ./volumes/tmp_data:/logs
+      - ./volumes/tmp_data/logrotate:/tmp

deployment/docker-compose.override.template.yml

@@ -7,6 +7,13 @@ services:
     ports:
       - "4432:5432"
 
+  redis:
+    volumes:
+      # fix redis permission issue because it's using user redis (1001)
+      # sudo chown -R 1001:1001 ./volumes/tmp_data/redis
+      - ./volumes/tmp_data/redis:/tmp
+      - ./redis/redis.conf:/opt/bitnami/redis/mounted-etc/overrides.conf
+
   dbbackups:
     volumes:
       - ./volumes/backups:/backups
@@ -23,6 +30,8 @@ services:
       - ./volumes/tegola_config:/opt/tegola_config
       - ./volumes/layer_tiles:/opt/layer_tiles
       - ./volumes/arcgis-configs:/home/web/arcgis
+      - ./volumes/tmp_data/django:/tmp
+      - ./volumes/tmp_data:/home/web/logs
 
   worker:
     volumes:
@@ -32,6 +41,7 @@ services:
       - ./volumes/tegola_config:/opt/tegola_config
       - ./volumes/layer_tiles:/opt/layer_tiles
       - ./volumes/arcgis-configs:/home/web/arcgis
+      - ./volumes/tmp_data/worker:/tmp
 
   celery_beat:
     volumes:
@@ -41,6 +51,7 @@ services:
       - ./volumes/tegola_config:/opt/tegola_config
       - ./volumes/layer_tiles:/opt/layer_tiles
       - ./volumes/arcgis-configs:/home/web/arcgis
+      - ./volumes/tmp_data/celery_beat:/tmp
 
   auth:
     build:
@@ -54,6 +65,7 @@ services:
       - ./volumes/tegola_config:/opt/tegola_config
       - ./volumes/layer_tiles:/opt/layer_tiles
       - ./volumes/arcgis-configs:/home/web/arcgis
+      - ./volumes/tmp_data/auth:/tmp
 
   webpack-watcher:
     command: bash -c 'cd /home/web/django_project/dashboard/; npm run dev-watch'
@@ -76,6 +88,7 @@ services:
       - ./volumes/tegola_config:/opt/tegola_config
       - ./volumes/layer_tiles:/opt/layer_tiles
       - ./volumes/arcgis-configs:/home/web/arcgis
+      - ./volumes/tmp_data:/home/web/logs
     ports:
       # for django test server
       - "51102:8080"
@@ -92,10 +105,16 @@ services:
       - ./volumes/layer_tiles:/home/web/layer_tiles
       - ./volumes/nginx_cache:/home/web/nginx_cache
       - ./volumes/arcgis-configs:/home/web/arcgis
+      - ./volumes/tmp_data/nginx:/tmp
     ports:
       - "${HTTP_PORT:-8888}:80"
       - "9001:9001"
     links:
       - auth
       - django
       - worker
+
+  logrotate:
+    volumes:
+      - ./volumes/tmp_data:/logs
+      - ./volumes/tmp_data/logrotate:/tmp

deployment/docker-compose.yml

@@ -11,6 +11,10 @@ volumes:
   nginx-cache:
   backups-data:
   nginx-conf:
+  tmp-data:
+  redis-conf:
+  tmp-logrotate:
+  redis-data:
 
 x-common-django:
   &default-common-django
@@ -70,13 +74,18 @@ x-common-django:
     - tegola-data:/opt/tegola_config
     - layer-tiles:/opt/layer_tiles
     - arcgis-configs:/home/web/arcgis
+    - tmp-data:/tmp
   restart: on-failure
 
 services:
   redis:
     image: bitnami/redis:7.0.2
     environment:
       - REDIS_PASSWORD=${REDIS_PASSWORD:-redis_password}
+    volumes:
+      - redis-data:/bitnami/redis/data
+      - tmp-data:/tmp
+      - redis-conf:/opt/bitnami/redis/mounted-etc/overrides.conf
 
   db:
     image: kartoza/postgis:14-3.2
@@ -123,7 +132,9 @@ services:
   celery_beat:
     <<: *default-common-django
     entrypoint: []
-    command: 'celery -A core beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler'
+    command: 'celery -A core beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler --logfile=/tmp/celery_beat.log'
+    links:
+      - db
 
   auth:
     <<: *default-common-django
@@ -151,7 +162,20 @@ services:
       - layer-tiles:/home/web/layer_tiles
       - arcgis-configs:/home/web/arcgis
       - nginx-cache:/home/web/nginx_cache
+      - tmp-data:/tmp
     links:
       - auth
       - django
       - worker
+
+  logrotate:
+    image: blacklabelops/logrotate:1.3
+    volumes:
+      - tmp-data:/logs
+      - tmp-logrotate:/tmp
+    environment:
+      - LOGS_DIRECTORIES=/logs
+      - LOGROTATE_STATUSFILE=/tmp/logrotate.status
+      - LOGROTATE_COPIES=${LOGROTATE_COPIES:-14}
+      - LOGROTATE_SIZE=${LOGROTATE_SIZE:-100M}
+      - LOGROTATE_INTERVAL=${LOGROTATE_INTERVAL:-daily}

deployment/docker/uwsgi.conf

@@ -11,7 +11,8 @@ workers = 4
 cheaper = 2
 env = DJANGO_SETTINGS_MODULE=core.settings.prod
 # disabled so we run in the foreground for docker
-#daemonize = /tmp/django.log
+# daemonize = /tmp/django.log
+logto = /tmp/django.log
 # reload-os-env reload env for each request
 # disabled because making type error
 # https://github.com/unbit/uwsgi/issues/1432
@@ -22,4 +23,4 @@ memory-report = true
 # increase harakiri timeout to 6mins
 harakiri = 300
 buffer-size = 8192
-disable-logging=True
+# disable-logging=True

deployment/docker/uwsgi_auth.conf

@@ -12,7 +12,8 @@ workers = 4
 cheaper = 2
 env = DJANGO_SETTINGS_MODULE=core.settings.prod
 # disabled so we run in the foreground for docker
-#daemonize = /tmp/django.log
+# daemonize = /tmp/auth.log
+logto = /tmp/auth.log
 # reload-os-env reload env for each request
 # disabled because making type error
 # https://github.com/unbit/uwsgi/issues/1432
@@ -22,4 +23,4 @@ env = DJANGO_SETTINGS_MODULE=core.settings.prod
 # gid = 1000
 memory-report = true
 harakiri = 20
-disable-logging=True
+# disable-logging=True

deployment/nginx/nginx.conf

@@ -1,7 +1,7 @@
 user  root;
 worker_processes  auto;
-error_log  /var/log/nginx/error.log notice;
-pid        /var/run/nginx.pid;
+error_log  /tmp/error.log notice;
+pid        /tmp/nginx.pid;
 
 events {
     worker_connections  1024;
@@ -19,7 +19,7 @@ http {
     log_format  main  '[$time_local] "$request" '
                       '$status $body_bytes_sent "$http_referer" '
                       '"$http_user_agent" "$http_x_forwarded_for"';
-    access_log  /var/log/nginx/access.log  main if=$logme;
+    access_log  /tmp/access.log  main if=$logme;
     sendfile        on;
     #tcp_nopush     on;
     keepalive_timeout  65;

deployment/redis/redis.conf

@@ -0,0 +1 @@
+logfile /tmp/redis.log

django_project/core/settings/project.py

@@ -137,3 +137,8 @@
 EXPORT_DATA_EXPIRY_IN_HOURS = int(os.environ.get(
     'EXPORT_DATA_EXPIRY_IN_HOURS', '48'
 ))
+
+# LOGS Directory
+LOGS_DIRECTORY = os.environ.get(
+    'LOGS_DIRECTORY', '/home/web/logs'
+)