Merge branch 'sictiru' of github.com:samuelclay/NewsBlur into sictiru

.gitignore

@@ -19,6 +19,7 @@ certbot.conf
 task_env.py
 app_env.py
 data/
+api/ip_addresses.txt
 .prom_cache
 config/certificates
 **/*.xcuserstate
@@ -53,10 +54,12 @@ docker/haproxy/haproxy.consul.cfg
 docker/nginx/nginx.consul.conf
 docker/prometheus/prometheus.yml
 docker/redis/redis_replica.conf
+docker/redis/redis_*_replica.conf
 docker/postgres/postgres.conf
 
 # Local configuration file (sdk path, etc)
 /originals
+/node/originals
 media/safari/NewsBlur.safariextz
 
 # IDE files
@@ -66,3 +69,4 @@ media/safari/NewsBlur.safariextz
 *.tfstate*
 .terraform*
 grafana.ini
+apps/api/ip_addresses.txt

.vscode/settings.json

@@ -1,12 +1,16 @@
 {
-    "python.linting.enabled": true,
-    "python.linting.pylintEnabled": false,
-    "python.linting.flake8Enabled": true,
-    "python.linting.pylamaEnabled": false,
-    "python.linting.flake8Args": [
-        "--ignore=E501,W293,W503,W504,E302,E722,E226,E221,E402,E401"
+    "isort.args": [
+        "--profile",
+        "black"
     ],
-    "python.pythonPath": "~/.virtualenvs/newsblur3/bin/python",
+    // "python.linting.enabled": true,
+    // "python.linting.pylintEnabled": false,
+    // "python.linting.flake8Enabled": true,
+    // "python.linting.pylamaEnabled": false,
+    // "python.linting.flake8Args": [
+    //     "--ignore=E501,W293,W503,W504,E302,E722,E226,E221,E402,E401"
+    // ],
+    // "python.pythonPath": "~/.virtualenvs/newsblur/bin/python",
     "editor.bracketPairColorization.enabled": true,
     "editor.guides.bracketPairs": "active",
     "git.ignoreLimitWarning": true,
@@ -27,15 +31,12 @@
         "docker/volumes": true,
         "requirements.txt": true, // It's just a symlink to config/requirements.txt, which has git history
     },
-    "python.formatting.blackArgs": [
-        "--line-length=110",
-        "--skip-string-normalization"
-    ],
+    // "python.formatting.blackArgs": [
+    //     "--line-length=110",
+    //     "--skip-string-normalization"
+    // ],
     "files.associations": {
         "*.yml": "ansible"
     },
-    "nrf-connect.toolchain.path": "${nrf-connect.toolchain:1.9.1}",
-    "C_Cpp.default.configurationProvider": "nrf-connect",
-    "editor.formatOnSave": false,
     "ansible.python.interpreterPath": "/opt/homebrew/bin/python3",
 }

Maintenance.md

@@ -176,3 +176,17 @@ Provision a new redis server, replicate the data, take newsblur down for mainten
    aps -l db-redis-story1,db-redis-story2 -t consul
    make maintenance_off
    make task
+
+### Switching to a new postgres server
+
+   # Old
+   docker exec -it -u postgres postgres psql -c "SELECT pg_start_backup('label', true)"
+   # New
+   ## Install `openssh-client` and `rsync`
+   docker stop postgres
+   rsync -Pav --stats --progress db-postgres.service.consul:/srv/newsblur/docker/volumes/postgres/data /srv/newsblur/docker/volumes/postgres/ --exclude postmaster.pid
+   docker start postgres
+   # New
+   docker exec -it -u postgres postgres /usr/lib/postgresql/13/bin/pg_ctl -D /var/lib/postgresql/data promote
+   # Old
+   docker exec -it -u postgres postgres psql -c "SELECT pg_stop_backup()"

Makefile

@@ -115,7 +115,8 @@ inventory:
 	./ansible/utils/generate_inventory.py
 oldinventory:
 	OLD=1 ./ansible/utils/generate_inventory.py
-
+hinventory:
+	./ansible/utils/generate_hetzner_inventory.py
 # Docker
 pull:
 	docker pull newsblur/newsblur_python3
@@ -184,7 +185,7 @@ maintenance_off:
 
 # Provision
 firewall:
-	ansible-playbook ansible/all.yml -l db --tags firewall
+	ansible-playbook ansible/all.yml -l db --tags ufw
 oldfirewall:
 	ANSIBLE_CONFIG=/srv/newsblur/ansible.old.cfg ansible-playbook ansible/all.yml  -l db --tags firewall
 repairmongo:

ansible.cfg

@@ -2,7 +2,7 @@
 host_key_checking = False
 remote_user = nb
 retry_files_enabled = False
-inventory = /srv/newsblur/ansible/inventories/digital_ocean.ini, /srv/newsblur/ansible/inventories/digital_ocean.yml
+inventory = /srv/newsblur/ansible/inventories/digital_ocean.ini, /srv/newsblur/ansible/inventories/digital_ocean.yml, /srv/newsblur/ansible/inventories/hetzner.ini, /srv/newsblur/ansible/inventories/hetzner.yml
 private_key_file = /srv/secrets-newsblur/keys/docker.key
 remote_tmp = ~/.ansible/tmp
 forks = 20

ansible/inventories/.gitignore

@@ -1 +1 @@
-*.ini
+digital_ocean*.ini

ansible/inventories/digital_ocean.yml

@@ -32,7 +32,9 @@ groups:
   search: inventory_hostname.startswith('db-elasticsearch')
   elasticsearch: inventory_hostname.startswith('db-elasticsearch')
   redis: inventory_hostname.startswith('db-redis')
-  redis_story: inventory_hostname.startswith('db-redis-story')
+  redis_story: inventory_hostname.startswith('db-redis-story') or inventory_hostname.startswith('hdb-redis-story')
+  redis_session: inventory_hostname.startswith('db-redis-session') or inventory_hostname.startswith('hdb-redis-session')
+  redis_user: inventory_hostname.startswith('db-redis-user') or inventory_hostname.startswith('hdb-redis-user')
   postgres: inventory_hostname.startswith('db-postgres')
   mongo: inventory_hostname.startswith('db-mongo') and not inventory_hostname.startswith('db-mongo-analytics')
   mongo_analytics: inventory_hostname.startswith('db-mongo-analytics')

ansible/inventories/hetzner.ini

@@ -0,0 +1 @@
+/srv/secrets-newsblur/configs/hetzner.ini

ansible/inventories/hetzner.yml

@@ -0,0 +1,60 @@
+plugin: constructed
+strict: False
+
+groups:
+
+  hall: inventory_hostname.startswith('h')
+
+  haproxy: inventory_hostname.startswith('hwww')
+
+  app: inventory_hostname.startswith('happ')
+  happ: inventory_hostname.startswith('happ')
+  web: inventory_hostname.startswith('happ')
+  hweb: inventory_hostname.startswith('happ')
+  django: inventory_hostname.startswith('happ-web')
+  hdjango: inventory_hostname.startswith('happ-web')
+  refresh: inventory_hostname.startswith('happ-refresh')
+  hrefresh: inventory_hostname.startswith('happ-refresh')
+  count: inventory_hostname.startswith('happ-count')
+  hcount: inventory_hostname.startswith('happ-count')
+  push: inventory_hostname.startswith('happ-push')
+  hpush: inventory_hostname.startswith('happ-push')
+  blogs: inventory_hostname.startswith('blog')
+
+  node: inventory_hostname.startswith('hnode')
+  hnode: inventory_hostname.startswith('hnode')
+  node_socket: inventory_hostname.startswith('hnode-socket')
+  hnode_socket: inventory_hostname.startswith('hnode-socket')
+  node_images: inventory_hostname.startswith('hnode-images')
+  hnode_images: inventory_hostname.startswith('hnode-images')
+  node_text: inventory_hostname.startswith('hnode-text')
+  hnode_text: inventory_hostname.startswith('hnode-text')
+  node_page: inventory_hostname.startswith('hnode-page')
+  hnode_page: inventory_hostname.startswith('hnode-page')
+  node_favicons: inventory_hostname.startswith('hnode-favicons')
+  hnode_favicons: inventory_hostname.startswith('hnode-favicons')
+
+  # debugs: inventory_hostname.startswith('hdebug')
+
+  htask: inventory_hostname.startswith('htask')
+  task: inventory_hostname.startswith('htask')
+  celery: inventory_hostname.startswith('htask-celery')
+  work: inventory_hostname.startswith('htask-work')
+
+  staging: inventory_hostname.startswith('hstaging')
+
+  hdb: inventory_hostname.startswith('hdb')
+  db: inventory_hostname.startswith('hdb')
+  search: inventory_hostname.startswith('hdb-elasticsearch')
+  elasticsearch: inventory_hostname.startswith('hdb-elasticsearch')
+  redis: inventory_hostname.startswith('hdb-redis')
+  redis_story: inventory_hostname.startswith('hdb-redis-story')
+  redis_session: inventory_hostname.startswith('hdb-redis-session')
+  redis_user: inventory_hostname.startswith('hdb-redis-user')
+  postgres: inventory_hostname.startswith('hdb-postgres')
+  mongo: inventory_hostname.startswith('hdb-mongo') and not inventory_hostname.startswith('hdb-mongo-analytics')
+  mongo_analytics: inventory_hostname.startswith('hdb-mongo-analytics')
+  consul: inventory_hostname.startswith('hdb-consul')
+  hconsul: inventory_hostname.startswith('hdb-consul')
+  metrics: inventory_hostname.startswith('hdb-metrics')
+  sentry: inventory_hostname.startswith('hdb-sentry')

ansible/playbooks/deploy_app.yml

@@ -127,6 +127,9 @@
         - name: Reload gunicorn
           command: "kill -HUP {{ psaux.stdout }}"
       when: not pulled.changed
+      rescue:
+        - name: Restart Docker Container
+          command: "docker restart newsblur_web"
       tags:
         - static
 

ansible/playbooks/deploy_redis.yml

@@ -6,16 +6,23 @@
     - ../env_vars/base.yml
 
   tasks:
+    - name: Extract part of hostname to determine container name
+      set_fact:
+        redis_role: "{{ inventory_hostname.split('-')[2] }}"
+      tags:
+        - never
+        - replicaofnoone
+
     - name: Turning off secondary for redis by deleting redis_replica.conf
       copy:
-        dest: /srv/newsblur/docker/redis/redis_replica.conf
+        dest: "/srv/newsblur/docker/redis/redis_{{ redis_role }}_replica.conf"
         content: ""
       tags:
         - never
         - replicaofnoone
 
     - name: Setting Redis REPLICAOF NO ONE
-      shell: docker exec redis redis-cli REPLICAOF NO ONE
+      shell: docker exec redis-{{ redis_role }} redis-cli REPLICAOF NO ONE
       tags:
         - never
         - replicaofnoone

ansible/playbooks/restart_server.yml

@@ -0,0 +1,6 @@
+---
+- hosts: all
+  become: yes
+  tasks:
+    - name: Restart the server
+      ansible.builtin.reboot:

ansible/playbooks/setup_metrics.yml

@@ -8,16 +8,16 @@
     - motd_role: db
 
   roles:
-    - {role: 'base', tags: 'base'}
-    - {role: 'ufw', tags: 'ufw'}
-    - {role: 'docker', tags: 'docker'}
-    - {role: 'repo', tags: ['repo', 'pull']}
-    - {role: 'dnsmasq', tags: 'dnsmasq'}
-    - {role: 'consul', tags: 'consul'}
-    - {role: 'consul-client', tags: 'consul'}
-    - {role: 'mongo-exporter', tags: 'mongo-exporter'}
-    - {role: 'postgres-exporter', tags: 'postgres-exporter'}
-    - {role: 'redis-exporter', tags: 'redis-exporter'}
-    - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
-    - {role: 'prometheus', tags: ['prometheus', 'metrics']}
-    - {role: 'grafana', tags: ['grafana', 'metrics']}
+    # - {role: 'base', tags: 'base'}
+    # - {role: 'ufw', tags: 'ufw'}
+    # - {role: 'docker', tags: 'docker'}
+    # - {role: 'repo', tags: ['repo', 'pull']}
+    # - {role: 'dnsmasq', tags: 'dnsmasq'}
+    # - {role: 'consul', tags: 'consul'}
+    # - {role: 'consul-client', tags: 'consul'}
+    # - {role: 'mongo-exporter', tags: 'mongo-exporter'}
+    - { role: "postgres-exporter", tags: "postgres-exporter" }
+    - { role: "redis-exporter", tags: "redis-exporter" }
+    - { role: "node-exporter", tags: ["node-exporter", "metrics"] }
+    - { role: "prometheus", tags: ["prometheus", "metrics"] }
+    - { role: "grafana", tags: ["grafana", "metrics"] }

ansible/playbooks/setup_node.yml

@@ -1,7 +1,6 @@
 ---
 - name: SETUP -> node containers
   hosts: node
-  become: true
   vars_files:
     - ../env_vars/base.yml
   vars:

ansible/playbooks/setup_redis.yml

@@ -12,8 +12,8 @@
     - {role: 'docker', tags: 'docker'}
     - {role: 'repo', tags: ['repo', 'pull']}
     - {role: 'dnsmasq', tags: 'dnsmasq'}
-    - {role: 'consul', tags: 'consul'}
-    - {role: 'consul-client', tags: 'consul'}
+    # - {role: 'consul', tags: 'consul'}
+    # - {role: 'consul-client', tags: 'consul'}
     - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
     - {role: 'redis', tags: 'redis'}
     - {role: 'flask_metrics', tags: ['flask-metrics', 'metrics', 'flask_metrics']}

ansible/playbooks/setup_staging.yml

@@ -23,5 +23,5 @@
     - {role: 'nginx', tags: 'nginx'}
     - {role: 'node', tags: 'node'}
     - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
-    - {role: 'prometheus', tags: ['prometheus', 'metrics']}
-    - {role: 'grafana', tags: ['grafana', 'metrics']}
+    # - {role: 'prometheus', tags: ['prometheus', 'metrics']}
+    # - {role: 'grafana', tags: ['grafana', 'metrics']}

ansible/playbooks/setup_task.yml

@@ -8,13 +8,12 @@
     - motd_role: task
   roles:
     - {role: 'base', tags: 'base'}
-    - {role: 'ufw', tags: 'ufw'}
     - {role: 'docker', tags: 'docker'}
     - {role: 'repo', tags: ['repo', 'pull']}
     - {role: 'dnsmasq', tags: 'dnsmasq'}
     - {role: 'consul', tags: 'consul'}
     - {role: 'consul-client', tags: 'consul'}
     - {role: 'apns', tags: 'apns'}
-    # - {role: 'netdata', tags: 'netdata'}
     - {role: 'node-exporter', tags: ['node-exporter', 'metrics']}
     - {role: 'celery_task', tags: 'celery'}
+    - {role: 'ufw', tags: 'ufw'}

ansible/roles/backups/tasks/main.yml

@@ -103,14 +103,15 @@
         become: yes
         command:
           docker run --rm --name=pg_basebackup --network=host -e POSTGRES_PASSWORD=newsblur -v /srv/newsblur/docker/volumes/postgres/data:/var/lib/postgresql/data postgres:13 pg_basebackup -h db-postgres.service.nyc1.consul -p 5432 -U newsblur -D /var/lib/postgresql/data -Fp -R -Xs -P -c fast
+
       - name: Create Postgres docker volumes with correct permissions
         become: yes
         file:
-          path: "{{ item  }}"
+          path: "{{ item }}"
           state: directory
           recurse: yes
-          owner: "{{ ansible_effective_user_id|int }}"
-          group: "{{ ansible_effective_group_id|int }}"  
+          owner: 999
+          group: 999
         with_items:
           - /srv/newsblur/docker/volumes/postgres/archive
           - /srv/newsblur/docker/volumes/postgres/backups
@@ -129,7 +130,7 @@
       - name: pg_ctl promote
         become: yes
         command:
-          docker exec -it postgres su - postgres -c "/usr/lib/postgresql/13/bin/pg_ctl -D /var/lib/postgresql/data promote"
+          docker exec postgres su - postgres -c "/usr/lib/postgresql/13/bin/pg_ctl -D /var/lib/postgresql/data promote"
     # when: (inventory_hostname | regex_replace('[0-9]+', '')) in ['db-postgres-secondary']
     tags:
       - never

ansible/roles/base/tasks/main.yml

@@ -6,6 +6,17 @@
   tags: packages
   # ignore_errors: yes
 
+- name: whoami
+  debug:
+    var: ansible_user_id
+  tags: whoami
+
+- name: Set timezone
+  become: yes
+  ansible.builtin.timezone:
+    name: 'America/New_York'
+  tags: timezone
+
 - name: Copy zshrc
   template: 
     src: zshrc.txt.j2
@@ -20,10 +31,20 @@
   become: yes
 
 - name: Cloning oh-my-zsh
-  git:
-    repo: https://github.com/robbyrussell/oh-my-zsh
-    dest: /home/nb/.oh-my-zsh
-    force: yes
+  block:
+    - name: Cloning oh-my-zsh
+      git:
+        repo: https://github.com/robbyrussell/oh-my-zsh
+        dest: /home/nb/.oh-my-zsh
+        force: yes
+  rescue:
+    - name: chown oh-my-zsh
+      become: yes
+      file:
+        path: /home/nb/.oh-my-zsh
+        owner: nb
+        group: nb
+        recurse: yes
 
 - name: Copy toprc
   copy: src=toprc.txt dest=~/.toprc

ansible/roles/base/templates/zshrc.txt.j2

@@ -43,7 +43,7 @@ alias cd..='cd ..'
 
 alias smtp='python -m smtpd -n -c DebuggingServer 127.0.0.1:1025'
 alias tlnb='echo "----------------\n"; tail -f /srv/newsblur/logs/newsblur.log'
-alias sp='sudo docker exec -it {% if 'task' in inventory_hostname %}{{  inventory_hostname|regex_replace('\d+', '') }}{% else %}newsblur_web{% endif %} python manage.py shell_plus'
+alias sp='sudo docker exec -it {% if 'task' in inventory_hostname %}{{  inventory_hostname|regex_replace('\-?\d+', '')|regex_replace('htask', 'task')|regex_replace('happ', 'app') }}{% else %}newsblur_web{% endif %} python manage.py shell_plus'
 alias dps='sudo docker ps -a'
 alias cdnb='cd /srv/newsblur'
 alias sshdo=/srv/newsblur/utils/ssh.sh

ansible/roles/celery_task/tasks/main.yml

@@ -43,7 +43,7 @@
       max-size: 100m
     healthcheck: 
       # test: celery inspect ping -A newsblur_web -d celery@$HOSTNAME
-      test: bash -c "(($(date +%s) - $(stat /srv/newsblur/logs/newsblur.log -c %Y) < 120)) && exit 0 || exit 1"
+      test: bash -c "(($(date +%s) - $(stat /srv/newsblur/logs/newsblur.log -c %Y) < 600)) && exit 0 || exit 1"
       interval: 60s
       timeout: 10s
       retries: 3