From 59050bf5d2204d586f2c7381711fefcfa6b60d6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Susannah=20Klane=C4=8Dek?= Date: Wed, 23 Oct 2019 18:37:27 +0200 Subject: [PATCH 1/7] Bump to Python 3.6 --- .travis.yml | 7 +++++++ README.md | 2 -- setup.py | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5dfdb5587..0bbf5ef93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,13 @@ matrix: - mysql - postgresql - redis-server + - python: 3.6 + env: TOXENV=py36 + services: + - docker + - mysql + - postgresql + - redis-server install: - pip install -U tox wheel codecov diff --git a/README.md b/README.md index c8df8bb56..3ad356d7e 100644 --- a/README.md +++ b/README.md @@ -38,5 +38,3 @@ $ pip install frontera Join our Google group at https://groups.google.com/a/scrapinghub.com/forum/#!forum/frontera or check GitHub issues and pull requests. - - diff --git a/setup.py b/setup.py index df0e30a3c..e00c6b88b 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', From f05104e641ede8b177ef48e2abd0f017fc8c7e24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Susannah=20Klane=C4=8Dek?= Date: Wed, 23 Oct 2019 18:52:10 +0200 Subject: [PATCH 2/7] Use thriftpy2 --- frontera/contrib/backends/hbase/utils.py | 4 ++-- setup.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frontera/contrib/backends/hbase/utils.py b/frontera/contrib/backends/hbase/utils.py index 0f58a55cb..6dc862e92 100644 --- a/frontera/contrib/backends/hbase/utils.py +++ b/frontera/contrib/backends/hbase/utils.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from happybase import Batch -from thriftpy.transport import TTransportException +from thriftpy2.transport import TTransportException import logging @@ -19,4 +19,4 @@ def send(self): self.logger.exception("Exception happened during batch persistence") self.logger.warning("Cleaning up the batch") self._reset_mutations() - pass \ No newline at end of file + pass diff --git a/setup.py b/setup.py index e00c6b88b..d029cde90 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,8 @@ install_requires=[ 'six>=1.8.0', 'w3lib>=1.15.0', - 'cityhash>=0.1.7' + 'cityhash>=0.1.7', + 'thriftpy2' ], extras_require={ 'sql': [ From e4885cbf4154eea79949f20fb90605d04b0d6097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Susannah=20Klane=C4=8Dek?= Date: Wed, 23 Oct 2019 19:06:46 +0200 Subject: [PATCH 3/7] Add Python 3.7 --- .travis.yml | 7 +++++++ setup.py | 1 + 2 files changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0bbf5ef93..837e16675 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,13 @@ matrix: - mysql - postgresql - redis-server + - python: 3.7 + env: TOXENV=py37 + services: + - docker + - mysql + - postgresql + - redis-server install: - pip install -U tox wheel codecov diff --git a/setup.py b/setup.py index d029cde90..e5e87805c 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', From 4a4d8b8c814273d76312e580fa72254a54a6b544 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Susannah=20Klane=C4=8Dek?= Date: Wed, 23 Oct 2019 19:13:11 +0200 Subject: [PATCH 4/7] Make test_zmq_message_bus() flaky --- requirements/tests.txt | 1 + tests/test_message_bus.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/requirements/tests.txt b/requirements/tests.txt index 0c63dc53c..0fa21a808 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,3 +1,4 @@ +flaky pytest>=2.6.4 PyMySQL>=0.6.3 psycopg2>=2.5.4 diff --git a/tests/test_message_bus.py b/tests/test_message_bus.py index 5293c2445..1addc6ca3 100644 --- a/tests/test_message_bus.py +++ b/tests/test_message_bus.py @@ -4,6 +4,7 @@ from frontera.contrib.messagebus.zeromq import MessageBus as ZeroMQMessageBus from frontera.contrib.messagebus.kafkabus import MessageBus as KafkaMessageBus from frontera.utils.fingerprint import sha1 +from flaky import flaky from kafka import KafkaClient from random import randint from time import sleep @@ -234,6 +235,7 @@ def __init__(self): super(IPv6MessageBusTester, self).__init__(settings) +@flaky def test_zmq_message_bus(): """ Test MessageBus with default settings, IPv6 and Star as ZMQ_ADDRESS From 4e49f76d209c274f3fbd21d4030c323da478f4e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Susannah=20Klane=C4=8Dek?= Date: Wed, 23 Oct 2019 19:35:00 +0200 Subject: [PATCH 5/7] Add .env to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 391d32d6e..4b9aaea27 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,6 @@ docs/_build/ # PyBuilder target/ + +# Virtualenv +.env/ From c5a70013911b71c3451df573b2208b4aec6aac95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Susannah=20Klane=C4=8Dek?= Date: Fri, 25 Oct 2019 10:58:41 +0200 Subject: [PATCH 6/7] Move thriftpy2 to test_requires --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e5e87805c..e870cb505 100644 --- a/setup.py +++ b/setup.py @@ -43,8 +43,7 @@ install_requires=[ 'six>=1.8.0', 'w3lib>=1.15.0', - 'cityhash>=0.1.7', - 'thriftpy2' + 'cityhash>=0.1.7' ], extras_require={ 'sql': [ @@ -94,6 +93,7 @@ "psycopg2>=2.5.4", "scrapy>=0.24", "tldextract>=1.5.1", + 'thriftpy2', "SQLAlchemy>=1.0.0", "cachetools", "mock", From ebb4d322ff3bf74adba43825c0575cbc539b39c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Susannah=20Klane=C4=8Dek?= Date: Fri, 25 Oct 2019 12:06:01 +0200 Subject: [PATCH 7/7] Also include thriftpy2 as an hbase requirement --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e870cb505..11bff20f3 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,8 @@ 'tldextract>=1.5.1', ], 'hbase': [ - 'happybase>=1.0.0' + 'happybase>=1.0.0', + 'thriftpy2' ], 'zeromq': [ 'pyzmq',