diff --git a/.gitignore b/.gitignore index 391d32d6e..4b9aaea27 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,6 @@ docs/_build/ # PyBuilder target/ + +# Virtualenv +.env/ diff --git a/.travis.yml b/.travis.yml index 5dfdb5587..837e16675 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,20 @@ matrix: - mysql - postgresql - redis-server + - python: 3.6 + env: TOXENV=py36 + services: + - docker + - mysql + - postgresql + - redis-server + - python: 3.7 + env: TOXENV=py37 + services: + - docker + - mysql + - postgresql + - redis-server install: - pip install -U tox wheel codecov diff --git a/README.md b/README.md index 478a1aff8..438c89730 100644 --- a/README.md +++ b/README.md @@ -43,5 +43,3 @@ $ pip install frontera Join our Google group at https://groups.google.com/a/scrapinghub.com/forum/#!forum/frontera or check GitHub issues and pull requests. - - diff --git a/frontera/contrib/backends/hbase/utils.py b/frontera/contrib/backends/hbase/utils.py index 0f58a55cb..6dc862e92 100644 --- a/frontera/contrib/backends/hbase/utils.py +++ b/frontera/contrib/backends/hbase/utils.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from happybase import Batch -from thriftpy.transport import TTransportException +from thriftpy2.transport import TTransportException import logging @@ -19,4 +19,4 @@ def send(self): self.logger.exception("Exception happened during batch persistence") self.logger.warning("Cleaning up the batch") self._reset_mutations() - pass \ No newline at end of file + pass diff --git a/requirements/tests.txt b/requirements/tests.txt index 0c63dc53c..0fa21a808 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,3 +1,4 @@ +flaky pytest>=2.6.4 PyMySQL>=0.6.3 psycopg2>=2.5.4 diff --git a/setup.py b/setup.py index df0e30a3c..11bff20f3 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,8 @@ 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', @@ -61,7 +63,8 @@ 'tldextract>=1.5.1', ], 'hbase': [ - 'happybase>=1.0.0' + 'happybase>=1.0.0', + 'thriftpy2' ], 'zeromq': [ 'pyzmq', @@ -91,6 +94,7 @@ "psycopg2>=2.5.4", "scrapy>=0.24", "tldextract>=1.5.1", + 'thriftpy2', "SQLAlchemy>=1.0.0", "cachetools", "mock", diff --git a/tests/test_message_bus.py b/tests/test_message_bus.py index 5293c2445..1addc6ca3 100644 --- a/tests/test_message_bus.py +++ b/tests/test_message_bus.py @@ -4,6 +4,7 @@ from frontera.contrib.messagebus.zeromq import MessageBus as ZeroMQMessageBus from frontera.contrib.messagebus.kafkabus import MessageBus as KafkaMessageBus from frontera.utils.fingerprint import sha1 +from flaky import flaky from kafka import KafkaClient from random import randint from time import sleep @@ -234,6 +235,7 @@ def __init__(self): super(IPv6MessageBusTester, self).__init__(settings) +@flaky def test_zmq_message_bus(): """ Test MessageBus with default settings, IPv6 and Star as ZMQ_ADDRESS