diff --git a/mercado_bitcoin/apis.py b/mercado_bitcoin/apis.py index 2bd435b..d4a0cb0 100644 --- a/mercado_bitcoin/apis.py +++ b/mercado_bitcoin/apis.py @@ -11,7 +11,6 @@ class MercadoBitcoinApi(ABC): - def __init__(self, coin: str) -> None: self.coin = coin self.base_endpoint = "https://www.mercadobitcoin.net/api" @@ -44,17 +43,19 @@ class TradesApi(MercadoBitcoinApi): def _get_unix_epoch(self, date: datetime.datetime) -> int: return int(date.timestamp()) - def _get_endpoint(self, date_from: datetime.datetime = None, date_to: datetime.datetime = None) -> str: + def _get_endpoint( + self, date_from: datetime.datetime = None, date_to: datetime.datetime = None + ) -> str: if date_from and not date_to: unix_date_from = self._get_unix_epoch(date_from) - endpoint = f'{self.base_endpoint}/{self.coin}/{self.type}/{unix_date_from}' + endpoint = f"{self.base_endpoint}/{self.coin}/{self.type}/{unix_date_from}" elif date_from and date_to: if date_from > date_to: raise RuntimeError("date_from cannot be greater than date_to") unix_date_from = self._get_unix_epoch(date_from) unix_date_to = self._get_unix_epoch(date_to) - endpoint = f'{self.base_endpoint}/{self.coin}/{self.type}/{unix_date_from}/{unix_date_to}' + endpoint = f"{self.base_endpoint}/{self.coin}/{self.type}/{unix_date_from}/{unix_date_to}" else: - endpoint = f'{self.base_endpoint}/{self.coin}/{self.type}' + endpoint = f"{self.base_endpoint}/{self.coin}/{self.type}" - return endpoint \ No newline at end of file + return endpoint diff --git a/mercado_bitcoin/checkpoints.py b/mercado_bitcoin/checkpoints.py new file mode 100644 index 0000000..922bdc7 --- /dev/null +++ b/mercado_bitcoin/checkpoints.py @@ -0,0 +1,63 @@ +import datetime + +from pynamodb.models import Model +from pynamodb.attributes import UnicodeAttribute +import logging + +logger = logging.getLogger() +logging.basicConfig(level=logging.INFO) + + +class CheckpointModel(Model): + class Meta: + table_name = "mercado_bitcoin_ingestor_checkpoints" + region = "us-east-1" + + report_id = UnicodeAttribute(hash_key=True) + checkpoint_date = UnicodeAttribute() + + +class DynamoCheckpoints: + def __init__(self, model: CheckpointModel, report_id: str, default_start_date: datetime.date): + self.default_start_date = default_start_date + self.model = model + self.report_id = report_id + self.create_table() + + def create_checkpoint(self, checkpoint_date): + checkpoint = self.model(self.report_id, checkpoint_date=f"{checkpoint_date}") + checkpoint.save() + + def update_checkpoint(self, checkpoint_date): + checkpoint = self.model.get(self.report_id) + checkpoint.checkpoint_date = f"{checkpoint_date}" + checkpoint.save() + + def create_or_update_checkpoint(self, checkpoint_date): + logger.info(f"Saving checkpoint for {self.report_id}: {checkpoint_date}") + if not self.checkpoint_exist: + self.create_checkpoint(checkpoint_date) + else: + self.update_checkpoint(checkpoint_date) + + @property + def checkpoint_exist(self): + try: + return list(self.model.query(self.report_id)) != [] + except KeyError: + logger.warning(f"KeyError: {self.report_id}") + return False + + def create_table(self): + logger.info(f"Creating dynamo table") + if not self.model.exists(): + self.model.create_table(billing_mode="PAY_PER_REQUEST", wait=True) + + def get_checkpoint(self): + if self.checkpoint_exist: + checkpoint = list(self.model.query(self.report_id))[0].checkpoint_date + logger.info(f"Checkpoint found for {self.report_id}: {checkpoint}") + return datetime.datetime.strptime(checkpoint, "%Y-%m-%d").date() + else: + logger.info(f"Checkpoint not found for {self.report_id} using default_start_date") + return self.default_start_date \ No newline at end of file diff --git a/mercado_bitcoin/ingestors.py b/mercado_bitcoin/ingestors.py index 441ba41..4b11c82 100644 --- a/mercado_bitcoin/ingestors.py +++ b/mercado_bitcoin/ingestors.py @@ -3,11 +3,13 @@ from typing import List from mercado_bitcoin.apis import DaySummaryApi +from mercado_bitcoin.checkpoints import DynamoCheckpoints, CheckpointModel class DataIngestor(ABC): - - def __init__(self, writer, coins: List[str], default_start_date: datetime.date) -> None: + def __init__( + self, writer, coins: List[str], default_start_date: datetime.date + ) -> None: self.default_start_date = default_start_date self.coins = coins self.writer = writer @@ -38,7 +40,43 @@ def ingest(self) -> None: class DaySummaryIngestor(DataIngestor): + def ingest(self) -> None: + date = self._load_checkpoint() + if date < datetime.date.today(): + for coin in self.coins: + api = DaySummaryApi(coin=coin) + data = api.get_data(date=date) + self.writer(coin=coin, api=api.type).write(data) + self._update_checkpoint(date + datetime.timedelta(days=1)) + + +class AwsDataIngestor(ABC): + def __init__( + self, writer, coins: List[str], default_start_date: datetime.date + ) -> None: + self.dynamodb_checkpoint = DynamoCheckpoints( + model=CheckpointModel, + report_id=self.__class__.__name__, + default_start_date=default_start_date, + ) + self.default_start_date = default_start_date + self.coins = coins + self.writer = writer + self._checkpoint = self._load_checkpoint() + + def _load_checkpoint(self) -> datetime.date: + return self.dynamodb_checkpoint.get_checkpoint() + + def _update_checkpoint(self, value): + self._checkpoint = value + self.dynamodb_checkpoint.create_or_update_checkpoint(checkpoint_date=self._checkpoint) + + @abstractmethod + def ingest(self) -> None: + pass + +class AwsDaySummaryIngestor(AwsDataIngestor): def ingest(self) -> None: date = self._load_checkpoint() if date < datetime.date.today(): diff --git a/mercado_bitcoin/lambda_function.py b/mercado_bitcoin/lambda_function.py new file mode 100644 index 0000000..83b9cc4 --- /dev/null +++ b/mercado_bitcoin/lambda_function.py @@ -0,0 +1,19 @@ +import datetime + +from mercado_bitcoin.ingestors import AwsDaySummaryIngestor +from mercado_bitcoin.writers import S3Writter +import logging + +logger = logging.getLogger() +logging.basicConfig(level=logging.INFO) + + +def lambda_handler(event, context): + logger.info(f"{event}") + logger.info(f"{context}") + + AwsDaySummaryIngestor( + writer=S3Writter, + coins=["BTC", "ETH", "LTC", "BCH"], + default_start_date=datetime.date(2021, 6, 1), + ).ingest() diff --git a/mercado_bitcoin/main.py b/mercado_bitcoin/main.py index 26d21e2..7224752 100644 --- a/mercado_bitcoin/main.py +++ b/mercado_bitcoin/main.py @@ -2,24 +2,20 @@ import time from schedule import repeat, every, run_pending -from mercado_bitcoin.ingestors import DaySummaryIngestor +from mercado_bitcoin.ingestors import AwsDaySummaryIngestor from mercado_bitcoin.writers import S3Writter if __name__ == "__main__": - day_summary_ingestor = DaySummaryIngestor( + day_summary_ingestor = AwsDaySummaryIngestor( writer=S3Writter, coins=["BTC", "ETH", "LTC", "BCH"], - default_start_date=datetime.date(2021, 6, 1) + default_start_date=datetime.date(2021, 6, 1), ) - @repeat(every(1).seconds) def job(): day_summary_ingestor.ingest() - while True: run_pending() time.sleep(0.5) - - diff --git a/mercado_bitcoin/writers.py b/mercado_bitcoin/writers.py index 8e8a14a..2e968e1 100644 --- a/mercado_bitcoin/writers.py +++ b/mercado_bitcoin/writers.py @@ -15,7 +15,6 @@ def __init__(self, data): class DataWriter: - def __init__(self, coin: str, api: str) -> None: self.api = api self.coin = coin @@ -26,7 +25,7 @@ def _write_row(self, row: str) -> None: with open(self.filename, "a") as f: f.write(row) - def _write_to_file(self, data: [List, dict]): + def _write_to_file(self, data: [List, dict]): if isinstance(data, dict): self._write_row(json.dumps(data) + "\n") elif isinstance(data, List): @@ -56,10 +55,5 @@ def write(self, data: [List, dict]): def _write_file_to_s3(self): self.client.put_object( - Body=self.tempfile, - Bucket="belisco-data-lake-raw", - Key=self.key + Body=self.tempfile, Bucket="belisco-data-lake-raw", Key=self.key ) - - - diff --git a/poetry.lock b/poetry.lock index 7512ab6..02d5619 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,22 @@ +[[package]] +name = "appdirs" +version = "1.4.4" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "argcomplete" +version = "1.12.3" +description = "Bash tab completion for argparse" +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +test = ["coverage", "flake8", "pexpect", "wheel"] + [[package]] name = "backoff" version = "1.10.0" @@ -6,6 +25,28 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "black" +version = "21.6b0" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +appdirs = "*" +click = ">=7.1.2" +mypy-extensions = ">=0.4.3" +pathspec = ">=0.8.1,<1" +regex = ">=2020.1.8" +toml = ">=0.10.1" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.6.0)", "aiohttp-cors (>=0.4.0)"] +python2 = ["typed-ast (>=1.4.2)"] +uvloop = ["uvloop (>=0.15.2)"] + [[package]] name = "boto3" version = "1.17.108" @@ -43,6 +84,19 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "cfn-flip" +version = "1.2.3" +description = "Convert AWS CloudFormation templates between JSON and YAML formats" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +Click = "*" +PyYAML = ">=4.1" +six = "*" + [[package]] name = "chardet" version = "4.0.0" @@ -51,6 +105,49 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "click" +version = "8.0.1" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "durationpy" +version = "0.5" +description = "Module for converting between datetime.timedelta and Go's Duration strings." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "future" +version = "0.18.2" +description = "Clean single-source support for Python 3 and 2" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "hjson" +version = "3.0.2" +description = "Hjson, a user interface for JSON." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "idna" version = "2.10" @@ -67,17 +164,118 @@ category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "kappa" +version = "0.6.0" +description = "A CLI tool for AWS Lambda developers" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +boto3 = ">=1.2.3" +click = ">=5.1" +placebo = ">=0.8.1" +PyYAML = ">=3.11" + +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "pathspec" +version = "0.8.1" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "pep517" +version = "0.10.0" +description = "Wrappers to build Python packages using PEP 517 hooks" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +toml = "*" + +[[package]] +name = "pip-tools" +version = "6.2.0" +description = "pip-tools keeps your pinned dependencies fresh." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +click = ">=7" +pep517 = "*" + +[package.extras] +coverage = ["pytest-cov"] +testing = ["pytest", "pytest-rerunfailures", "pytest-xdist"] + +[[package]] +name = "placebo" +version = "0.9.0" +description = "Make boto3 calls that look real but have no effect" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "pynamodb" +version = "5.1.0" +description = "A Pythonic Interface to DynamoDB" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +botocore = ">=1.12.54" + +[package.extras] +signals = ["blinker (>=1.3,<2.0)"] + [[package]] name = "python-dateutil" -version = "2.8.1" +version = "2.6.1" description = "Extensions to the standard Python datetime module" category = "main" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +python-versions = "*" [package.dependencies] six = ">=1.5" +[[package]] +name = "python-slugify" +version = "5.0.2" +description = "A Python Slugify application that handles Unicode" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +text-unidecode = ">=1.3" + +[package.extras] +unidecode = ["Unidecode (>=1.1.1)"] + +[[package]] +name = "pyyaml" +version = "5.4.1" +description = "YAML parser and emitter for Python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + [[package]] name = "ratelimit" version = "2.2.1" @@ -86,6 +284,14 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "regex" +version = "2021.7.6" +description = "Alternative regular expression module, to replace re." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "requests" version = "2.25.1" @@ -126,6 +332,52 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "text-unidecode" +version = "1.3" +description = "The most basic Text::Unidecode port" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "tqdm" +version = "4.61.2" +description = "Fast, Extensible Progress Meter" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +telegram = ["requests"] + +[[package]] +name = "troposphere" +version = "3.0.1" +description = "AWS CloudFormation creation library" +category = "dev" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +cfn-flip = ">=1.0.2" + +[package.extras] +policy = ["awacs (>=2.0.0)"] + [[package]] name = "urllib3" version = "1.26.6" @@ -139,16 +391,75 @@ brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "werkzeug" +version = "2.0.1" +description = "The comprehensive WSGI web application library." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +watchdog = ["watchdog"] + +[[package]] +name = "wsgi-request-logger" +version = "0.4.6" +description = "Apache-like combined logging for WSGI Web Applications" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "zappa" +version = "0.50.0" +description = "Server-less Python Web Services for AWS Lambda and API Gateway" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +argcomplete = "*" +boto3 = "*" +durationpy = "*" +future = "*" +hjson = "*" +jmespath = "*" +kappa = "0.6.0" +pip-tools = "*" +python-dateutil = "<2.7.0" +python-slugify = "*" +PyYAML = "*" +requests = ">=2.20.0" +six = "*" +toml = "*" +tqdm = "*" +troposphere = "*" +Werkzeug = "*" +wsgi-request-logger = "*" + [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "7d1495b251fb5778be2080270e936d0ff928f073173bddddf90f997e5b9aa9df" +content-hash = "5333658e9f4060bca0b53866e02d7e009c19c9501b049c3cbf9fc8269948e4fc" [metadata.files] +appdirs = [ + {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, + {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, +] +argcomplete = [ + {file = "argcomplete-1.12.3-py2.py3-none-any.whl", hash = "sha256:291f0beca7fd49ce285d2f10e4c1c77e9460cf823eef2de54df0c0fec88b0d81"}, + {file = "argcomplete-1.12.3.tar.gz", hash = "sha256:2c7dbffd8c045ea534921e63b0be6fe65e88599990d8dc408ac8c542b72a5445"}, +] backoff = [ {file = "backoff-1.10.0-py2.py3-none-any.whl", hash = "sha256:5e73e2cbe780e1915a204799dba0a01896f45f4385e636bcca7a0614d879d0cd"}, {file = "backoff-1.10.0.tar.gz", hash = "sha256:b8fba021fac74055ac05eb7c7bfce4723aedde6cd0a504e5326bcb0bdd6d19a4"}, ] +black = [ + {file = "black-21.6b0-py3-none-any.whl", hash = "sha256:dfb8c5a069012b2ab1e972e7b908f5fb42b6bbabcba0a788b86dc05067c7d9c7"}, + {file = "black-21.6b0.tar.gz", hash = "sha256:dc132348a88d103016726fe360cb9ede02cecf99b76e3660ce6c596be132ce04"}, +] boto3 = [ {file = "boto3-1.17.108-py2.py3-none-any.whl", hash = "sha256:484bba256137c2d2f8351175553dee0e888e8bd5872f5406c8984e02715acf4d"}, {file = "boto3-1.17.108.tar.gz", hash = "sha256:10122ff0f942d7400b18b726edaead20600178f8246cb21b40420073350613b5"}, @@ -161,10 +472,31 @@ certifi = [ {file = "certifi-2021.5.30-py2.py3-none-any.whl", hash = "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"}, {file = "certifi-2021.5.30.tar.gz", hash = "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee"}, ] +cfn-flip = [ + {file = "cfn_flip-1.2.3.tar.gz", hash = "sha256:2bed32a1f4dca26dc64178d52511fd4ef778b5ccbcf32559cac884ace75bde6a"}, +] chardet = [ {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, ] +click = [ + {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, + {file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +durationpy = [ + {file = "durationpy-0.5.tar.gz", hash = "sha256:5ef9416b527b50d722f34655becfb75e49228eb82f87b855ed1911b3314b5408"}, +] +future = [ + {file = "future-0.18.2.tar.gz", hash = "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"}, +] +hjson = [ + {file = "hjson-3.0.2-py3-none-any.whl", hash = "sha256:5546438bf4e1b52bc964c6a47c4ed10fa5fba8a1b264e22efa893e333baad2db"}, + {file = "hjson-3.0.2.tar.gz", hash = "sha256:2838fd7200e5839ea4516ece953f3a19892c41089f0d933ba3f68e596aacfcd5"}, +] idna = [ {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, @@ -173,13 +505,118 @@ jmespath = [ {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"}, {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"}, ] +kappa = [ + {file = "kappa-0.6.0-py2-none-any.whl", hash = "sha256:4d6b7b3accce4a0aaaac92b36237a6304f0f2fffbbe3caea3f7c9f52d12c9989"}, + {file = "kappa-0.6.0.tar.gz", hash = "sha256:4b5b372872f25d619e427e04282551048dc975a107385b076b3ffc6406a15833"}, +] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] +pathspec = [ + {file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"}, + {file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"}, +] +pep517 = [ + {file = "pep517-0.10.0-py2.py3-none-any.whl", hash = "sha256:eba39d201ef937584ad3343df3581069085bacc95454c80188291d5b3ac7a249"}, + {file = "pep517-0.10.0.tar.gz", hash = "sha256:ac59f3f6b9726a49e15a649474539442cf76e0697e39df4869d25e68e880931b"}, +] +pip-tools = [ + {file = "pip-tools-6.2.0.tar.gz", hash = "sha256:9ed38c73da4993e531694ea151f77048b4dbf2ba7b94c4a569daa39568cc6564"}, + {file = "pip_tools-6.2.0-py3-none-any.whl", hash = "sha256:77727ef7457d1865e61fe34c2b1439f9b971b570cc232616a22ce82ab89d357d"}, +] +placebo = [ + {file = "placebo-0.9.0.tar.gz", hash = "sha256:03157f8527bbc2965b71b88f4a139ef8038618b346787f20d63e3c5da541b047"}, +] +pynamodb = [ + {file = "pynamodb-5.1.0-py3-none-any.whl", hash = "sha256:8a38fa76522878ef1a8a0b62e6dcc1f883af73182a6c30a050481d316f589d34"}, + {file = "pynamodb-5.1.0.tar.gz", hash = "sha256:7f351d70b9f4da95ea2d7e50299640e4c46c83b7b24bea5daf110acd2e5aef2b"}, +] python-dateutil = [ - {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"}, - {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"}, + {file = "python-dateutil-2.6.1.tar.gz", hash = "sha256:891c38b2a02f5bb1be3e4793866c8df49c7d19baabf9c1bad62547e0b4866aca"}, + {file = "python_dateutil-2.6.1-py2.py3-none-any.whl", hash = "sha256:95511bae634d69bc7329ba55e646499a842bc4ec342ad54a8cdb65645a0aad3c"}, +] +python-slugify = [ + {file = "python-slugify-5.0.2.tar.gz", hash = "sha256:f13383a0b9fcbe649a1892b9c8eb4f8eab1d6d84b84bb7a624317afa98159cab"}, + {file = "python_slugify-5.0.2-py2.py3-none-any.whl", hash = "sha256:6d8c5df75cd4a7c3a2d21e257633de53f52ab0265cd2d1dc62a730e8194a7380"}, +] +pyyaml = [ + {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, + {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, + {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, + {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, + {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, + {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, + {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, + {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, + {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, ] ratelimit = [ {file = "ratelimit-2.2.1.tar.gz", hash = "sha256:af8a9b64b821529aca09ebaf6d8d279100d766f19e90b5059ac6a718ca6dee42"}, ] +regex = [ + {file = "regex-2021.7.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e6a1e5ca97d411a461041d057348e578dc344ecd2add3555aedba3b408c9f874"}, + {file = "regex-2021.7.6-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:6afe6a627888c9a6cfbb603d1d017ce204cebd589d66e0703309b8048c3b0854"}, + {file = "regex-2021.7.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ccb3d2190476d00414aab36cca453e4596e8f70a206e2aa8db3d495a109153d2"}, + {file = "regex-2021.7.6-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:ed693137a9187052fc46eedfafdcb74e09917166362af4cc4fddc3b31560e93d"}, + {file = "regex-2021.7.6-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:99d8ab206a5270c1002bfcf25c51bf329ca951e5a169f3b43214fdda1f0b5f0d"}, + {file = "regex-2021.7.6-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:b85ac458354165405c8a84725de7bbd07b00d9f72c31a60ffbf96bb38d3e25fa"}, + {file = "regex-2021.7.6-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:3f5716923d3d0bfb27048242a6e0f14eecdb2e2a7fac47eda1d055288595f222"}, + {file = "regex-2021.7.6-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5983c19d0beb6af88cb4d47afb92d96751fb3fa1784d8785b1cdf14c6519407"}, + {file = "regex-2021.7.6-cp36-cp36m-win32.whl", hash = "sha256:c92831dac113a6e0ab28bc98f33781383fe294df1a2c3dfd1e850114da35fd5b"}, + {file = "regex-2021.7.6-cp36-cp36m-win_amd64.whl", hash = "sha256:791aa1b300e5b6e5d597c37c346fb4d66422178566bbb426dd87eaae475053fb"}, + {file = "regex-2021.7.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:59506c6e8bd9306cd8a41511e32d16d5d1194110b8cfe5a11d102d8b63cf945d"}, + {file = "regex-2021.7.6-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:564a4c8a29435d1f2256ba247a0315325ea63335508ad8ed938a4f14c4116a5d"}, + {file = "regex-2021.7.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:59c00bb8dd8775473cbfb967925ad2c3ecc8886b3b2d0c90a8e2707e06c743f0"}, + {file = "regex-2021.7.6-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:9a854b916806c7e3b40e6616ac9e85d3cdb7649d9e6590653deb5b341a736cec"}, + {file = "regex-2021.7.6-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:db2b7df831c3187a37f3bb80ec095f249fa276dbe09abd3d35297fc250385694"}, + {file = "regex-2021.7.6-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:173bc44ff95bc1e96398c38f3629d86fa72e539c79900283afa895694229fe6a"}, + {file = "regex-2021.7.6-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:15dddb19823f5147e7517bb12635b3c82e6f2a3a6b696cc3e321522e8b9308ad"}, + {file = "regex-2021.7.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ddeabc7652024803666ea09f32dd1ed40a0579b6fbb2a213eba590683025895"}, + {file = "regex-2021.7.6-cp37-cp37m-win32.whl", hash = "sha256:f080248b3e029d052bf74a897b9d74cfb7643537fbde97fe8225a6467fb559b5"}, + {file = "regex-2021.7.6-cp37-cp37m-win_amd64.whl", hash = "sha256:d8bbce0c96462dbceaa7ac4a7dfbbee92745b801b24bce10a98d2f2b1ea9432f"}, + {file = "regex-2021.7.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:edd1a68f79b89b0c57339bce297ad5d5ffcc6ae7e1afdb10f1947706ed066c9c"}, + {file = "regex-2021.7.6-cp38-cp38-manylinux1_i686.whl", hash = "sha256:422dec1e7cbb2efbbe50e3f1de36b82906def93ed48da12d1714cabcd993d7f0"}, + {file = "regex-2021.7.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:cbe23b323988a04c3e5b0c387fe3f8f363bf06c0680daf775875d979e376bd26"}, + {file = "regex-2021.7.6-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:0eb2c6e0fcec5e0f1d3bcc1133556563222a2ffd2211945d7b1480c1b1a42a6f"}, + {file = "regex-2021.7.6-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:1c78780bf46d620ff4fff40728f98b8afd8b8e35c3efd638c7df67be2d5cddbf"}, + {file = "regex-2021.7.6-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:bc84fb254a875a9f66616ed4538542fb7965db6356f3df571d783f7c8d256edd"}, + {file = "regex-2021.7.6-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:598c0a79b4b851b922f504f9f39a863d83ebdfff787261a5ed061c21e67dd761"}, + {file = "regex-2021.7.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:875c355360d0f8d3d827e462b29ea7682bf52327d500a4f837e934e9e4656068"}, + {file = "regex-2021.7.6-cp38-cp38-win32.whl", hash = "sha256:e586f448df2bbc37dfadccdb7ccd125c62b4348cb90c10840d695592aa1b29e0"}, + {file = "regex-2021.7.6-cp38-cp38-win_amd64.whl", hash = "sha256:2fe5e71e11a54e3355fa272137d521a40aace5d937d08b494bed4529964c19c4"}, + {file = "regex-2021.7.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6110bab7eab6566492618540c70edd4d2a18f40ca1d51d704f1d81c52d245026"}, + {file = "regex-2021.7.6-cp39-cp39-manylinux1_i686.whl", hash = "sha256:4f64fc59fd5b10557f6cd0937e1597af022ad9b27d454e182485f1db3008f417"}, + {file = "regex-2021.7.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:89e5528803566af4df368df2d6f503c84fbfb8249e6631c7b025fe23e6bd0cde"}, + {file = "regex-2021.7.6-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:2366fe0479ca0e9afa534174faa2beae87847d208d457d200183f28c74eaea59"}, + {file = "regex-2021.7.6-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f9392a4555f3e4cb45310a65b403d86b589adc773898c25a39184b1ba4db8985"}, + {file = "regex-2021.7.6-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:2bceeb491b38225b1fee4517107b8491ba54fba77cf22a12e996d96a3c55613d"}, + {file = "regex-2021.7.6-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:f98dc35ab9a749276f1a4a38ab3e0e2ba1662ce710f6530f5b0a6656f1c32b58"}, + {file = "regex-2021.7.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:319eb2a8d0888fa6f1d9177705f341bc9455a2c8aca130016e52c7fe8d6c37a3"}, + {file = "regex-2021.7.6-cp39-cp39-win32.whl", hash = "sha256:eaf58b9e30e0e546cdc3ac06cf9165a1ca5b3de8221e9df679416ca667972035"}, + {file = "regex-2021.7.6-cp39-cp39-win_amd64.whl", hash = "sha256:4c9c3155fe74269f61e27617529b7f09552fbb12e44b1189cebbdb24294e6e1c"}, + {file = "regex-2021.7.6.tar.gz", hash = "sha256:8394e266005f2d8c6f0bc6780001f7afa3ef81a7a2111fa35058ded6fce79e4d"}, +] requests = [ {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, @@ -192,7 +629,34 @@ six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +text-unidecode = [ + {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"}, + {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +tqdm = [ + {file = "tqdm-4.61.2-py2.py3-none-any.whl", hash = "sha256:5aa445ea0ad8b16d82b15ab342de6b195a722d75fc1ef9934a46bba6feafbc64"}, + {file = "tqdm-4.61.2.tar.gz", hash = "sha256:8bb94db0d4468fea27d004a0f1d1c02da3cdedc00fe491c0de986b76a04d6b0a"}, +] +troposphere = [ + {file = "troposphere-3.0.1-py3-none-any.whl", hash = "sha256:cde54935c60999316ef0e5587a63c69527f07f2c0eda2db6e4113b629457a19c"}, + {file = "troposphere-3.0.1.tar.gz", hash = "sha256:0dde8c6b6c29086654edc2e11ec8beb6090e177e36a842229a9df3ce254650fa"}, +] urllib3 = [ {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, ] +werkzeug = [ + {file = "Werkzeug-2.0.1-py3-none-any.whl", hash = "sha256:6c1ec500dcdba0baa27600f6a22f6333d8b662d22027ff9f6202e3367413caa8"}, + {file = "Werkzeug-2.0.1.tar.gz", hash = "sha256:1de1db30d010ff1af14a009224ec49ab2329ad2cde454c8a708130642d579c42"}, +] +wsgi-request-logger = [ + {file = "wsgi-request-logger-0.4.6.tar.gz", hash = "sha256:445d7ec52799562f812006394d0b4a7064b37084c6ea6bd74ea7a2136c97ed83"}, +] +zappa = [ + {file = "zappa-0.50.0-py3-none-any.whl", hash = "sha256:1c6b95ae2cd9fa22a967d2b247a6b7959387e9b42789e532fb07cff149aeef6b"}, + {file = "zappa-0.50.0.tar.gz", hash = "sha256:e37b987b7ee5cec9eb4d4c1b899e6b05020f0472b00de2783da41912d2163b01"}, +] diff --git a/pyproject.toml b/pyproject.toml index 2764e20..8131318 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,9 +10,13 @@ requests = "^2.25.1" ratelimit = "^2.2.1" backoff = "^1.10.0" boto3 = "^1.17.108" +pynamodb = "^5.1.0" [tool.poetry.dev-dependencies] +black = "^21.6b0" +zappa = "^0.50.0" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" + diff --git a/tests/test_apis.py b/tests/test_apis.py index e335272..fddd124 100644 --- a/tests/test_apis.py +++ b/tests/test_apis.py @@ -11,10 +11,22 @@ class TestDaySummaryApi: @pytest.mark.parametrize( "coin, date, expected", [ - ("BTC", datetime.date(2021, 6, 21), "https://www.mercadobitcoin.net/api/BTC/day-summary/2021/6/21"), - ("ETH", datetime.date(2021, 6, 21), "https://www.mercadobitcoin.net/api/ETH/day-summary/2021/6/21"), - ("ETH", datetime.date(2019, 1, 2), "https://www.mercadobitcoin.net/api/ETH/day-summary/2019/1/2"), - ] + ( + "BTC", + datetime.date(2021, 6, 21), + "https://www.mercadobitcoin.net/api/BTC/day-summary/2021/6/21", + ), + ( + "ETH", + datetime.date(2021, 6, 21), + "https://www.mercadobitcoin.net/api/ETH/day-summary/2021/6/21", + ), + ( + "ETH", + datetime.date(2019, 1, 2), + "https://www.mercadobitcoin.net/api/ETH/day-summary/2019/1/2", + ), + ], ) def test_get_endpoint(self, coin, date, expected): api = DaySummaryApi(coin=coin) @@ -26,27 +38,44 @@ class TestTradesApi: @pytest.mark.parametrize( "coin, date_from, date_to, expected", [ - ("TEST", datetime.datetime(2019, 1, 1), datetime.datetime(2019, 1, 2), - "https://www.mercadobitcoin.net/api/TEST/trades/1546300800/1546387200"), - ("TEST", datetime.datetime(2021, 6, 12), datetime.datetime(2021, 6, 15), - "https://www.mercadobitcoin.net/api/TEST/trades/1623452400/1623711600"), - ("TEST", None, None, - "https://www.mercadobitcoin.net/api/TEST/trades"), - ("TEST", None, datetime.datetime(2021, 6, 15), - "https://www.mercadobitcoin.net/api/TEST/trades"), - ("TEST", datetime.datetime(2021, 6, 12), None, - "https://www.mercadobitcoin.net/api/TEST/trades/1623452400"), - ] + ( + "TEST", + datetime.datetime(2019, 1, 1), + datetime.datetime(2019, 1, 2), + "https://www.mercadobitcoin.net/api/TEST/trades/1546300800/1546387200", + ), + ( + "TEST", + datetime.datetime(2021, 6, 12), + datetime.datetime(2021, 6, 15), + "https://www.mercadobitcoin.net/api/TEST/trades/1623452400/1623711600", + ), + ("TEST", None, None, "https://www.mercadobitcoin.net/api/TEST/trades"), + ( + "TEST", + None, + datetime.datetime(2021, 6, 15), + "https://www.mercadobitcoin.net/api/TEST/trades", + ), + ( + "TEST", + datetime.datetime(2021, 6, 12), + None, + "https://www.mercadobitcoin.net/api/TEST/trades/1623452400", + ), + ], ) def test_get_endpoint(self, coin, date_from, date_to, expected): - actual = TradesApi(coin=coin)._get_endpoint(date_from=date_from, date_to=date_to) + actual = TradesApi(coin=coin)._get_endpoint( + date_from=date_from, date_to=date_to + ) assert actual == expected def test_get_endpoint_date_from_greater_than_date_to(self): with pytest.raises(RuntimeError): TradesApi(coin="TEST")._get_endpoint( date_from=datetime.datetime(2021, 6, 15), - date_to=datetime.datetime(2021, 6, 12) + date_to=datetime.datetime(2021, 6, 12), ) @pytest.mark.parametrize( @@ -57,7 +86,7 @@ def test_get_endpoint_date_from_greater_than_date_to(self): (datetime.datetime(2021, 6, 12), 1623452400), (datetime.datetime(2021, 6, 12, 0, 0, 5), 1623452405), (datetime.datetime(2021, 6, 15), 1623711600), - ] + ], ) def test_get_unix_epoch(self, date, expected): actual = TradesApi(coin="TEST")._get_unix_epoch(date) @@ -67,9 +96,7 @@ def test_get_unix_epoch(self, date, expected): @pytest.fixture() @patch("mercado_bitcoin.apis.MercadoBitcoinApi.__abstractmethods__", set()) def fixture_mercado_bitcoin_api(): - return MercadoBitcoinApi( - coin="test" - ) + return MercadoBitcoinApi(coin="test") def mocked_requests_get(*args, **kwargs): @@ -94,20 +121,35 @@ def raise_for_status(self) -> None: class TestMercadoBitcoinApi: @patch("requests.get") - @patch("mercado_bitcoin.apis.MercadoBitcoinApi._get_endpoint", return_value="valid_endpoint") - def test_get_data_requests_is_called(self, mock_get_endpoint, mock_requests, fixture_mercado_bitcoin_api): + @patch( + "mercado_bitcoin.apis.MercadoBitcoinApi._get_endpoint", + return_value="valid_endpoint", + ) + def test_get_data_requests_is_called( + self, mock_get_endpoint, mock_requests, fixture_mercado_bitcoin_api + ): fixture_mercado_bitcoin_api.get_data() mock_requests.assert_called_once_with("valid_endpoint") @patch("requests.get", side_effect=mocked_requests_get) - @patch("mercado_bitcoin.apis.MercadoBitcoinApi._get_endpoint", return_value="valid_endpoint") - def test_get_data_with_valid_endpoint(self, mock_get_endpoint, mock_requests, fixture_mercado_bitcoin_api): + @patch( + "mercado_bitcoin.apis.MercadoBitcoinApi._get_endpoint", + return_value="valid_endpoint", + ) + def test_get_data_with_valid_endpoint( + self, mock_get_endpoint, mock_requests, fixture_mercado_bitcoin_api + ): actual = fixture_mercado_bitcoin_api.get_data() expected = {"foo": "bar"} assert actual == expected @patch("requests.get", side_effect=mocked_requests_get) - @patch("mercado_bitcoin.apis.MercadoBitcoinApi._get_endpoint", return_value="invalid_endpoint") - def test_get_data_with_valid_endpoint(self, mock_get_endpoint, mock_requests, fixture_mercado_bitcoin_api): + @patch( + "mercado_bitcoin.apis.MercadoBitcoinApi._get_endpoint", + return_value="invalid_endpoint", + ) + def test_get_data_with_valid_endpoint( + self, mock_get_endpoint, mock_requests, fixture_mercado_bitcoin_api + ): with pytest.raises(Exception): fixture_mercado_bitcoin_api.get_data() diff --git a/tests/test_ingestors.py b/tests/test_ingestors.py index ef23fbc..8ae492a 100644 --- a/tests/test_ingestors.py +++ b/tests/test_ingestors.py @@ -11,10 +11,10 @@ @patch("mercado_bitcoin.ingestors.DataIngestor.__abstractmethods__", set()) def data_ingestor_fixture(): return DataIngestor( - writer=DataWriter, - coins=["TEST", "HOW"], - default_start_date=datetime.date(2021, 6, 21) - ) + writer=DataWriter, + coins=["TEST", "HOW"], + default_start_date=datetime.date(2021, 6, 21), + ) @patch("mercado_bitcoin.ingestors.DataIngestor.__abstractmethods__", set()) @@ -35,20 +35,29 @@ def test_load_checkpoint_existing_checkpoint(self, mock, data_ingestor_fixture): expected = datetime.date(2021, 6, 25) assert actual == expected - @patch("mercado_bitcoin.ingestors.DataIngestor._write_checkpoint", return_value=None) + @patch( + "mercado_bitcoin.ingestors.DataIngestor._write_checkpoint", return_value=None + ) def test_update_checkpoint_checkpoint_updated(self, mock, data_ingestor_fixture): data_ingestor_fixture._update_checkpoint(value=datetime.date(2019, 1, 1)) actual = data_ingestor_fixture._checkpoint expected = datetime.date(2019, 1, 1) assert actual == expected - @patch("mercado_bitcoin.ingestors.DataIngestor._write_checkpoint", return_value=None) + @patch( + "mercado_bitcoin.ingestors.DataIngestor._write_checkpoint", return_value=None + ) def test_update_checkpoint_checkpoint_written(self, mock, data_ingestor_fixture): data_ingestor_fixture._update_checkpoint(value=datetime.date(2019, 1, 1)) mock.assert_called_once() @patch("builtins.open", new_callable=mock_open, read_data="2021-06-25") - @patch("mercado_bitcoin.ingestors.DataIngestor._checkpoint_filename", return_value="foobar.checkpoint") - def test_write_checkpoint(self, mock_checkpoint_filename, mock_open_file, data_ingestor_fixture): + @patch( + "mercado_bitcoin.ingestors.DataIngestor._checkpoint_filename", + return_value="foobar.checkpoint", + ) + def test_write_checkpoint( + self, mock_checkpoint_filename, mock_open_file, data_ingestor_fixture + ): data_ingestor_fixture._write_checkpoint() - mock_open_file.assert_called_with(mock_checkpoint_filename, 'w') + mock_open_file.assert_called_with(mock_checkpoint_filename, "w") diff --git a/tests/test_integration.py b/tests/test_integration.py index ebb8123..4c1fd64 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -6,10 +6,24 @@ class TestDaySummaryApi: def test_get_data(self): actual = DaySummaryApi(coin="BTC").get_data(date=datetime.date(2021, 1, 1)) - expected = {'date': '2021-01-01', 'opening': 152700.00002, 'closing': 153458.29999999, 'lowest': 151539, 'highest': 153975, 'volume': 12583384.54790148, 'quantity': 82.27265844, 'amount': 4824, 'avg_price': 152947.34346135} + expected = { + "date": "2021-01-01", + "opening": 152700.00002, + "closing": 153458.29999999, + "lowest": 151539, + "highest": 153975, + "volume": 12583384.54790148, + "quantity": 82.27265844, + "amount": 4824, + "avg_price": 152947.34346135, + } assert actual == expected def test_get_data_better(self): - actual = DaySummaryApi(coin="BTC").get_data(date=datetime.date(2021, 1, 1)).get("date") + actual = ( + DaySummaryApi(coin="BTC") + .get_data(date=datetime.date(2021, 1, 1)) + .get("date") + ) expected = "2021-01-01" assert actual == expected diff --git a/zappa_settings.json b/zappa_settings.json new file mode 100644 index 0000000..bd892b1 --- /dev/null +++ b/zappa_settings.json @@ -0,0 +1,18 @@ +{ + "dev": { + "app_function": "mercado_bitcoin.lambda_function", + "aws_region": "us-east-1", + "profile_name": "andre_aws", + "project_name": "ingestao-com-sc", + "runtime": "python3.8", + "s3_bucket": "zappa-ddwyelw2n", + "events": [ + { + "function": "mercado_bitcoin.lambda_function.lambda_handler", + "expression": "rate(1 minute)" + }, + ], + "memory_size": 128, + "cloudwatch_log_level": "INFO" + } +} \ No newline at end of file