commit inicial

.gitignore

@@ -0,0 +1,254 @@
+
+# Created by https://www.toptal.com/developers/gitignore/api/python,pycharm
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,pycharm
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+# https://plugins.jetbrains.com/plugin/7973-sonarlint
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator-enh.xml
+.idea/**/markdown-navigator/
+
+# Cache file creation bug
+# See https://youtrack.jetbrains.com/issue/JBR-2257
+.idea/$CACHE_FILE$
+
+# CodeStream plugin
+# https://plugins.jetbrains.com/plugin/12206-codestream
+.idea/codestream.xml
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# End of https://www.toptal.com/developers/gitignore/api/python,pycharm
+
+day-summary/*
+*.checkpoint

mercado_bitcoin/apis.py

@@ -0,0 +1,60 @@
+import datetime
+from abc import ABC, abstractmethod
+
+import logging
+import ratelimit
+import requests
+from backoff import on_exception, expo, constant
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+class MercadoBitcoinApi(ABC):
+
+    def __init__(self, coin: str) -> None:
+        self.coin = coin
+        self.base_endpoint = "https://www.mercadobitcoin.net/api"
+
+    @abstractmethod
+    def _get_endpoint(self, **kwargs) -> str:
+        pass
+
+    @on_exception(constant, ratelimit.exception.RateLimitException, max_tries=10)
+    @ratelimit.limits(calls=30, period=30)
+    @on_exception(expo, requests.exceptions.HTTPError, max_tries=10)
+    def get_data(self, **kwargs) -> dict:
+        endpoint = self._get_endpoint(**kwargs)
+        logger.info(f"Getting data from endpoint: {endpoint}")
+        response = requests.get(endpoint)
+        response.raise_for_status()
+        return response.json()
+
+
+class DaySummaryApi(MercadoBitcoinApi):
+    type = "day-summary"
+
+    def _get_endpoint(self, date: datetime.date) -> str:
+        return f"{self.base_endpoint}/{self.coin}/{self.type}/{date.year}/{date.month}/{date.day}"
+
+
+class TradesApi(MercadoBitcoinApi):
+    type = "trades"
+
+    def _get_unix_epoch(self, date: datetime.datetime) -> int:
+        return int(date.timestamp())
+
+    def _get_endpoint(self, date_from: datetime.datetime = None, date_to: datetime.datetime = None) -> str:
+        if date_from and not date_to:
+            unix_date_from = self._get_unix_epoch(date_from)
+            endpoint = f'{self.base_endpoint}/{self.coin}/{self.type}/{unix_date_from}'
+        elif date_from and date_to:
+            if date_from > date_to:
+                raise RuntimeError("date_from cannot be greater than date_to")
+            unix_date_from = self._get_unix_epoch(date_from)
+            unix_date_to = self._get_unix_epoch(date_to)
+            endpoint = f'{self.base_endpoint}/{self.coin}/{self.type}/{unix_date_from}/{unix_date_to}'
+        else:
+            endpoint = f'{self.base_endpoint}/{self.coin}/{self.type}'
+
+        return endpoint

mercado_bitcoin/ingestors.py

@@ -0,0 +1,49 @@
+import datetime
+from abc import ABC, abstractmethod
+from typing import List
+
+from mercado_bitcoin.apis import DaySummaryApi
+
+
+class DataIngestor(ABC):
+
+    def __init__(self, writer, coins: List[str], default_start_date: datetime.date) -> None:
+        self.default_start_date = default_start_date
+        self.coins = coins
+        self.writer = writer
+        self._checkpoint = self._load_checkpoint()
+
+    @property
+    def _checkpoint_filename(self) -> str:
+        return f"{self.__class__.__name__}.checkpoint"
+
+    def _write_checkpoint(self):
+        with open(self._checkpoint_filename, "w") as f:
+            f.write(f"{self._checkpoint}")
+
+    def _load_checkpoint(self) -> datetime.date:
+        try:
+            with open(self._checkpoint_filename, "r") as f:
+                return datetime.datetime.strptime(f.read(), "%Y-%m-%d").date()
+        except FileNotFoundError:
+            return self.default_start_date
+
+    def _update_checkpoint(self, value):
+        self._checkpoint = value
+        self._write_checkpoint()
+
+    @abstractmethod
+    def ingest(self) -> None:
+        pass
+
+
+class DaySummaryIngestor(DataIngestor):
+
+    def ingest(self) -> None:
+        date = self._load_checkpoint()
+        if date < datetime.date.today():
+            for coin in self.coins:
+                api = DaySummaryApi(coin=coin)
+                data = api.get_data(date=date)
+                self.writer(coin=coin, api=api.type).write(data)
+            self._update_checkpoint(date + datetime.timedelta(days=1))

mercado_bitcoin/main.py

@@ -0,0 +1,26 @@
+import datetime
+import time
+
+from schedule import repeat, every, run_pending
+from mercado_bitcoin.ingestors import DaySummaryIngestor
+from mercado_bitcoin.writers import DataWriter
+
+
+if __name__ == "__main__":
+    day_summary_ingestor = DaySummaryIngestor(
+        writer=DataWriter,
+        coins=["BTC", "ETH", "LTC", "BCH"],
+        default_start_date=datetime.date(2021, 6, 1)
+    )
+
+
+    @repeat(every(1).seconds)
+    def job():
+        day_summary_ingestor.ingest()
+
+
+    while True:
+        run_pending()
+        time.sleep(0.5)
+
+