WIP: Optimizations

.gitignore

@@ -7,4 +7,5 @@ rosetta-data*
 lbs.txt
 node_modules
 helium-constructor/public
-badger
+badger
+rocksdb

Dockerfile

@@ -1,5 +1,3 @@
-ARG NETWORK=mainnet
-
 FROM ubuntu:20.04 as node-builder
 
 ARG BUILD_VERSION="23.3.4.7"
@@ -84,7 +82,8 @@ ENV CC=gcc CXX=g++ CFLAGS="-U__sun__" \
 WORKDIR /usr/src/
 
 # Add our code
-RUN git clone https://github.com/helium/blockchain-node.git
+# TODO: git checkout specific release when ready
+RUN git clone https://github.com/syuan100/blockchain-node.git && echo ''
 
 
 FROM node-builder AS node-mainnet
@@ -93,17 +92,7 @@ ARG BUILD_TARGET=docker_rosetta
 
 WORKDIR /usr/src/blockchain-node
 
-RUN ./rebar3 as ${BUILD_TARGET} tar -n blockchain_node
-
-RUN mkdir -p /opt/blockchain_node \
-	&& tar -zxvf _build/${BUILD_TARGET}/rel/*/*.tar.gz -C /opt/blockchain_node
-
-
-FROM node-builder AS node-testnet
-
-ARG BUILD_TARGET=docker_rosetta_testnet
-
-WORKDIR /usr/src/blockchain-node
+RUN git checkout bump_snap_sy
 
 RUN ./rebar3 as ${BUILD_TARGET} tar -n blockchain_node
 
@@ -135,30 +124,57 @@ RUN apt update \
       && curl -L https://golang.org/dl/go1.17.1.linux-amd64.tar.gz | tar xzf -
 
 ENV PATH="/src/go/bin:$PATH" \
-    CGO_ENABLED=0
+    CGO_ENABLED=1
 
-COPY . rosetta-helium
+WORKDIR /app
+RUN apt install -y git
+RUN git clone https://github.com/facebook/rocksdb.git
+
+WORKDIR /app/rocksdb
+RUN git checkout tags/v6.20.3
+RUN apt install -y --no-install-recommends \
+	build-essential make libgflags-dev \
+	zlib1g-dev libbz2-dev liblz4-dev libsnappy-dev \
+	libzstd-dev libgflags-dev
+
+RUN make shared_lib
+RUN make install
+
+WORKDIR /src
 
-RUN cd rosetta-helium && go build -o rosetta-helium
+# TODO: git clone from url instead of direct copy
+COPY . rosetta-helium
 
+WORKDIR /src/rosetta-helium
+RUN go build -o rosetta-helium
 
-FROM node-${NETWORK} as rosetta-helium-final
+FROM node-mainnet as rosetta-helium-final
 
-ARG NETWORK
 ARG DEBIAN_FRONTEND=noninteractive
 
 EXPOSE 8080
 EXPOSE 44158
 
 RUN apt update \
     && apt install -y --no-install-recommends \
-         ca-certificates git npm
+         ca-certificates git npm 
+
+WORKDIR /app
+COPY --from=rosetta-builder /app/rocksdb rocksdb
+
+WORKDIR /app/rocksdb
+RUN apt install -y --no-install-recommends \
+	build-essential make libgflags-dev \
+	zlib1g-dev libbz2-dev liblz4-dev libsnappy-dev \
+	libzstd-dev
+
+RUN make install
 
 WORKDIR /app
 
 COPY --from=rosetta-builder /src/rosetta-helium/rosetta-helium rosetta-helium
 COPY --from=rosetta-builder /src/rosetta-helium/ghost-transactions ghost-transactions
-COPY --from=rosetta-builder /src/rosetta-helium/docker/${NETWORK}.sh start.sh
+COPY --from=rosetta-builder /src/rosetta-helium/docker/mainnet.sh start.sh
 COPY --from=rosetta-builder /src/rosetta-helium/helium-constructor helium-constructor
 
 RUN cd helium-constructor \

Dockerfile_quick

@@ -1,6 +1,6 @@
 ARG UBUNTU_BUILDER=ubuntu:20.04
-ARG MAINNET_RUNNER=quay.io/team-helium/blockchain-node:blockchain-node-ubuntu18-1.1.53
-ARG TESTNET_RUNNER=quay.io/team-helium/blockchain-node:blockchain-node-testnet-ubuntu18-1.1.53
+ARG MAINNET_RUNNER=quay.io/team-helium/blockchain-node:blockchain-node-ubuntu18-1.1.57
+ARG TESTNET_RUNNER=quay.io/team-helium/blockchain-node:blockchain-node-testnet-ubuntu18-1.1.57
 ARG NETWORK=mainnet
 
 

Dockerfile_testnet

@@ -0,0 +1,158 @@
+FROM ubuntu:20.04 as node-builder
+
+ARG BUILD_VERSION="23.3.4.7"
+ARG BUILD_SHA256="37e39a43c495861ce69de06e1a013a7eac81d15dc6eebd2d2022fd68791f4b2d"
+ENV OTP_VERSION=$BUILD_VERSION \
+    REBAR3_VERSION="3.16.1"
+
+LABEL org.opencontainers.image.version=$OTP_VERSION
+
+# We'll install the build dependencies, and purge them on the last step to make
+# sure our final image contains only what we've just built:
+RUN set -xe \
+	&& OTP_DOWNLOAD_URL="https://github.com/erlang/otp/releases/download/OTP-${BUILD_VERSION}/otp_src_${BUILD_VERSION}.tar.gz" \
+	&& OTP_DOWNLOAD_SHA256="${BUILD_SHA256}" \
+	&& fetchDeps=' \
+		curl \
+		ca-certificates' \
+	&& apt-get update \
+	&& apt-get install -y --no-install-recommends $fetchDeps \
+	&& curl -fSL -o otp-src.tar.gz "$OTP_DOWNLOAD_URL" \
+	&& echo "$OTP_DOWNLOAD_SHA256  otp-src.tar.gz" | sha256sum -c - \
+	&& runtimeDeps=' \
+		libssl1.1 \
+	' \
+	&& buildDeps=' \
+		autoconf \
+		dpkg-dev \
+		gcc \
+		g++ \
+		make \
+		libncurses-dev \
+		libssl-dev \
+	' \
+	&& apt-get install -y --no-install-recommends $runtimeDeps \
+	&& apt-get install -y --no-install-recommends $buildDeps \
+	&& export ERL_TOP="/usr/src/otp_src_${OTP_VERSION%%@*}" \
+	&& mkdir -vp $ERL_TOP \
+	&& tar -xzf otp-src.tar.gz -C $ERL_TOP --strip-components=1 \
+	&& rm otp-src.tar.gz \
+	&& ( cd $ERL_TOP \
+	  && ./otp_build autoconf \
+	  && gnuArch="$(dpkg-architecture --query DEB_HOST_GNU_TYPE)" \
+	  && ./configure --build="$gnuArch" \
+	  && make -j$(nproc) \
+	  && make install ) \
+	&& find /usr/local -name examples | xargs rm -rf \
+	&& REBAR3_DOWNLOAD_URL="https://github.com/erlang/rebar3/archive/${REBAR3_VERSION}.tar.gz" \
+	&& REBAR3_DOWNLOAD_SHA256="a14711b09f6e1fc1b080b79d78c304afebcbb7fafed9d0972eb739f0ed89121b" \
+	&& mkdir -p /usr/src/rebar3-src \
+	&& curl -fSL -o rebar3-src.tar.gz "$REBAR3_DOWNLOAD_URL" \
+	&& echo "$REBAR3_DOWNLOAD_SHA256 rebar3-src.tar.gz" | sha256sum -c - \
+	&& tar -xzf rebar3-src.tar.gz -C /usr/src/rebar3-src --strip-components=1 \
+	&& rm rebar3-src.tar.gz \
+	&& cd /usr/src/rebar3-src \
+	&& HOME=$PWD ./bootstrap \
+	&& install -v ./rebar3 /usr/local/bin/ \
+	&& rm -rf /usr/src/rebar3-src \
+	&& apt-get purge -y --auto-remove $buildDeps $fetchDeps \
+	&& rm -rf $ERL_TOP /var/lib/apt/lists/*
+
+RUN set -xe \
+	    && apt update \
+	    && apt-get install -y --no-install-recommends \
+     	        libssl-dev make automake autoconf libncurses5-dev gcc \
+	        libdbus-1-dev libbz2-dev bison flex libgmp-dev liblz4-dev \
+	        libsodium-dev sed wget curl build-essential libtool git \
+		ca-certificates \
+	    && mkdir -p /opt/cmake \
+	    && wget -O /opt/cmake/cmake.tgz \
+	        https://github.com/Kitware/CMake/releases/download/v3.21.3/cmake-3.21.3-linux-x86_64.tar.gz \
+            && tar -zxf /opt/cmake/cmake.tgz -C /opt/cmake --strip-components=1
+
+# Install Rust toolchain
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+ENV CC=gcc CXX=g++ CFLAGS="-U__sun__" \
+    ERLANG_ROCKSDB_OPTS="-DWITH_BUNDLE_SNAPPY=ON -DWITH_BUNDLE_LZ4=ON" \
+    ERL_COMPILER_OPTIONS="[deterministic]" \
+    PATH="/root/.cargo/bin:/opt/cmake/bin:$PATH" \
+    RUSTFLAGS="-C target-feature=-crt-static"
+
+WORKDIR /usr/src/
+
+# Add our code
+# TODO: git checkout specific release when ready
+RUN git clone https://github.com/syuan100/blockchain-node.git
+
+
+FROM node-builder AS node-testnet
+
+ARG BUILD_TARGET=docker_rosetta_testnet
+
+WORKDIR /usr/src/blockchain-node
+
+RUN git checkout commit-hooks-with-heights
+
+RUN ./rebar3 as ${BUILD_TARGET} tar -n blockchain_node
+
+RUN mkdir -p /opt/blockchain_node \
+	&& tar -zxvf _build/${BUILD_TARGET}/rel/*/*.tar.gz -C /opt/blockchain_node
+
+
+FROM ubuntu:20.04 AS rosetta-builder
+
+RUN set -xe \
+	&& ulimit -n 100000 \
+        && apt update \
+ 	&& apt install -y --no-install-recommends libdbus-1-3 libgmp10 libsodium23 \
+	&& rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/blockchain_node
+
+ENV COOKIE=blockchain_node \
+    # Write files generated during startup to /tmp
+    RELX_OUT_FILE_PATH=/tmp \
+    # add to path, for easy exec interaction
+    PATH=/sbin:/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:$PATH:/opt/blockchain_node/bin
+
+WORKDIR /src
+
+RUN apt update \
+      && apt install -y --no-install-recommends \
+         curl ca-certificates git \      
+      && curl -L https://golang.org/dl/go1.17.1.linux-amd64.tar.gz | tar xzf -
+
+ENV PATH="/src/go/bin:$PATH" \
+    CGO_ENABLED=0
+
+# TODO: git clone from url instead of direct copy
+COPY . rosetta-helium
+
+RUN cd rosetta-helium && go build -o rosetta-helium
+
+
+FROM node-testnet as rosetta-helium-final
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+EXPOSE 8080
+EXPOSE 44158
+
+RUN apt update \
+    && apt install -y --no-install-recommends \
+         ca-certificates git npm
+
+WORKDIR /app
+
+COPY --from=rosetta-builder /src/rosetta-helium/rosetta-helium rosetta-helium
+COPY --from=rosetta-builder /src/rosetta-helium/ghost-transactions ghost-transactions
+COPY --from=rosetta-builder /src/rosetta-helium/docker/testnet.sh start.sh
+COPY --from=rosetta-builder /src/rosetta-helium/helium-constructor helium-constructor
+
+RUN cd helium-constructor \
+      && npm install \
+      && npm run build \
+      && chmod +x /app/start.sh
+
+ENTRYPOINT ["/app/start.sh"]

README.md

@@ -18,12 +18,12 @@ This project was created by [@syuan100](https://github.com/syuan100) and support
 #### Build container from source
 Mainnet:
 ```text
-DOCKER_BUILDKIT=1 docker build . -t rosetta-helium:latest
+docker build . -t rosetta-helium:latest
 ```
 
 Testnet:
 ```text
-DOCKER_BUILDKIT=1 docker build . -t rosetta-helium:latest --build-arg NETWORK=testnet
+docker build . -f Dockerfile_testnet -t rosetta-helium:latest
 ```
 
 *Note: `DOCKER_BUILDKIT=1` is not necessary but including it may reduce the image size due to the nature of the conditional build.*
@@ -63,8 +63,6 @@ docker run -d --rm --init --ulimit "nofile=1000000:1000000" -v "$(pwd)/helium-da
 Mainnet:
 ```text
 rosetta-cli check:data --configuration-file rosetta-cli-config/mainnet/config.json
-
-rosetta-cli check:construction --configuration-file rosetta-cli-config/mainnet/config.json
 ```
 
 Testnet:

docker/mainnet.sh

@@ -2,8 +2,8 @@
 
 set -euo pipefail
 
-cat /opt/blockchain_node/config/sys.config | grep -oP '(?<=\{blessed_snapshot_block_height\, ).*?(?=\})' > /app/lbs.txt &
+echo '1259282' > /app/lbs.txt &
 
 /opt/blockchain_node/bin/blockchain_node foreground &
-/app/rosetta-helium &
+/app/rosetta-helium --data="/data" &
 node /app/helium-constructor/public/index.js

docker/testnet.sh

@@ -3,5 +3,5 @@
 set -euo pipefail
 
 /opt/blockchain_node/bin/blockchain_node foreground &
-/app/rosetta-helium --testnet &
+/app/rosetta-helium --testnet --data="/data" &
 NETWORK=testnet node /app/helium-constructor/public/index.js