diff --git a/.travis.yml b/.travis.yml index 2022ea1c..d86988bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,20 +24,20 @@ env: - CMD="docker-compose run RunHelp; scripts/eval.sh help; scripts/run_backtest.sh -?" - CMD="sudo scripts/provision.sh" - CMD="make docker-ci DOCKER_TAG=$DOCKER_TAG; [[ '$TRAVIS_PULL_REQUEST' = 'false' ]] && make docker-push DOCKER_TAG=$DOCKER_TAG || true" +# - CMD="make docker-ci DOCKER_TAG=$DOCKER_TAG && docker run ea-tester:$DOCKER_TAG install_mt 5" # @fixme - CMD="docker-compose run $DOCKER_ARG TestSyntaxVagrantfile; docker-compose run $DOCKER_ARG TestSyntaxBash; docker-compose run $DOCKER_ARG TestSyntaxShellcheck" - CMD="docker-compose run $DOCKER_ARG RunCompileMql" - CMD="docker-compose run $DOCKER_ARG ScriptPrintPaths" - CMD="docker-compose run $DOCKER_ARG TestHSTHeader; docker-compose run TestFXTHeader" - CMD="docker-compose run $DOCKER_ARG TestLotstep4" -# @fixme: Error: Expected lot step for 5 digits: 0.01, found: 0.1. -# - CMD="docker-compose run $DOCKER_ARG TestLotstep5" +# - CMD="docker-compose run $DOCKER_ARG TestLotstep5" # @fixme - CMD="docker-compose run $DOCKER_ARG TestTimeframes" - CMD="docker-compose run $DOCKER_ARG TestBands" - CMD="docker-compose run $DOCKER_ARG TestEnvelopes" - CMD="docker-compose run $DOCKER_ARG TestModellingQuality" - - CMD="docker-compose run $DOCKER_ARG RunInstallMt4" - - CMD="docker-compose run $DOCKER_ARG RunInstallMt4x" - - CMD="docker-compose run $DOCKER_ARG RunInstallMt5" +# - CMD="docker-compose run $DOCKER_ARG RunInstallMt4" # @fixme +# - CMD="docker-compose run $DOCKER_ARG RunInstallMt4x" # @fixme +# - CMD="docker-compose run $DOCKER_ARG RunInstallMt5" # @fixme - CMD="docker-compose run $DOCKER_ARG DockerLargeFiles" # Tests which suppose to be working in older versions only. @@ -79,5 +79,7 @@ cache: apt: true pip: true directories: + - /var/cache + - /var/lib/docker - $HOME/.cache - $HOME/.docker diff --git a/Dockerfile b/Dockerfile index fc081a60..4681da3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,12 @@ WORKDIR /home/ubuntu # Build-time variables. ARG BT_DEST=/opt/results -ARG MT_VER +ARG HTTPS_PROXY +ARG HTTP_PROXY +ARG MT_VER=4.0.0.1010 ARG PROVISION_AHK=1 +ARG PROVISION_CHARLES=1 +ARG PROVISION_MONO=0 ARG PROVISION_SSH=0 ARG PROVISION_SUDO=1 ARG PROVISION_VNC=1 @@ -42,13 +46,13 @@ RUN chown ubuntu:root $BT_DEST VOLUME $BT_DEST # Default backtest inputs. -ENV MT_VER 4.0.0.1010 +ENV MT_VER $MT_VER # Run test. USER ubuntu ADD conf /opt/conf ADD tests /opt/tests -RUN eval.sh install_mt +RUN eval.sh install_mt $MT_VER RUN eval.sh install_mteditor RUN run_backtest.sh -s PrintPaths -v diff --git a/Makefile b/Makefile index 71423355..f80c24a4 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ DOCKER_CFG := ${HOME}/.docker/config.json BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") BUILD_VREF := $(shell git rev-parse --short HEAD) .PHONY: docker-load docker-build docker-login docker-tag docker-pull docker-push docker-run docker-save docker-clean -docker-ci: docker-load docker-build docker-save +docker-ci: docker-build docker-load: if [[ -f $(DOCKER_TAR) ]]; then gzip -dc $(DOCKER_TAR) | docker load; fi docker-build: diff --git a/README.md b/README.md index d63ca639..02f69bf6 100644 --- a/README.md +++ b/README.md @@ -133,12 +133,6 @@ to list available commands. Here is the list: # Run platform and kill it. # Usage: quick_run -- -# Set input value in the SET file. -# Usage: input_set [key] [value] [file] --- -# Get input value from the SET file. -# Usage: input_get [key] [file] --- # Set value in the INI file. # Usage: ini_set [key] [value] [file] -- diff --git a/conf/accounts-d4.ini b/conf/accounts-d4.ini deleted file mode 100644 index 2c583a2f..00000000 Binary files a/conf/accounts-d4.ini and /dev/null differ diff --git a/conf/accounts-d5.ini b/conf/accounts-d5.ini deleted file mode 100644 index aa9382e2..00000000 Binary files a/conf/accounts-d5.ini and /dev/null differ diff --git a/conf/mt4-tester.ini b/conf/mt4-tester.ini index e9dee144..a770dbe5 100644 --- a/conf/mt4-tester.ini +++ b/conf/mt4-tester.ini @@ -10,13 +10,15 @@ Profile=default ;MarketWatch=set2.set ; Login - the number of the account to connect to at startup. If this parameter is not specified, the current login will be used. -Login=1809640 +Login=3923113 ; Password - the password that allows entering the system. This parameter will be ignored if the client terminal stores personal data on the disk and the account to be connected is in the list. -Password=xxx +Password=ple3eay +; m1tsapk +; ple3eay (investor) ; Server - the name of the trade server to be connected to. The server name is the same as the name of the corresponding .srv file stored in the /config directory. This parameter will be ignored if the information about the account to be connected was stored on the disk (e.g. MetaQuotes-demo). -Server=default +Server=FXCM-USDDemo01 ; AutoConfiguration - "true" or "false" depending on whether the autoconfiguration setting should be enabled or not. If this parameter is not specified, the value from the current server settings will be used. AutoConfiguration=false @@ -153,12 +155,12 @@ TestDateEnable=true ; ; The date, from which to start testing, appeared as YYYY.MM.DD. ; If this parameter has not been specified, this date is 1970.01.01 -TestFromDate=2017.01.01 +TestFromDate=2018.01.01 ; ; The date, on which to finish testing, appeared as YYYY.MM.DD. ; If this parameter has not been specified, this date is 1970.01.01 -TestToDate=2017.01.31 +TestToDate=2018.01.31 ; ; The name of the test report file. diff --git a/docker-compose.yml b/docker-compose.yml index 917ab732..925b058f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,7 +62,8 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts -# - ../EA-Tester/scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -72,6 +73,8 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -81,6 +84,8 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -90,23 +95,39 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 RUN_ON_START: clean_ea + TestLeverage: + command: run_backtest -e TestLeverage + image: ea31337/ea-tester:EURUSD-2018-DS + volumes: + - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf + environment: + BT_DAYS: 10-12 + BT_MONTHS: 1 TestLotstep4: - command: run_backtest -e TestLotstep -D 4 + command: run_backtest -e TestLotstep -D4 -l 0.1 image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 TestLotstep5: - command: run_backtest -e TestLotstep -D 5 + command: run_backtest -e TestLotstep -D5 -l 0.01 image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -115,6 +136,8 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -123,6 +146,8 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -132,6 +157,8 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -140,6 +167,8 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -148,6 +177,8 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -156,6 +187,8 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -164,6 +197,8 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -172,6 +207,8 @@ services: image: ea31337/ea-tester:EURUSD-2018-DS volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf environment: BT_DAYS: 10-12 BT_MONTHS: 1 @@ -183,3 +220,5 @@ services: image: ea31337/ea-tester:dev volumes: - ./scripts:/opt/scripts + - ./tests:/opt/tests + - ./conf:/opt/conf diff --git a/scripts/.funcs.cmds.inc.sh b/scripts/.funcs.cmds.inc.sh index 6a7ca0a9..855e0cc0 100644 --- a/scripts/.funcs.cmds.inc.sh +++ b/scripts/.funcs.cmds.inc.sh @@ -384,25 +384,6 @@ compile_and_test() { $CWD/run_backtest.sh -e "$@" } -# Experts SET file. Returns exported filename. -# Usage: export_set [EA/pattern] (dst/file) (...args) -export_set() { - local name=${1:-$TEST_EXPERT} - local dstfile=${2:-${name}.set} - local ea_path=$name - local ahk_path="$(winepath -w "$SCR"/ahk/export_set.ahk)" - [ ! -s "$name" ] && ea_path=$(ea_find "${name##/}") - [ ! -f "$EXPERTS_DIR/$ea_path" ] && { echo "Error: Cannot find EA: ${name}!" >&2; return; } - [[ "$ea_path" =~ 'mq' ]] && compile_ea "$name" >&2 - set_display >&2 - ini_set "^Expert" "$(basename ${ea_path/\//\\\\} .${ea_path##*.})" "$TERMINAL_INI" - WINEPATH="$(winepath -w "$TERMINAL_DIR");C:\\Apps\\AHK" \ - timeout 60 \ - wine AutoHotkey /ErrorStdOut "$ahk_path" "${dstfile}" ${@:3} - [ -n "$OPT_VERBOSE" ] && times >&2 - echo "${dstfile}" -} - # Copy ini settings from templates. # Usage: ini_copy ini_copy() { @@ -428,8 +409,14 @@ ea_find() { file=${file##*/} fi [ -f "$file" ] && { echo "$file"; return; } - local exact=$(find -L . "$ROOT" ~ -maxdepth 4 -type f '(' -iname "$file" -o -iname "${file%.*}.mq?" -o -name "${file%.*}.ex?" ')' -print -quit) - local match=$(find -L . "$ROOT" ~ -maxdepth 4 -type f '(' -iname "*${file%.*}*.mq?" -o -iname "*${file%.*}*.ex?" ')' -print -quit) + exact=$( + find -L . "$ROOT" ~ -maxdepth 4 -type f '(' -iname "$file" -o -iname "${file%.*}.mq?" ')' -print -quit || \ + find -L . "$ROOT" ~ -maxdepth 4 -type f '(' -iname "$file" -o -name "${file%.*}.ex?" ')' -print -quit + ) + [ -z "$exact" ] && match=$( + find -L . "$ROOT" ~ -maxdepth 4 -type f -iname "*${file%.*}*.mq?" -print -quit || \ + find -L . "$ROOT" ~ -maxdepth 4 -type f -iname "*${file%.*}*.ex?" -print -quit + ) [ -n "$exact" ] && echo ${exact#./} || echo ${match#./} cd - &>/dev/null } @@ -448,8 +435,14 @@ script_find() { file=${file##*/} fi [ -f "$file" ] && { echo "$file"; return; } - local exact=$(find -L . "$ROOT" ~ -maxdepth 4 -type f '(' -iname "$file" -o -name "${file%.*}.ex?" ')' -print -quit) - local match=$(find -L . "$ROOT" ~ -maxdepth 4 -type f '(' -iname "*${file%.*}*.mq?" -o -iname "*${file%.*}*.ex?" ')' -print -quit) + exact=$( + find -L . "$ROOT" ~ -maxdepth 4 -type f '(' -iname "$file" -o -iname "${file%.*}.mq?" ')' -print -quit || \ + find -L . "$ROOT" ~ -maxdepth 4 -type f '(' -iname "$file" -o -iname "${file%.*}.ex?" ')' -print -quit + ) + [ -z "$exact" ] && match=$( + find -L . "$ROOT" ~ -maxdepth 4 -type f -iname "*${file%.*}*.mq?" -print -quit || \ + find -L . "$ROOT" ~ -maxdepth 4 -type f -iname "*${file%.*}*.ex?" -print -quit + ) [ -n "$exact" ] && echo ${exact#./} || echo ${match#./} cd - &>/dev/null } @@ -826,57 +819,6 @@ quick_run() { show_logs 40 } -# Set input value in the SET file. -# Usage: input_set [key] [value] [file] -input_set() { - local key="$1" - local value="$2" - local file="${3:-$TESTER_DIR/$EA_SETFILE}" - [ -s "$file" ] - read -ra vargs <<<$EX_ARGS - vargs+=("-u NONE") - if [ -n "$value" ]; then - echo "Setting '$key' to '$value' in $(basename "$file")" >&2 - ex +"%s/$key=\zs.*$/$value/" -scwq! ${vargs[@]} "$file" >&2 || exit 1 - else - echo "Value for '$key' is empty, ignoring." >&2 - fi -} - -# Get input value from the SET file. -# Usage: input_get [key] [file] -input_get() { - local key="$1" - local file="${2:-$TESTER_DIR/$EA_SETFILE}" - [ -s "$file" ] - value="$(grep -om1 "$key=[.0-9a-zA-Z-]\+" "$file" | cut -d= -f2-)" - echo $value -} - -# Copy input value from the SET file to MQL file. -# Usage: input_copy [key] [src_file] [dst_file] (dst_file2...) -input_copy() { - local key=$1 - local file_src=$2 - local retries=5 - read -ra files <<<${@:3} - read -ra vargs <<<$EX_ARGS - vargs+=("-u NONE") - [ -s "$file_src" ] - for file_dst in "${files[@]}"; do - [ -s "$file_dst" ] - value=$(input_get "^$key" "$file_src") - echo "Setting '$key' to '$value' in $(basename "$file_dst")" >&2 - retries=5 - while ! ex +"%s/\s${key}[^=]=[^0-9]\zs[^;]\+/${value}/" -scwq! ${vargs[@]} "$file_dst" >&2; do - sleep 1 - ((retries-=1)) - echo "Retrying ($retries left)..." >&2 - [ $retries -le 0 ] && break - done - done -} - # Set value in the INI file. # Usage: ini_set [key] [value] [file] ini_set() { @@ -1028,5 +970,4 @@ set_digits() { set_symbol_value $digits $SRAW_OFF_DIGITS psize="0.$(for ((i=1;i<=digits-1;i++)); do printf 0; done)1" set_symbol_double $psize $SRAW_OFF_PSIZE - cp $VFLAG "$ROOT/conf/accounts-d${digits}.ini" "$TERMINAL_DIR"/config/accounts.ini } diff --git a/scripts/.funcs.inc.sh b/scripts/.funcs.inc.sh index 32486053..e97f3d54 100644 --- a/scripts/.funcs.inc.sh +++ b/scripts/.funcs.inc.sh @@ -57,11 +57,10 @@ join_by() { } # Check required files. +# Usage: check_files check_files() { - if [ "$SERVER" != "default" ]; then - local symbols_raw_default="$HISTORY_DIR/default/symbols.raw" - local symbols_raw="$HISTORY_DIR/$SERVER/symbols.raw" - [ -s "$symbols_raw" ] || cp $VFLAG "$symbols_raw_default" "$symbols_raw" + if [ "$SERVER" != "default" ] && [ -w "$HISTORY_DIR/$SERVER" ]; then + [ -s "$symbols_raw" ] || cp $VFLAG "$HISTORY_DIR/default/symbols.raw" "$HISTORY_DIR/$SERVER/symbols.raw" fi } @@ -92,6 +91,18 @@ get_time() { fi } +# Get the max of two values. +# Usage: get_min (int) (int) +get_min() { + echo $(( $1 < ${2:-0} ? $1 : ${2:-0} )) +} + +# Get the max of two values. +# Usage: get_max (int) (int) +get_max() { + echo $(( $1 > ${2:-0} ? $1 : ${2:-0} )) +} + # Check logs for errors. # Usage: check_log_errors check_log_errors() { diff --git a/scripts/.funcs.sets.inc.sh b/scripts/.funcs.sets.inc.sh index 707e72f9..e6a76db4 100644 --- a/scripts/.funcs.sets.inc.sh +++ b/scripts/.funcs.sets.inc.sh @@ -14,8 +14,86 @@ CWD="${CWD:-$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)}" # Usage: set_opt_params [param] [start] [end] (step) set_opt_params() { # Optimization settings (F-On, 1-Min, 2-Step, 3-Max). - input_set ^$1,F 1 # On. - input_set ^$1,1 $2 # Min. - input_set ^$1,2 ${4:-1} # Step. - input_set ^$1,3 $3 # Max. + input_set ^"$1",F 1 # On. + input_set ^"$1",1 "$2" # Min. + input_set ^"$1",2 "${4:-1}" # Step. + input_set ^"$1",3 "$3" # Max. +} + +# Disables optimization params +# Usage: dis_opt_params [param] +dis_opt_params() { + input_set ^"$1",F 0 # Off. +} + +# Set input value in the SET file. +# Usage: input_set [key] [value] [file] +input_set() { + local key="$1" + local value="$2" + local file="${3:-$TESTER_DIR/$EA_SETFILE}" + file=${file:-$SETFILE} + [ -s "$file" ] + read -ra vargs <<<$EX_ARGS + vargs+=("-u NONE") + if [ -n "$value" ]; then + echo "Setting '$key' to '$value' in $(basename "$file")" >&2 + ex +"%s/$key=\\zs.*$/$value/" -scwq! ${vargs[@]} "$file" >&2 || exit 1 + else + echo "Value for '$key' is empty, ignoring." >&2 + fi +} + +# Get input value from the SET file. +# Usage: input_get [key] [file] +input_get() { + local key="$1" + local file="${2:-$TESTER_DIR/$EA_SETFILE}" + file=${file:-$SETFILE} + [ -s "$file" ] + value="$(grep -om1 "$key=[.0-9a-zA-Z-]\\+" "$file" | cut -d= -f2-)" + echo "$value" +} + +# Copy input value from the SET file to MQL file. +# Usage: input_copy [key] [src_file] [dst_file] (dst_file2...) +input_copy() { + local key=$1 + local file_src=$2 + local retries=5 + read -ra files <<<${@:3} + read -ra vargs <<<$EX_ARGS + vargs+=("-u NONE") + [ -s "$file_src" ] + for file_dst in "${files[@]}"; do + [ -s "$file_dst" ] + value=$(input_get "^$key" "$file_src") + echo "Setting '$key' to '$value' in $(basename "$file_dst")" >&2 + retries=5 + while ! ex +"%s/\\s${key}[^=]=[^0-9]\\zs[^;]\\+/${value}/" -scwq! ${vargs[@]} "$file_dst" >&2; do + sleep 1 + ((retries-=1)) + echo "Retrying ($retries left)..." >&2 + [ $retries -le 0 ] && break + done + done +} + +# Experts SET file. Returns exported filename. +# Usage: export_set [EA/pattern] (dst/file) (...args) +export_set() { + local name=${1:-$TEST_EXPERT} + local dstfile=${2:-${name}.set} + local ea_path=$name + ahk_path="$(winepath -w "$SCR"/ahk/export_set.ahk)" + [ ! -s "$name" ] && ea_path=$(ea_find "${name##/}") + [ ! -f "$EXPERTS_DIR/$ea_path" ] && { echo "Error: Cannot find EA: ${name}!" >&2; return; } + [[ "$ea_path" =~ 'mq' ]] && compile_ea "$name" >&2 + set_display >&2 + ini_set "^Expert" "$(basename "${ea_path/\//\\\\}" ."${ea_path##*.}")" "$TERMINAL_INI" + WINEPATH="$(winepath -w "$TERMINAL_DIR");C:\\Apps\\AHK" \ + timeout 60 \ + wine AutoHotkey /ErrorStdOut "$ahk_path" "${dstfile}" "${@:3}" + [ -n "$OPT_VERBOSE" ] && times >&2 + echo "${dstfile}" } diff --git a/scripts/.vars.inc.sh b/scripts/.vars.inc.sh index 9a8b5ba2..759f9dc1 100644 --- a/scripts/.vars.inc.sh +++ b/scripts/.vars.inc.sh @@ -23,10 +23,14 @@ is_vm && set -x TERMINAL_EXE="$(find "$ROOT" "$OUT" "$HOME" -not -path "*/WebInstall/*" -name terminal.exe -print -quit)" TERMINAL_DIR="${TERMINAL_DIR:-$([ -f "$TERMINAL_EXE" ] && dirname "$TERMINAL_EXE" || true)}" MTEDITOR_EXE="$([ -d "$TERMINAL_DIR" ] && find "$TERMINAL_DIR" -name metaeditor.exe -print -quit || true)" +TERMINAL5_EXE="$(find "$ROOT" "$OUT" "$HOME" -not -path "*/WebInstall/*" -name terminal64.exe -print -quit)" +TERMINAL5_DIR="${TERMINAL5_DIR:-$([ -f "$TERMINAL5_EXE" ] && dirname "$TERMINAL5_EXE" || true)}" +MTEDITOR5_EXE="$([ -d "$TERMINAL5_DIR" ] && find "$TERMINAL5_DIR" -name metaeditor64.exe -print -quit || true)" +MTTESTER5_EXE="$([ -d "$TERMINAL5_DIR" ] && find "$TERMINAL5_DIR" -name metatester64.exe -print -quit || true)" is_vm && set +x MQL_DIR="MQL4" if [ -n "$TERMINAL_DIR" ]; then - TERMINAL_ARG="/skipupdate /portable" + TERMINAL_ARG="${TERMINAL_ARG:-/skipupdate /portable}" TERMINAL_CNF="${TERMINAL_DIR}/config" TERMINAL_INI="${TERMINAL_INI:-$TERMINAL_CNF/$CONF_TERM}" TESTER_INI="${TESTER_INI:-$TERMINAL_CNF/$CONF_TEST}" diff --git a/scripts/get_bt_data.sh b/scripts/get_bt_data.sh index ad3ff648..c1328cc9 100755 --- a/scripts/get_bt_data.sh +++ b/scripts/get_bt_data.sh @@ -130,9 +130,9 @@ case $bt_src in wait # Wait for the background tasks to finish. echo "Extracting..." >&2 gunzip -kh >& /dev/null && keep="-k" || true # Check if gunzip supports -k parameter. - find "$dest_dir" -type f -name "*.gz" -print0 | while IFS= read -r -d '' file; do - gunzip $vflag $keep "$file" - done + cd "$dest_dir" + gunzip $vflag $keep ${fxt_files[@]} ${hst_files[@]} + cd - &> /dev/null echo "Moving..." >&2 find "$dest_dir" -type f -name "*.fxt" -exec mv $vflag "{}" "$TICKDATA_DIR" ';' find "$dest_dir" -type f -name "*.hst" -exec mv $vflag "{}" "$HISTORY_DIR/${SERVER:-default}" ';' diff --git a/scripts/install_mt4.sh b/scripts/install_mt4.sh index fc192b5b..f0b506cb 100755 --- a/scripts/install_mt4.sh +++ b/scripts/install_mt4.sh @@ -2,18 +2,37 @@ # Script to install MT4 platform using winetricks. [ -n "$OPT_NOERR" ] || set -e [ -n "$OPT_TRACE" ] && set -x -CWD="$(cd -P -- "$(dirname -- "$0")" 2>/dev/null && pwd -P || pwd -P)" -WURL=${WURL:-https://raw.githubusercontent.com/Winetricks/winetricks/master/src/winetricks} +CWD="$( (cd -P -- "$(dirname -- "$0")" 2>/dev/null && pwd -P) || pwd -P)" +type winetricks >/dev/null + +# Load variables. +export WINETRICKS_DOWNLOADER_RETRIES=${WINETRICKS_DOWNLOADER_RETRIES:-10} +export WINETRICKS_DOWNLOADER=curl + +# Initializing +echo "Initializing..." >&2 +curl -s ifconfig.me/all.json # Load the shell functions. . "$CWD/.funcs.inc.sh" . "$CWD/.funcs.cmds.inc.sh" +# Activates display. +echo "Configuring display..." >&2 +set_display + +# Updates Wine configuration. +echo "Updating configuration..." >&2 +wineboot -u + echo "Installing winhttp..." >&2 -sh -s winhttp < <(wget -qO- $WURL) +winetricks -q winhttp + +echo "Installing .NET..." >&2 +winetricks -q dotnet472 echo "Installing platform..." >&2 -sh -s "$CWD"/install_mt4.verb < <(wget -qO- $WURL) +winetricks -q -v mt4 echo "Installation successful." >&2 echo "${BASH_SOURCE[0]} done." >&2 diff --git a/scripts/install_mt4x.sh b/scripts/install_mt4x.sh index 622c0f05..54ac8f33 100755 --- a/scripts/install_mt4x.sh +++ b/scripts/install_mt4x.sh @@ -2,24 +2,22 @@ # Script to install MT platform using xdotool. [ -n "$OPT_NOERR" ] || set -e [ -n "$OPT_TRACE" ] && set -x -CWD="$(cd -P -- "$(dirname -- "$0")" 2>/dev/null && pwd -P || pwd -P)" -DTMP=$(mktemp -d) +CWD="$( (cd -P -- "$(dirname -- "$0")" 2>/dev/null && pwd -P) || pwd -P)" EXEFILE=mt4setup.exe -WURL=${WURL:-https://raw.githubusercontent.com/Winetricks/winetricks/master/src/winetricks} # Check the dependencies. -type wget xdotool xwininfo wine ar >/dev/null +type winetricks wget xdotool xwininfo wine ar >/dev/null # Load the shell functions. . "$CWD/.funcs.inc.sh" . "$CWD/.funcs.cmds.inc.sh" echo "Installing winhttp..." >&2 -sh -s winhttp < <(wget -qO- $WURL) +winetricks -q winhttp echo "Downloading MT4 installer..." >&2 [ ! -f "$HOME/$EXEFILE" ] \ - && wget -O "$HOME/$EXEFILE" -ct3 --content-disposition ${MT_URL:-"https://download.mql5.com/cdn/web/8472/mt4/xmuk4setup.exe"} + && wget -O "$HOME/$EXEFILE" -ct3 --content-disposition "${MT_URL:-"https://download.mql5.com/cdn/web/8472/mt4/xmuk4setup.exe"}" [ -f "$HOME/$EXEFILE" ] # Prints information of the window status in the background. @@ -34,11 +32,11 @@ wine "$HOME/$EXEFILE" & echo "Waiting for installer to initialize..." >&2 while ! WID=$(xdotool getactivewindow 2>/dev/null); do sleep 2; done -echo "Installing $(xdotool getwindowname $WID)..." >&2 +echo "Installing $(xdotool getwindowname "$WID")..." >&2 while WID=$(xdotool getactivewindow); do - xdotool key --window $WID --delay 500 Alt+N + xdotool key --window "$WID" --delay 500 Alt+N sleep 20 - xdotool key --window $WID --delay 500 Tab space + xdotool key --window "$WID" --delay 500 Tab space done # Workaround for Chrome launching when installer finishes. @@ -47,7 +45,7 @@ pkill -KILL chrome || true echo "Waiting for platform to start..." >&2 while ! WID=$(xdotool getactivewindow 2>/dev/null); do sleep 5; done -xwininfo -id $WID -tree +xwininfo -id "$WID" -tree # Close running MT4 instance, first the two login popup window, secondly application itself. echo "Closing platform..." >&2 diff --git a/scripts/install_mt5.sh b/scripts/install_mt5.sh index 523b6bd6..ef5117f8 100755 --- a/scripts/install_mt5.sh +++ b/scripts/install_mt5.sh @@ -3,21 +3,43 @@ [ -n "$OPT_NOERR" ] || set -e [ -n "$OPT_TRACE" ] && set -x CWD="$(cd -P -- "$(dirname -- "$0")" 2>/dev/null; pwd -P)" -WURL="https://raw.githubusercontent.com/Winetricks/winetricks/master/src/winetricks" +type winetricks >/dev/null + +# Load variables. +export WINETRICKS_DOWNLOADER_RETRIES=${WINETRICKS_DOWNLOADER_RETRIES:-10} +export WINETRICKS_DOWNLOADER=curl + +# Initializing +echo "Initializing..." >&2 +curl -s ifconfig.me/all.json # Load the shell functions. . "$CWD/.funcs.inc.sh" . "$CWD/.funcs.cmds.inc.sh" -# Prints information of the window status in the background. +# Activates display. +echo "Configuring display..." >&2 set_display -live_stats & + +# Updates Wine configuration. +echo "Updating configuration..." >&2 +wineboot -u echo "Installing winhttp..." >&2 -sh -s winhttp < <(wget -qO- $WURL) +winetricks -q winhttp + +echo "Installing .NET..." >&2 +winetricks -q dotnet472 echo "Installing platform..." >&2 -sh -s "$CWD"/install_mt5.verb < <(wget -qO- $WURL) +winetricks -q "$CWD"/install_mt5.verb -echo "Installation successful." >&2 +. "$CWD"/.vars.inc.sh +if [ -n "$TERMINAL5_DIR" ]; then + echo "Terminal path: $TERMINAL5_DIR" >&2 + echo "Installation successful." >&2 +else + echo "Installation failed!" >&2 + exit 1 +fi echo "${BASH_SOURCE[0]} done." >&2 diff --git a/scripts/options.txt b/scripts/options.txt index 885329d1..9205884a 100644 --- a/scripts/options.txt +++ b/scripts/options.txt @@ -97,7 +97,7 @@ Supported options: -X (file) Invoke script file n exit after the successful test. -y (year) - Year to test. Default: 2017 + Year to test. Default: 2018 Variable (uint/string): BT_YEARS -_ Dry run. @@ -109,7 +109,13 @@ Other supported variables (without options assigned): - BT_DAYS (uint/string) Specify days to test. Default: 1-31. - BT_TESTMODEL (uint) - This controls type of backtest data being downloaded and used. + This controls type of backtest data being tested. + Values: 0 (default) - Every tick, 1 - Control points, 2 - Open prices only +- BT_PERIOD_FXT (string) + Overrides FXT file to use for the test. + By default, it is the same as BT_PERIOD. +- BT_TESTMODEL_FXT (uint) + This controls type of backtest data being downloaded in FXT format. Values: 0 (default) - Every tick, 1 - Control points, 2 - Open prices only - OPT_NOERR (bool) Disables errexit flag to prevent exiting script on failure. diff --git a/scripts/provision.sh b/scripts/provision.sh index 1f17bda9..8b9e2dd5 100755 --- a/scripts/provision.sh +++ b/scripts/provision.sh @@ -5,8 +5,8 @@ # # Initialize script. -[ -n "$OPT_NOERR" ] || set -e -[ -n "$OPT_TRACE" ] && set -x +(( "$OPT_NOERR" )) || set -e +(( "$OPT_TRACE" )) && set -x if [ ! -d /vagrant -a ! -d /home/travis -a ! -f /.dockerenv ]; then echo "Error: This script needs to be run within container." >&2 exit 1 @@ -70,6 +70,8 @@ case "$(uname -s)" in # For Ubuntu/Debian. echo "Configuring APT..." >&2 + apt-config dump | grep -we Recommends -e Suggests | sed s/1/0/ | tee /etc/apt/apt.conf.d/99norecommend + apt-config dump | grep -we Recommends -e Suggests if command -v dpkg-reconfigure > /dev/null; then # Perform an unattended installation of a Debian packages. @@ -88,7 +90,7 @@ case "$(uname -s)" in fi # Update APT index. - [ -z "$NO_APT_UPDATE" ] && ( + ! (( "${NO_APT_UPDATE:-0}" )) && ( echo "Updating APT packages..." >&2 apt-get -qq update ) @@ -97,17 +99,32 @@ case "$(uname -s)" in command -v curl &>/dev/null || apt-get install -qq curl command -v wget &>/dev/null || apt-get install -qq wget + # CA certificates to allow SSL-based applications to check for the authenticity of SSL connections. + echo "Installing CA certificates..." >&2 + apt-get install -qq ca-certificates # Add PPA/Wine repository. echo "Adding PPA/Wine repository..." >&2 # Adds GPG release key. - apt-key add < <(curl -sq https://dl.winehq.org/wine-builds/winehq.key) + apt-key add < <(curl -S https://dl.winehq.org/wine-builds/winehq.key) # APT dependencies (for the add-apt-repository). command -v add-apt-repository || apt-get install -qq software-properties-common python-software-properties # Adds APT Wine repository. add-apt-repository -y "deb http://dl.winehq.org/wine-builds/ubuntu/ ${DISTRIB_CODENAME:-xenial} main" + # Install Charles proxy. + if (( "$PROVISION_CHARLES" )); then + # Install Charles Root Certificate (if available). + curl -L chls.pro/ssl > /usr/local/share/ca-certificates/charles.crt && update-ca-certificates + # Adds GPG release key. + apt-key add < <(curl -S https://www.charlesproxy.com/packages/apt/PublicKey) + # Adds APT Wine repository. + add-apt-repository -y "deb https://www.charlesproxy.com/packages/apt/ charles-proxy main" + # Install HTTPS transport driver. + apt-get install -qq apt-transport-https + fi + # Update APT index. - [ -z "$NO_APT_UPDATE" ] && ( + ! (( "${NO_APT_UPDATE:-0}" )) && ( echo "Updating APT packages..." >&2 apt-get -qq update ) @@ -116,27 +133,30 @@ case "$(uname -s)" in echo "Installing APT packages..." >&2 apt-get install -qq build-essential # Install C, C++ compilers and development (make). apt-get install -qq language-pack-en # Language pack to prevent an invalid locale. - apt-get install -qq ca-certificates apt-get install -qq dbus # Required for Debian AMI on EC2. # Install wine and dependencies. # @see: https://wiki.winehq.org/Ubuntu - apt-get install -qq winehq-devel wine-gecko --install-recommends # Install Wine. + apt-get install -qq winehq-staging # Install Wine. + apt-get install -qq wine-gecko winbind # Install Wine recommended libraries. apt-get install -qq xvfb xdotool x11-utils xterm # Virtual frame buffer and X11 utils. # Install Winetricks. - curl -sL https://raw.githubusercontent.com/Winetricks/winetricks/master/src/winetricks | install /dev/stdin /usr/local/bin/winetricks + winetricks_url="https://raw.githubusercontent.com/kenorb-contrib/winetricks/mt4/src/winetricks" + curl -sL ${winetricks_url} | install /dev/stdin /usr/local/bin/winetricks # Install AHK. - if [ -n "$PROVISION_AHK" ]; then + if (( "$PROVISION_AHK" )); then echo "Installing AutoHotkey..." >&2 su - $user -c " set -x export DISPLAY=:1.0 + export WINEDLLOVERRIDES=mscoree,mshtml= echo \$DISPLAY xdpyinfo &>/dev/null || (! pgrep -a Xvfb && Xvfb \$DISPLAY -screen 0 1024x768x16) & + wineboot -i wget -qP /tmp -nc 'https://github.com/Lexikos/AutoHotkey_L/releases/download/v1.1.30.01/AutoHotkey_1.1.30.01_setup.exe' && \ - wine /tmp/AutoHotkey_*.exe /S /D='C:\\Apps\\AHK' && \ + wine64 /tmp/AutoHotkey_*.exe /S /D='C:\\Apps\\AHK' && \ rm -v /tmp/AutoHotkey_*.exe && \ (pkill Xvfb || true) " @@ -149,14 +169,44 @@ case "$(uname -s)" in fi fi + # Install Charles proxy. + if (( "$PROVISION_CHARLES" )); then + apt-get install -qq charles-proxy3 + fi + + # Install Mono. + if (( "$PROVISION_MONO" )); then + echo "Installing Wine Mono..." >&2 + apt-get install -qq wine-mono + su - $user -c " + set -x + export DISPLAY=:1.0 + export WINEDLLOVERRIDES=mscoree,mshtml= + echo \$DISPLAY + xdpyinfo &>/dev/null || (! pgrep -a Xvfb && Xvfb \$DISPLAY -screen 0 1024x768x16) & + wget -qP /tmp -nc 'http://dl.winehq.org/wine/wine-mono/4.8.3/wine-mono-4.8.3.msi' && \ + wine64 msiexec /i /tmp/wine-mono-4.8.3.msi + rm -v /tmp/*.msi && \ + (pkill Xvfb || true) + " + mono_path=$(su - $user -c 'winepath -u "C:\windows\mono"'); + if [ -d "$mono_path" ]; then + echo "Mono installed successfully!" >&2 + else + echo "Error: Mono installation failed!" >&2 + exit 1 + fi + fi + # Setup VNC. - if [ -n "$PROVISION_VNC" ]; then + if (( "$PROVISION_VNC" )); then echo "Installing VNC..." >&2 apt-get install -qq x11vnc fluxbox fi # Install other CLI tools. - apt-get install -qq less binutils coreutils moreutils cabextract zip unzip # Common CLI utils. + apt-get install -qq less binutils coreutils moreutils # Common CLI utils. + apt-get install -qq cabextract zip unzip p7zip-full # Compression tools. apt-get install -qq git realpath links tree pv bc # Required commands. apt-get install -qq html2text jq # Required parsers. apt-get install -qq imagemagick # ImageMagick. @@ -171,14 +221,14 @@ case "$(uname -s)" in ) & # Setup SSH if requested. - if [ -n "$PROVISION_SSH" ]; then + if (( "$PROVISION_SSH" )); then apt-get install -qq openssh-server [ ! -d /var/run/sshd ] && mkdir -v /var/run/sshd sed -i'.bak' 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd fi # Setup sudo if requested. - if [ -n "$PROVISION_SUDO" ]; then + if (( "$PROVISION_SUDO" )); then apt-get install -qq sudo sed -i'.bak' "s/^%sudo.*$/%sudo ALL=(ALL:ALL) NOPASSWD:ALL/g" /etc/sudoers fi diff --git a/scripts/run_backtest.sh b/scripts/run_backtest.sh index 6a1b6c36..3fc5ccf7 100755 --- a/scripts/run_backtest.sh +++ b/scripts/run_backtest.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Script to run backtest test. -# E.g. run_backtest.sh -v -t -e MACD -f "/path/to/file.set" -c USD -p EURUSD -d 2000 -m 1-2 -y 2017 -s 20 -b DS -r Report -O "_optimization_results" +# E.g. run_backtest.sh -v -t -e MACD -f "/path/to/file.set" -c USD -p EURUSD -d 2000 -m 1-2 -y 2018 -s 20 -b DS -r Report -O "_optimization_results" # Initialize variables. [ -n "$OPT_NOERR" ] || set -e @@ -17,7 +17,7 @@ type git pgrep xargs ex xxd od perl xdpyinfo >/dev/null usage() { printf "Usage: %s (args)\n\n" "$0" cat $CWD/options.txt - printf "\n\nExample: %s -v -t -e MACD -p EURUSD -c USD -d 2000 -y 2017 -m 1-2 -S 20 -b DS -T M30\n" "$0" + printf "\n\nExample: %s -v -t -e MACD -p EURUSD -c USD -d 2000 -y 2018 -m 1-2 -S 20 -b DS -T M30\n" "$0" } # Invoke on test success. @@ -293,7 +293,7 @@ while getopts $ARGS arg; do SCRIPT=${OPTARG} ;; - y) # Year to test (e.g. 2017, 2011-2015). + y) # Year to test (e.g. 2017, 2018, 2011-2015). BT_YEARS=${OPTARG} ;; @@ -664,7 +664,7 @@ fi # Sets a lot step if present. if [ -n "$BT_LOTSTEP" ]; then - echo "Configuring lot step ($BT_LOTSTEP)..." >&2 + echo "Setting lot step in FXT files ($BT_LOTSTEP)..." >&2 set_lotstep $BT_LOTSTEP fi @@ -710,14 +710,16 @@ fi # Download backtest data if required. BT_PERIOD=$(ini_get ^TestPeriod) +BT_PERIOD_FXT=${BT_PERIOD_FXT:-$BT_PERIOD} +BT_TESTMODEL_FXT=${BT_TESTMODEL_FXT:-0} if [ -n "$TEST_EXPERT" ]; then echo "Checking backtest data (${BT_SRC:-DS})..." - bt_key=$BT_SYMBOL-$(join_by - ${BT_YEARS[@]:-2017})-${BT_SRC:-DS} + bt_key=$BT_SYMBOL-$(join_by - ${BT_YEARS[@]:-2018})-${BT_SRC:-DS} bt_data=$(ini_get "bt_data" "$CUSTOM_INI") # Generate backtest files if not present. if [ -z "$(find "$TERMINAL_DIR" -name "${BT_SYMBOL}*_0.fxt" -print -quit)" ] || [ "${bt_data%.*}" != "$bt_key" ]; then env SERVER=$SERVER OPT_VERBOSE=$OPT_VERBOSE OPT_TRACE=$OPT_TRACE \ - $SHELL $SCR/get_bt_data.sh $BT_SYMBOL "$(join_by - ${BT_YEARS[@]:-2017})" ${BT_SRC:-DS} ${BT_PERIOD} ${BT_TESTMODEL} + "$SHELL" "$SCR"/get_bt_data.sh "$BT_SYMBOL" "$(join_by - "${BT_YEARS[@]:-2018}")" "${BT_SRC:-DS}" "${BT_PERIOD_FXT}" "${BT_TESTMODEL_FXT}" if [ -n "$OPT_VERBOSE" ]; then cd "$TERMINAL_DIR" find . '(' -name "*.hst" -o -name "*.fxt" ')' -ls diff --git a/tests/TestLeverage.mq4 b/tests/TestLeverage.mq4 new file mode 100644 index 00000000..77f1ba75 --- /dev/null +++ b/tests/TestLeverage.mq4 @@ -0,0 +1,36 @@ +//+------------------------------------------------------------------+ +//| TestLeverage | +//| Copyright 2016-2019, 31337 Investments Ltd | +//| https://github.com/EA31337 | +//+------------------------------------------------------------------+ + +/* + * This file is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +//+------------------------------------------------------------------+ +//| Test whether account leverage is correct. +//| @docs: https://www.mql5.com/en/docs/constants/environment_state/accountinformation +//+------------------------------------------------------------------+ + +#property strict + +/** + * Implements Init even handler. + */ +int OnInit() { + long leverage = AccountInfoInteger(ACCOUNT_LEVERAGE); + PrintFormat("Account leverage : %d", leverage); + return INIT_SUCCEEDED; +} diff --git a/tests/TestLotstep.mq4 b/tests/TestLotstep.mq4 index e2932a5c..47c57003 100644 --- a/tests/TestLotstep.mq4 +++ b/tests/TestLotstep.mq4 @@ -27,10 +27,11 @@ */ #property strict + int OnInit() { long symbol_spread = SymbolInfoInteger(_Symbol, SYMBOL_SPREAD); int real_spread = (int)MathRound((Ask - Bid) * MathPow(10, Digits)); - double lot_step = MarketInfo(_Symbol, MODE_LOTSTEP); + double lot_step = SymbolInfoDouble(_Symbol, SYMBOL_VOLUME_STEP); // Same as: MarketInfo(symbol, MODE_LOTSTEP); Print("Testing lot step..."); PrintFormat("Symbol digits : %g", Digits); PrintFormat("Lot step : %g", lot_step);