Merge pull request #137 from EA31337/dev

Improves compiling process of EA and usage
EA31337 · Jan 15, 2019 · 3037bc4 · 3037bc4
2 parents 36c74f3 + 9b047da
commit 3037bc4
Show file tree

Hide file tree

Showing 9 changed files with 420 additions and 109 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -25,8 +25,9 @@ env:
   - secure: "waZbxfpNTIp/etY6lwR8GB+aSWDIw73tu6HxFdwVYdDJ/25SlY+gkJMCPqlpkjA1Aenv1I3TDDd/3dQIMYUeNbgIKOx6AtWkddnfTykqYYFJ4j6MlWPxZKaxDLjehLvjj0ZqkfXcswgXDaIoNnunKbLKiVohprLhO9Yg6I/j1LfzmPPGmFez2Mg2K0jFmri1i8prxpL4cctH0cR9NEZ+Lw13nDjY4b7AMoY6mzt6TFe9/yEt1OzYVGAh36hWgI1AL8h86cyDlUYHMRrzZjMKUGZFyOkkz3BRA6wvyOtjRhNlh2SIY+Y+HiwQaLONeQGpZ9YjNCUb+f5GIxXyEry9RV1oaMNIUnXNSQstUvgzy3QO2muD468KhfvPafpPj0gDHmfGEH+2IMcrz/WYNobsdN9JOjvPWutnxgYAzd27jwDIQ4D2WewZ3COgrRXxCAgwsOQKvlzDaFAOvPObOnCmIVsbOpONE3nSRGnrQpxJl0aaspoHLnVyKAUeCTAMhtSZirhWO+Iob9mhicB+9xEMRpxFnRD/FBUcSY/SyVlZjXxK9zIqn9t5KBK0iVZ10l/O5nV6sM7YfpAz8d4MqJDAGT05tKn/o2uDJtVvnl6szCIP5iQwsMUsQqQCuNPbmfm1V/q2ipjxLwiwOx2NRbSjFx3/5NnyXiIqehmvlfnfUjQ="
   - secure: "oCTQnzDJX44mVNMpSutYkBWyi5hCoIRAvsDBWi/VAMBY7lFDiwbKQocCdorNlN/GBT8EISmyOlLGt78hjAIPckarbmPou5zIteIRuqaQ2xK1ShbseFAjpZaftOjljqE6ZoD5L6K5Ld0NQwiJqhBIQsv2cFwJY2706vl7Vgf/Kjn/B8/2OhPxgz8w+BpzBD+0z2JnBEpEON9wU8XeWXUzPKKYFHDHH/o839TgUjqPAaN4gU0XUrH1pygk3kdkHvC1lekoyevUePFaU6icwEPU6T7HGfw7ar+1OemzPzSQ5b/IG8U4NslUQQwjP8T4UfO964RuiJRnHVY0Y2FZZ/7XQHE5TBvh44XGuNAwhklQ4MBnz9rRpId4HMQKRaLKBGpjYeUqz/fCYDEoocRfQt3qqiPRoC/Z9yGnFy56B6iCx33MD5wzZDY3l0IBbwBK6iMrLcv5l2IYMyAIZUYmviwj18szkHr0ieyOAn+FGnnLy2ydDdYMHkwu1PVeI/nvXn78lyCHGs1aBssKiP1KK50przryb7Ti60xc2P9IAP8+NumZ8z6Onfl64XxwWIM4YicR+JZgDCZKN+ajGwIXqIzLsxj6eop0hll3XQvaYilOfDFj6UkDidZTWyhaYGdsW0JaD84r1OGpO9MwZQkSObuqWjgq4kf2qgtoA2WJMdQPDJg="
   matrix:
+  - CMD="make docker-ci DOCKER_TAG=$DOCKER_TAG; [[ '$TRAVIS_PULL_REQUEST' = 'false' ]] && make docker-push DOCKER_TAG=$DOCKER_TAG || true"
   - CMD="scripts/eval.sh help; scripts/run_backtest.sh -?"
-  - CMD="shellcheck -e SC1004,SC1083,SC1090,SC1091,SC1117,SC2004,SC2005,SC2015,SC2027,SC2032,SC2033,SC2034,SC2035,SC2046,SC2068,SC2070,SC2071,SC2086,SC2091,SC2103,SC2116,SC2124,SC2128,SC2145,SC2154,SC2155,SC2162,SC2164,SC2166,SC2181,SC2188,SC2206,SC2221,SC2222,SC2230 scripts/*.sh scripts/.*.sh"
+  - CMD="shellcheck -e SC1004,SC1083,SC1090,SC1091,SC1117,SC2004,SC2005,SC2015,SC2027,SC2032,SC2033,SC2034,SC2035,SC2046,SC2068,SC2070,SC2071,SC2086,SC2091,SC2103,SC2116,SC2124,SC2128,SC2145,SC2154,SC2155,SC2162,SC2164,SC2166,SC2181,SC2188,SC2206,SC2207,SC2221,SC2222,SC2230 scripts/*.sh scripts/.*.sh"
   - CMD="$DOCKER_RUN 'help; run_backtest;'"
   - CMD="$DOCKER_RUN 'for file in \$(ls /opt/tests/scripts); do run_backtest -s \$file; done'"
   - CMD="$DOCKER_RUN 'for file in TestFXTHeader; do run_backtest -v -t -p EURUSD -y 2017 -m 1 -e \$file; done'"
@@ -40,7 +41,6 @@ env:
   - CMD="$DOCKER_RUN install_mt 4x"
   - CMD="$DOCKER_RUN install_mt 5"
   - CMD="docker run -it --entrypoint /lib64/ld-linux-x86-64.so.2 -u root $DOCKER_REPO /usr/bin/find / -xdev -type f -size +10M -ls"
-  - CMD="make docker-ci DOCKER_TAG=$DOCKER_TAG; [[ '$TRAVIS_PULL_REQUEST' = 'false' ]] && make docker-push DOCKER_TAG=$DOCKER_TAG || true"
   - CMD="sudo scripts/provision.sh"
 
 # Tests which suppose to be working in older versions only.
@@ -64,6 +64,7 @@ before_install:
 install:
 - make docker-load
 before_script:
+- docker info
 - docker images
 script:
 - set -e

diff --git a/Dockerfile b/Dockerfile
@@ -18,7 +18,7 @@ LABEL org.label-schema.build-date=$BUILD_DATE \
       org.label-schema.schema-version="1.0"
 
 # Setup the default user.
-RUN useradd -d /home/ubuntu -ms /bin/bash -g root -G sudo -p ubuntu ubuntu
+RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u 1000 -p ubuntu ubuntu
 WORKDIR /home/ubuntu
 
 # Build-time variables.
@@ -35,9 +35,14 @@ ENV PROVISION_SUDO 1
 ENV PROVISION_HASH KwFCBBn659lGNLNiIGd5131XnknI
 RUN provision.sh
 
-# Default backtest inputs.
+# Setup results directory.
 ENV BT_DEST /opt/results
-ENV MT_VER=4.0.0.1010
+RUN mkdir -v -m a=rwx $BT_DEST
+RUN chown ubuntu:root $BT_DEST
+VOLUME $BT_DEST
+
+# Default backtest inputs.
+ENV MT_VER 4.0.0.1010
 
 # Run test.
 USER ubuntu

diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-## Brief
+# About
 
 Headless VM backtesting for MT4 platform under Linux.
 
@@ -8,4 +8,280 @@ Headless VM backtesting for MT4 platform under Linux.
 | --------------: |:-----------:|
 | Travis CI build | [![Build Status](https://api.travis-ci.org/EA31337/EA-Tester.svg?branch=master)](https://travis-ci.org/EA31337/EA-Tester) |
 | AppVeyor build  | [![Build status](https://ci.appveyor.com/api/projects/status/r4g7ughqovcv5ph5/branch/master?svg=true)](https://ci.appveyor.com/project/kenorb/ea-tester) |
-| Docker build    | [![Docker build](https://images.microbadger.com/badges/image/ea31337/fx-mt-vm.svg)](https://microbadger.com/images/ea31337/fx-mt-vm "Docker build") |
+| Docker build    | [![Docker build](https://images.microbadger.com/badges/image/ea31337/fx-mt-vm.svg)](https://microbadger.com/images/ea31337/fx-mt-vm "Docker build") |
+
+## Usage
+
+The easiest way is to [install Docker](https://www.docker.com/get-started), then run tester from the terminal:
+
+    docker run ea31337/ea-tester help
+
+to list available commands. Here is the list:
+
+```
+# Run backtest.
+# Usage: run_backtest [args]
+--
+# Clone git repository.
+# Usage: clone_repo [url] [args...]
+--
+# Get the backtest data.
+# Usage: get_bt_data [currency] [year] [DS/MQ/N1-5/W1-5/C1-5/Z1-5/R1-5] [period]
+--
+# Change the working directory.
+# Usage: chdir [dir]
+--
+# Check logs for specific text.
+# Usage: check_logs [filter] [args]
+--
+# Display logs in real-time.
+# Usage: live_logs [invert-match] [interval]
+--
+# Display performance stats in real-time.
+# Usage: live_stats [interval]
+--
+# Delete compiled EAs.
+# Usage: clean_ea
+--
+# Clean files (e.g. previous report and log files).
+# Usage: clean_files
+--
+# Delete backtest data files.
+# Usage: clean_bt
+--
+# Check the version of the given binary file.
+# Usage: filever [file/terminal.exe]
+--
+# Install platform.
+# Usage: install_mt [ver/4/5/4.0.0.1010]
+--
+# Replaces MetaEditor with the specific version.
+# Usage: install_mteditor [ver/5.0.0.1804]
+--
+# Configure virtual display and wine.
+# Usage: set_display
+--
+# Detect and configure proxy.
+# Usage: set_proxy
+--
+# Display recent logs.
+# Usage: show_logs
+--
+# Copy script file given the file path.
+# Usage: script_copy [file]
+--
+# Copy library file (e.g. dll) given the file path.
+# Usage: lib_copy [file]
+--
+# Copy a file given the file path.
+# Usage: file_copy [file]
+--
+# Copy srv files into terminal dir.
+# Usage: srv_copy [file]
+--
+# Download the file.
+# Usage: file_get [url]
+--
+# Compile the given EA name.
+# Usage: compile_ea [pattern] [log_file]
+--
+# Compile and test the given EA.
+# Usage: compile_and_test [pattern] [args...]
+--
+# Copy ini settings from templates.
+# Usage: ini_copy
+--
+# Find the EA file.
+# Usage: ea_find [filename/url/pattern]
+--
+# Find the script file.
+# Usage: script_find [filename/url/pattern]
+--
+# Copy EA file to the platform experts dir.
+# Usage: ea_copy [file]
+--
+# Copy script file to the platform scripts dir.
+# Usage: script_copy [file]
+--
+# Copy library file (e.g. dll) to the platform lib dir.
+# Usage: lib_copy [file]
+--
+# Copy a file to the platform files dir.
+# Usage: file_copy [file]
+--
+# Convert HTML to text format.
+# Usage: convert_html2txt [file_src] [file_dst]
+--
+# Convert HTML to text format (full version).
+# Usage: convert_html2txt_full [file_src] [file_dst]
+--
+# Convert HTML to JSON format
+# Usage: convert_html2json [file_src] [file_dst]
+--
+# Compile given script name.
+# Usage: compile_script
+--
+# Sort optimization test result values by profit factor.
+# Usage: sort_opt_results [file/report.html]
+--
+# Post results to gist.
+# Usage: gist_results [dir] [files/pattern]
+--
+# Enhance a GIF report file.
+# Usage: enhance_gif -c1 [color_name] -c2 [color_name] -t "Some text with \\\n new lines"
+--
+# Run platform and kill it.
+# Usage: quick_run
+--
+# Set input value in the SET file.
+# Usage: input_set [key] [value] [file]
+--
+# Get input value from the SET file.
+# Usage: input_get [key] [file]
+--
+# Set value in the INI file.
+# Usage: ini_set [key] [value] [file]
+--
+# Delete value from the INI file.
+# Usage: ini_del [key] [file]
+--
+# Set value in the EA INI file.
+# Usage: ini_set_ea [key] [value]
+--
+# Set inputs in the EA INI file.
+# Usage: ini_set_inputs [set_file] [ini_file]
+--
+# Get value from the INI/HTM file.
+# Usage: ini_get [key] [file]
+--
+# Get all values from the INI/HTM file.
+# Usage: value_get_all [file]
+--
+# Set spread in ini and FXT files.
+# Usage: set_spread [points/10]
+--
+# Set lot step in FXT files.
+# Usage: set_lotstep [size/0.01]
+--
+# Set digits in symbol raw file.
+# Usage: set_digits [value/5]
+```
+
+### Backtesting
+
+Here is the example command for running backtest on MACD using historical data from 2018:
+
+    docker run ea31337/ea-tester run_backtest -e MACD -y 2018 -v -t
+
+To backtest EA31337 bot, check: [Backtesting using Docker](/~https://github.com/EA31337/EA31337/wiki/Backtesting-using-Docker).
+
+To customize the command, here are the supported parameters and variables:
+
+```
+  -A (command)
+    Action to evaluate (e.g. "file_get URL").
+    Variable (string): CODE
+  -b (option)
+    Source of backtest data to test. Default: DS
+    Variable (string): BT_SRC
+  -B (filename)
+    Specify early booting script to execute.
+    Variable (string): INCLUDE_BOOT
+  -c (currency)
+    Base currency for test. Default: USD
+    Variable (string): BT_CURRENCY
+  -C
+    Clear previous backtest data files.
+  -d (amount)
+    Deposit amount to test. Default: 10000
+    Variable (int): BT_DEPOSIT
+  -D (digits)
+    Specify market digits (e.g. 5 or 4).
+    Variable (uint): BT_DIGITS
+  -e (filename/url/pattern)
+    EA name to test (TestExpert).
+    Variable (string): EXPERT
+  -E (filename/url/pattern)
+    EA name to run (Expert).
+    Variable (string): EXPERT
+  -f (filename)
+    The .set file to run the test.
+    Variable (string): SETORG
+  -F
+    Convert test report file to full detailed text format.
+    Variable (bool): OPT_FORMAT_FULL
+  -g
+    Post results to Gist. It enables: -j and -t
+    Variables (bool): OPT_GIST
+  -G
+    Enhance gif report files.
+    Variable (bool): OPT_GIF_ENHANCE
+  -i (file)
+    Invoke file with custom rules.
+    Variable (string): INCLUDE
+  -I (options)
+    Change tester INI file with custom settings (e.g. Server=MetaQuotes-Demo,Login=123).
+    Variable (string): TEST_OPTS
+  -j
+    Convert test report file into JSON format.
+    Variable (bool): OPT_FORMAT_JSON
+  -l (double)
+    Specify a lot step (e.g. 0.01).
+    Variable (float): BT_LOTSTEP
+  -L (limit)
+    EA common/limit test parameters separated by comma (e.g. genetic=0,maxdrawdown=20.00).
+    Variable (string): EA_OPTS
+  -m (month)
+    Month to test. Default: 1-12.
+    Variable (uint/string): BT_MONTHS
+  -M (version)
+    Specify version of MetaTrader (e.g. 4, 4x, 5, 4.0.0.1010). Default: 4.0.0.1010
+  -o
+    Run test in optimization mode.
+    Variable (bool): OPT_OPTIMIZATION
+  -O (dir)
+    Output directory to save the test results. Default: /opt/results for Docker container.
+    Variable (string): BT_DEST
+  -p (pair)
+    Symbol pair to test (e.g. EURUSD).
+  -P (param)
+    Set EA param in SET file (e.g. VerboseInfo=1,TakeProfit=0).
+    Variable (string): SET_OPTS
+  -r (string)
+    The name of the test report file. Default: tester/Report
+    Variable (string): TEST_REPORT
+  -R
+    Set files to read-only.
+  -s (file/url/pattern)
+    Script to run (Script).
+    Variable (string): SCRIPT
+  -S (spread)
+    Spread to test in points.
+    Variable (uint): BT_SPREAD
+  -t
+    Convert test report file into brief text format.
+    Variable (bool): OPT_FORMAT_BRIEF
+  -T (timeframe)
+    Timeframe (TestPeriod) to test. Default: M30
+    Variable (string): BT_PERIOD
+  -v
+    Verbose mode.
+    Variable (bool): OPT_VERBOSE
+  -V
+    Run test in visual mode (TestVisualEnable).
+    Variable (bool): VISUAL_MODE
+  -x
+    Run the script in trace/debug mode.
+  -X (file)
+    Invoke file on exit after the successful test.
+  -y (year)
+    Year to test. Default: 2017
+    Variable (uint/string): BT_YEARS
+  -?
+    Display help.
+
+Other variables (without arguments assigned):
+- BT_TESTMODE (uint)
+  This controls type of backtest data being downloaded.
+  Values: 0 (default) - Every tick, 1 - Control points, 2 - Open prices only
+```