#!/bin/sh pwd_dir=$(pwd) cd `dirname $0` rm -f run_functional_tests.log show_help() { cat < For use with --dockerize, run tests using partially migrated 'postgres', 'mysql', or 'sqlite' databases. --external_url External URL to use for Galaxy testing (only certain tests). --external_master_key Master API key used to configure external tests. --external_user_key User API used for external tests - not required if external_master_key is specified. --skip_flakey_fails Skip flakey tests on error (sets GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR=1). Environment Variables: In addition to the above command-line options, many environment variables can be used to control the Galaxy functional testing processing. Command-line options above like (--external_url) will set environment variables - in such cases the command line argument takes precedent over environment variables set at the time of running this script. Functional Test Environment Variables GALAXY_TEST_DBURI Database connection string used for functional test database for Galaxy. GALAXY_TEST_INSTALL_DBURI Database connection string used for functional test database for Galaxy's install framework. GALAXY_TEST_INSTALL_DB_MERGED Set to use same database for Galaxy and install framework, this defaults to True for Galaxy tests an False for shed tests. GALAXY_TEST_DB_TEMPLATE If GALAXY_TEST_DBURI is unset, this URL can be retrieved and should be an sqlite database that will be upgraded and tested against. GALAXY_TEST_TMP_DIR Temp directory used for files required by Galaxy server setup for Galaxy functional tests. GALAXY_TEST_SAVE Location to save certain test files (such as tool outputs). GALAXY_TEST_EXTERNAL Target an external Galaxy as part of testing. GALAXY_TEST_JOB_CONFIG_FILE Job config file to use for the test. GALAXY_CONFIG_MASTER_API_KEY Master or admin API key to use as part of testing with GALAXY_TEST_EXTERNAL. GALAXY_TEST_USER_API_KEY User API key to use as part of testing with GALAXY_TEST_EXTERNAL. GALAXY_TEST_VERBOSE_ERRORS Enable more verbose errors during API tests. GALAXY_TEST_UPLOAD_ASYNC Upload tool test inputs asynchronously (may overwhelm sqlite database). GALAXY_TEST_RAW_DIFF Don't slice up tool test diffs to keep output managable - print all output. (default off) GALAXY_TEST_DEFAULT_WAIT Max time allowed for a tool test before Galaxy gives up (default 86400) - tools may define a maxseconds attribute to extend this. GALAXY_TEST_TOOL_DEPENDENCY_DIR tool dependency dir to use for Galaxy during functional tests. GALAXY_TEST_FILE_DIR Test data sources (default to test-data,/~https://github.com/galaxyproject/galaxy-test-data.git) GALAXY_TEST_DIRECTORY $GALAXY_ROOT/test GALAXY_TEST_TOOL_DATA_PATH Set to override tool data path during tool shed tests. GALAXY_TEST_FETCH_DATA Fetch remote test data to GALAXY_TEST_DATA_REPO_CACHE as part of tool tests if it is not available locally (default to True). Requires git to be available on the command-line. GALAXY_TEST_DATA_REPO_CACHE Where to cache remote test data to (default to test-data-cache). GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR Skip tests annotated with @flakey on test errors. HTTP_ACCEPT_LANGUAGE Defaults to 'en' GALAXY_TEST_NO_CLEANUP Do not cleanup main test directory after tests, the deprecated option TOOL_SHED_TEST_NO_CLEANUP does the same thing. GALAXY_TEST_HOST Host to use for Galaxy server setup for testing. GALAXY_TEST_PORT Port to use for Galaxy server setup for testing. GALAXY_TEST_TOOL_PATH Path defaulting to 'tools'. GALAXY_TEST_SHED_TOOL_CONF Shed toolbox conf (defaults to config/shed_tool_conf.xml) used when testing installed to tools with -installed. TOOL_SHED_TEST_HOST Host to use for shed server setup for testing. TOOL_SHED_TEST_PORT Port to use for shed server setup for testing. TOOL_SHED_TEST_FILE_DIR Defaults to test/shed_functional/test_data. TOOL_SHED_TEST_TMP_DIR Defaults to random /tmp directory - place for tool shed test server files to be placed. TOOL_SHED_TEST_OMIT_GALAXY Do not launch a Galaxy server for tool shed testing. Unit Test Environment Variables GALAXY_TEST_INCLUDE_SLOW - Used in unit tests to trigger slower tests that aren't included by default with --unit/-u. EOF } show_list() { python tool_list.py echo "===========================================================================================================================================" echo "'${0##*/} -id bbb' for testing one tool with id 'bbb' ('bbb' is the tool id)" echo "'${0##*/} -sid ccc' for testing one section with sid 'ccc' ('ccc' is the string after 'section::')" } exists() { type "$1" >/dev/null 2>/dev/null } DOCKER_DEFAULT_IMAGE='galaxy/testing-base:18.05.3' test_script="./scripts/functional_tests.py" report_file="run_functional_tests.html" xunit_report_file="" structured_data_report_file="" with_framework_test_tools_arg="" skip_client_build="--skip-client-build" if [ "$1" = "--dockerize" ]; then shift DOCKER_EXTRA_ARGS=${DOCKER_ARGS:-""} DOCKER_RUN_EXTRA_ARGS=${DOCKER_RUN_EXTRA_ARGS:-""} DOCKER_IMAGE=${DOCKER_IMAGE:-${DOCKER_DEFAULT_IMAGE}} if [ "$1" = "--db" ]; then db_type=$2 shift 2 else db_type="sqlite" fi if [ "$1" = "--external_tmp" ]; then # If /tmp is a tmpfs there may be better performance by reusing # the parent's temp file system. Also, it seems to decrease the # frequency or errors such as the following: # /bin/sh: 1: /tmp/tmpiWU3kJ/tmp_8zLxx/job_working_directory_mwwDmg/000/274/galaxy_274.sh: Text file busy tmp=$(mktemp -d) chmod 1777 $tmp DOCKER_RUN_EXTRA_ARGS="-v ${tmp}:/tmp ${DOCKER_RUN_EXTRA_ARGS}" shift fi MY_UID=$(id -u) # Skip client build process in the Docker container for all tests, the Jenkins task builds the client # locally before testing - you will need to do this also if using this script for Selenium testing. DOCKER_RUN_EXTRA_ARGS="-e GALAXY_TEST_UID=${MY_UID} -e GALAXY_SKIP_CLIENT_BUILD=1 ${DOCKER_RUN_EXTRA_ARGS}" echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS -e "BUILD_NUMBER=$BUILD_NUMBER" -e "GALAXY_TEST_DATABASE_TYPE=$db_type" --rm -v `pwd`:/galaxy $DOCKER_IMAGE "$@" exit $? fi # If in Jenkins environment, create xunit-${BUILD_NUMBER}.xml by default. if [ -n "$BUILD_NUMBER" ]; then xunit_report_file="xunit-${BUILD_NUMBER}.xml" fi run_default_functional_tests="1" # Loop through and consume the main arguments. # Some loops will consume more than one argument (there are extra "shift"s in some cases). while : do case "$1" in -h|--help|-\?) show_help exit 0 ;; -l|-list|--list) show_list exit 0 ;; -id|--id) if [ $# -gt 1 ]; then test_id=$2; shift 2 else echo "--id requires an argument" 1>&2 exit 1 fi ;; -s|-sid|--sid) if [ $# -gt 1 ]; then section_id=$2 shift 2 else echo "--sid requires an argument" 1>&2 exit 1 fi ;; -a|-api|--api) with_framework_test_tools_arg="-with_framework_test_tools" test_script="./scripts/functional_tests.py" report_file="./run_api_tests.html" if [ $# -gt 1 ]; then api_script=$2 shift 2 else api_script="./test/api" shift 1 fi ;; -selenium|--selenium) with_framework_test_tools_arg="-with_framework_test_tools" test_script="./scripts/functional_tests.py" report_file="./run_selenium_tests.html" skip_client_build="" selenium_test=1; if [ $# -gt 1 ]; then selenium_script=$2 shift 2 else selenium_script="./test/selenium_tests" shift 1 fi ;; -t|-toolshed|--toolshed) test_script="./test/shed_functional/functional_tests.py" report_file="run_toolshed_tests.html" if [ $# -gt 1 ]; then toolshed_script=$2 shift 2 else toolshed_script="./test/shed_functional/functional" shift 1 fi ;; -clean_pyc|--clean_pyc) find lib -iname '*pyc' -exec rm -rf {} \; find test -iname '*pyc' -exec rm -rf {} \; shift ;; -skip_flakey_fails|--skip_flakey_fails) GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR=1 export GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR shift ;; -with_framework_test_tools|--with_framework_test_tools) with_framework_test_tools_arg="-with_framework_test_tools" shift ;; --external_url) GALAXY_TEST_EXTERNAL=$2 shift 2 ;; --external_master_key) GALAXY_CONFIG_MASTER_KEY=$2 shift 2 ;; --external_user_key) GALAXY_TEST_USER_API_KEY=$2 shift 2 ;; -f|-framework|--framework) report_file="run_framework_tests.html" framework_test=1; shift 1 ;; -d|-data_managers|--data_managers) data_managers_test=1; shift 1 ;; -m|-migrated|--migrated) migrated_test=1; shift ;; -i|-installed|--installed) installed_test=1; shift ;; -r|--report_file) if [ $# -gt 1 ]; then report_file=$2 shift 2 else echo "--report_file requires an argument" 1>&2 exit 1 fi ;; --xunit_report_file) if [ $# -gt 1 ]; then xunit_report_file=$2 shift 2 else echo "--xunit_report_file requires an argument" 1>&2 exit 1 fi ;; --structured_data_report_file) if [ $# -gt 1 ]; then structured_data_report_file=$2 shift 2 else echo "--structured_data_report_file requires an argument" 1>&2 exit 1 fi ;; --verbose_errors) GALAXY_TEST_VERBOSE_ERRORS=True export GALAXY_TEST_VERBOSE_ERRORS shift ;; -c|--coverage) # Must have coverage installed (try `which coverage`) - only valid with --unit # for now. Would be great to get this to work with functional tests though. coverage_arg="--with-coverage" NOSE_WITH_COVERAGE=true shift ;; --debug) #TODO ipdb would be nicer. NOSE_PDB=True export NOSE_PDB shift ;; -u|-unit|--unit) report_file="run_unit_tests.html" test_script="./scripts/nosetests.py" if [ $# -gt 1 ]; then unit_extra=$2 shift 2 else unit_extra='--exclude=functional --exclude="^get" --exclude=controllers --exclude=runners --exclude dictobj --exclude=jstree lib test/unit' shift 1 fi ;; -i|-integration|--integration) report_file="run_integration_tests.html" test_script="./scripts/nosetests.py" if [ $# -gt 1 ]; then integration_extra=$2 shift 2 else integration_extra='test/integration' shift 1 fi ;; --no_cleanup) GALAXY_TEST_NO_CLEANUP=1 export GALAXY_TEST_NO_CLEANUP TOOL_SHED_TEST_NO_CLEANUP=1 export TOOL_SHED_TEST_NO_CLEANUP GALAXY_INSTALL_TEST_NO_CLEANUP=1 export GALAXY_INSTALL_TEST_NO_CLEANUP echo "Skipping Python test clean up." shift ;; -watch|--watch) # Have grunt watch test or directory for changes, only # valid for javascript testing. watch=1 shift ;; --skip-venv) skip_venv='--skip-venv' shift ;; --no-create-venv) no_create_venv='--no-create-venv' shift ;; --no-replace-pip) no_replace_pip='--no-replace-pip' shift ;; --replace-pip) replace_pip='--replace-pip' shift ;; --skip-common-startup) # Don't run ./scripts/common_startup.sh (presumably it has already # been done, or you know what you're doing). skip_common_startup=1 shift ;; --) # Do not default to running the functional tests in this case, caller # is opting to run specific tests so don't interfere with that by default. unset run_default_functional_tests; shift break ;; -*) echo "invalid option: $1" 1>&2; show_help exit 1 ;; *) if [ -n "$1" ]; then test_target="$1" shift fi # Maybe we shouldn't break here but for now to pass more than one argument to the # underlying test driver (scripts/nosetests.py) use -- instead. break ;; esac done if [ -z "$skip_common_startup" ]; then if [ -n "$GALAXY_TEST_DBURI" ]; then GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=$GALAXY_TEST_DBURI export GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION fi ./scripts/common_startup.sh $skip_venv $no_create_venv $no_replace_pip $replace_pip $skip_client_build --dev-wheels || exit 1 fi . ./scripts/common_startup_functions.sh setup_python if [ -n "$migrated_test" ] ; then [ -n "$test_id" ] && class=":TestForTool_$test_id" || class="" extra_args="functional.test_toolbox$class -migrated" elif [ -n "$installed_test" ] ; then [ -n "$test_id" ] && class=":TestForTool_$test_id" || class="" extra_args="functional.test_toolbox$class -installed" elif [ -n "$framework_test" ] ; then [ -n "$test_id" ] && class=":TestForTool_$test_id" || class="" extra_args="functional.test_toolbox$class -framework" elif [ -n "$selenium_test" ] ; then extra_args="$selenium_script -selenium" elif [ -n "$data_managers_test" ] ; then [ -n "$test_id" ] && class=":TestForDataManagerTool_$test_id" || class="" extra_args="functional.test_data_managers$class -data_managers" elif [ -n "$toolshed_script" ]; then extra_args="$toolshed_script" elif [ -n "$api_script" ]; then extra_args="$api_script" elif [ -n "$section_id" ]; then extra_args=`python tool_list.py $section_id` elif [ -n "$test_id" ]; then class=":TestForTool_$test_id" extra_args="functional.test_toolbox$class" elif [ -n "$unit_extra" ]; then extra_args="--with-doctest $unit_extra" elif [ -n "$integration_extra" ]; then extra_args="$integration_extra" elif [ -n "$test_target" ] ; then extra_args="$test_target" elif [ -n "$run_default_functional_tests" ] ; then extra_args='--exclude="^get" functional' else extra_args="" fi if [ -n "$xunit_report_file" ]; then xunit_args="--with-xunit --xunit-file $xunit_report_file" else xunit_args="" fi if [ -n "$structured_data_report_file" ]; then structured_data_args="--with-structureddata --structured-data-file $structured_data_report_file" else structured_data_args="" fi if [ -n "$with_framework_test_tools_arg" ]; then GALAXY_TEST_TOOL_CONF="config/tool_conf.xml.sample,test/functional/tools/samples_tool_conf.xml" export GALAXY_TEST_TOOL_CONF fi python $test_script $coverage_arg -v --with-nosehtml --html-report-file $report_file $xunit_args $structured_data_args $extra_args "$@" exit_status=$? echo "Testing complete. HTML report is in \"$report_file\"." 1>&2 exit ${exit_status}