Merge end-to-end tests

.buildkite/pipeline.yaml

@@ -15,9 +15,9 @@ steps:
 
   - label: ":chains: End-to-End Tests"
     agents:
-      # We use a medium sized instance instead of the normal small ones because
+      # We use a xlarge sized instance instead of the normal small ones because
       # e2e tests otherwise take +-8min
-      queue: "medium"
+      queue: "xlarge"
     command:
       # TODO: Remove hacky chmod for BuildKite
       - "echo '--- Setup'"
@@ -96,7 +96,7 @@ steps:
           image: "node:10"
 
   - wait
- 
+
   - label: "🐴 Trigger riot-web"
     trigger: "riot-web"
     branches: "develop"

README.md

@@ -168,3 +168,8 @@ Ensure you've followed the above development instructions and then:
 ```bash
 yarn test
 ```
+
+## End-to-End tests
+
+Make sure you've got your riot development server running (by doing `yarn start` in riot-web), and then in this project, run `yarn run e2etests`.
+See `test/end-to-end-tests/README.md` for more information.

package.json

@@ -54,7 +54,8 @@
     "clean": "rimraf lib",
     "prepare": "yarn clean && yarn build && git rev-parse HEAD > git-revision.txt",
     "test": "karma start --single-run=true --browsers VectorChromeHeadless",
-    "test-multi": "karma start"
+    "test-multi": "karma start",
+    "e2etests": "./test/end-to-end-tests/run.sh --riot-url http://localhost:8080"
   },
   "dependencies": {
     "babel-plugin-syntax-dynamic-import": "^6.18.0",

scripts/ci/end-to-end-tests.sh

@@ -28,14 +28,14 @@ REACT_SDK_DIR=`pwd`
 echo "--- Building Riot"
 scripts/ci/build.sh
 # run end to end tests
-echo "--- Fetching end-to-end tests from master"
-scripts/fetchdep.sh matrix-org matrix-react-end-to-end-tests master
-pushd matrix-react-end-to-end-tests
+pushd test/end-to-end-tests
 ln -s $REACT_SDK_DIR/$RIOT_WEB_DIR riot/riot-web
 # PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true ./install.sh
 # CHROME_PATH=$(which google-chrome-stable) ./run.sh
 echo "--- Install synapse & other dependencies"
 ./install.sh
+# install (only) static webserver to server symlinked local copy of riot
+./riot/install.sh --without-riot
 mkdir logs
 echo "+++ Running end-to-end tests"
 TESTS_STARTED=1

test/end-to-end-tests/.gitignore

@@ -0,0 +1,3 @@
+node_modules
+*.png
+riot/env

test/end-to-end-tests/README.md

@@ -0,0 +1,43 @@
+# Matrix React Web App End-to-End tests
+
+This repository contains tests for the matrix-react-sdk web app. The tests fire up a headless chrome and simulate user interaction (end-to-end). Note that end-to-end has little to do with the end-to-end encryption matrix supports, just that we test the full stack, going from user interaction to expected DOM in the browser.
+
+## Setup
+
+Run `./install.sh`. This will:
+ - install synapse, fetches the master branch at the moment. If anything fails here, please refer to the synapse README to see if you're missing one of the prerequisites.
+ - install riot, this fetches the master branch at the moment.
+ - install dependencies (will download copy of chrome)
+
+## Running the tests
+
+Run tests with `./run.sh`.
+
+### Debug tests locally.
+
+`./run.sh` will run the tests against the riot copy present in `riot/riot-web` served by a static python http server. You can symlink your `riot-web` develop copy here but that doesn't work well with webpack recompiling. You can run the test runner directly and specify parameters to get more insight into a failure or run the tests against your local webpack server.
+
+```
+./synapse/stop.sh && \
+./synapse/start.sh && \
+node start.js <parameters>
+```
+It's important to always stop and start synapse before each run of the tests to clear the in-memory sqlite database it uses, as the tests assume a blank slate.
+
+start.js accepts these parameters (and more, see `node start.js --help`) that can help running the tests locally:
+
+ - `--riot-url <url>` don't use the riot copy and static server provided by the tests, but use a running server like the webpack watch server to run the tests against. Make sure to have the following local config:
+   - `welcomeUserId` disabled as the tests assume there is no riot-bot currently.
+ - `--slow-mo` type at a human speed, useful with `--windowed`.
+ - `--throttle-cpu <factor>` throttle cpu in the browser by the given factor. Useful to reproduce failures because of insufficient timeouts happening on the slower CI server.
+ - `--windowed` run the tests in an actual browser window Try to limit interacting with the windows while the tests are running. Hovering over the window tends to fail the tests, dragging the title bar should be fine though.
+ - `--dev-tools` open the devtools in the browser window, only applies if `--windowed` is set as well.
+
+Developer Guide
+===============
+
+Please follow the standard Matrix contributor's guide:
+/~https://github.com/matrix-org/synapse/tree/master/CONTRIBUTING.rst
+
+Please follow the Matrix JS/React code style as per:
+/~https://github.com/matrix-org/matrix-react-sdk/blob/master/code_style.md

test/end-to-end-tests/TODO.md

@@ -0,0 +1,8 @@
+
+ - join a peekable room by directory
+ - join a peekable room by invite
+ - join a non-peekable room by directory
+ - join a non-peekable room by invite
+ - leave a room and check we move to the "next" room
+ - get kicked from a room and check that we show the correct message
+ - get banned "

test/end-to-end-tests/has_custom_riot.js

@@ -0,0 +1,24 @@
+/*
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// used from run.sh as getopts doesn't support long parameters
+const idx = process.argv.indexOf("--riot-url");
+let hasRiotUrl = false;
+if (idx !== -1) {
+    const value = process.argv[idx + 1];
+    hasRiotUrl = !!value;
+}
+process.stdout.write(hasRiotUrl ? "1" : "0" );

test/end-to-end-tests/install.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# run with PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true sh install.sh if chrome is already installed
+set -e
+./synapse/install.sh
+# both CI and local testing don't need a riot fetched from master,
+# so not installing this by default anymore
+# ./riot/install.sh
+yarn install

test/end-to-end-tests/package.json

@@ -0,0 +1,19 @@
+{
+    "name": "e2e-tests",
+    "version": "1.0.0",
+    "description": "",
+    "main": "index.js",
+    "scripts": {
+        "test": "echo \"Error: no test specified\" && exit 1"
+    },
+    "author": "",
+    "license": "ISC",
+    "dependencies": {
+        "cheerio": "^1.0.0-rc.2",
+        "commander": "^2.19.0",
+        "puppeteer": "^1.14.0",
+        "request": "^2.88.0",
+        "request-promise-native": "^1.0.7",
+        "uuid": "^3.3.2"
+    }
+}

test/end-to-end-tests/riot/.gitignore

@@ -0,0 +1,2 @@
+riot-web
+riot.pid

test/end-to-end-tests/riot/config-template/config.json

@@ -0,0 +1,33 @@
+{
+    "default_hs_url": "http://localhost:5005",
+    "default_is_url": "https://vector.im",
+    "disable_custom_urls": false,
+    "disable_guests": false,
+    "disable_login_language_selector": false,
+    "disable_3pid_login": false,
+    "brand": "Riot",
+    "integrations_ui_url": "https://scalar.vector.im/",
+    "integrations_rest_url": "https://scalar.vector.im/api",
+    "bug_report_endpoint_url": "https://riot.im/bugreports/submit",
+    "features": {
+        "feature_groups": "labs",
+        "feature_pinning": "labs"
+    },
+    "default_federate": true,
+    "welcomePageUrl": "home.html",
+    "default_theme": "light",
+    "roomDirectory": {
+        "servers": [
+            "localhost:5005"
+        ]
+    },
+    "piwik": {
+        "url": "https://piwik.riot.im/",
+        "whitelistedHSUrls": ["http://localhost:5005"],
+        "whitelistedISUrls": ["https://vector.im", "https://matrix.org"],
+        "siteId": 1
+    },
+    "enable_presence_by_hs_url": {
+        "https://matrix.org": false
+    }
+}

test/end-to-end-tests/riot/install.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -e
+RIOT_BRANCH=develop
+
+with_riot=1
+
+for i in $@; do
+    if [ "$i" == "--without-riot" ] ; then
+        with_riot=0
+    fi
+done
+
+BASE_DIR=$(cd $(dirname $0) && pwd)
+cd $BASE_DIR
+# Install ComplexHttpServer (a drop in replacement for Python's SimpleHttpServer
+# but with support for multiple threads) into a virtualenv.
+(
+    virtualenv -p python3 env
+    source env/bin/activate
+
+    # Having been bitten by pip SSL fail too many times, I don't trust the existing pip
+    # to be able to --upgrade itself, so grab a new one fresh from source.
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+    python get-pip.py
+    rm get-pip.py
+
+    pip install ComplexHttpServer
+
+    deactivate
+)
+
+if [ -d $BASE_DIR/riot-web ]; then
+    echo "riot is already installed"
+    exit
+fi
+
+if [ $with_riot -eq 1 ]; then
+    curl -L /~https://github.com/vector-im/riot-web/archive/${RIOT_BRANCH}.zip --output riot.zip
+    unzip -q riot.zip
+    rm riot.zip
+    mv riot-web-${RIOT_BRANCH} riot-web
+    cd riot-web
+    yarn install
+    yarn run build
+fi

test/end-to-end-tests/riot/start.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -e
+
+PORT=5000
+BASE_DIR=$(cd $(dirname $0) && pwd)
+PIDFILE=$BASE_DIR/riot.pid
+CONFIG_BACKUP=config.e2etests_backup.json
+
+if [ -f $PIDFILE ]; then
+    exit
+fi
+
+cd $BASE_DIR/
+echo -n "starting riot on http://localhost:$PORT ... "
+pushd riot-web/webapp/ > /dev/null
+
+# backup config file before we copy template
+if [ -f config.json ]; then
+    mv config.json $CONFIG_BACKUP
+fi
+cp $BASE_DIR/config-template/config.json .
+
+LOGFILE=$(mktemp)
+# run web server in the background, showing output on error
+(
+    source $BASE_DIR/env/bin/activate
+    python -m ComplexHTTPServer $PORT > $LOGFILE 2>&1 &
+    PID=$!
+    echo $PID > $PIDFILE
+    # wait so subshell does not exit
+    # otherwise sleep below would not work
+    wait $PID; RESULT=$?
+
+    # NOT expected SIGTERM (128 + 15)
+    # from stop.sh?
+    if [ $RESULT -ne 143 ]; then
+        echo "failed"
+        cat $LOGFILE
+        rm $PIDFILE 2> /dev/null
+    fi
+    rm $LOGFILE
+    exit $RESULT
+)&
+# to be able to return the exit code for immediate errors (like address already in use)
+# we wait for a short amount of time in the background and exit when the first
+# child process exits
+sleep 0.5 &
+# wait the first child process to exit (python or sleep)
+wait -n; RESULT=$?
+# return exit code of first child to exit
+if [ $RESULT -eq 0 ]; then
+    echo "running"
+fi
+exit $RESULT

test/end-to-end-tests/riot/stop.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+
+BASE_DIR=$(cd $(dirname $0) && pwd)
+PIDFILE=riot.pid
+CONFIG_BACKUP=config.e2etests_backup.json
+
+cd $BASE_DIR
+
+if [ -f $PIDFILE ]; then
+    echo "stopping riot server ..."
+    PID=$(cat $PIDFILE)
+    rm $PIDFILE
+    kill $PID
+
+    # revert config file
+    cd riot-web/webapp
+    rm config.json
+    if [ -f $CONFIG_BACKUP ]; then
+        mv $CONFIG_BACKUP config.json
+    fi
+fi

test/end-to-end-tests/run.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -e
+
+BASE_DIR=$(cd $(dirname $0) && pwd)
+pushd $BASE_DIR
+
+if [ ! -d "synapse/installations" ] || [ ! -d "node_modules" ]; then
+    echo "please, first run $BASE_DIR/install.sh"
+    exit 1
+fi
+
+has_custom_riot=$(node has_custom_riot.js $@)
+
+if [ ! -d "riot/riot-web" ] && [ $has_custom_riot -ne "1" ]; then
+    echo "please provide an instance of riot to test against by passing --riot-url <url> or running $BASE_DIR/riot/install.sh"
+    exit 1
+fi
+
+stop_servers() {
+    if [ $has_custom_riot -ne "1" ]; then
+	   ./riot/stop.sh
+	fi
+    ./synapse/stop.sh
+}
+
+handle_error() {
+	EXIT_CODE=$?
+	stop_servers
+	exit $EXIT_CODE
+}
+
+trap 'handle_error' ERR
+
+./synapse/start.sh
+if [ $has_custom_riot -ne "1" ]; then
+    ./riot/start.sh
+fi
+node start.js $@
+stop_servers