Merge pull request #3 from matrix-org/bwindels/ci_script

Make tests run on CI environment

README.md

@@ -34,17 +34,15 @@ puppeteer.launch({headless: false});
 ## How to run
 
 ### Setup
- - install synapse with `sh synapse/install.sh`, this fetches the master branch at the moment. If anything fails here, please refer to the synapse README to see if you're missing one of the prerequisites.
- - install riot with `sh riot/install.sh`, this fetches the master branch at the moment.
- - install dependencies with `npm install` (will download copy of chrome)
- - have riot-web running on `localhost:8080`
- - have a local synapse running at `localhost:8008`
+
+Run `./install.sh`. This will:
+ - install synapse, fetches the master branch at the moment. If anything fails here, please refer to the synapse README to see if you're missing one of the prerequisites.
+ - install riot, this fetches the master branch at the moment.
+ - install dependencies (will download copy of chrome)
 
 ### Run tests
 
-Run tests with `sh run.sh`.
-
-You should see the terminal split with on top the server output (both riot static server, and synapse), and on the bottom the tests running.
+Run tests with `./run.sh`.
 
 Developer Guide
 ===============

helpers.js

@@ -54,14 +54,15 @@ function logConsole(page) {
 
 function logXHRRequests(page) {
   let buffer = "";
-  page.on('request', req => {
+  page.on('requestfinished', async (req) => {
     const type = req.resourceType();
-    if (type === 'xhr' || type === 'fetch') {
-      buffer += `${req.method()} ${req.url()} \n`;
-      if (req.method() === "POST") {
-        buffer += "  Post data: " + req.postData();
-      }
-    }
+    const response = await req.response();
+    //if (type === 'xhr' || type === 'fetch') {
+      buffer += `${type} ${response.status()} ${req.method()} ${req.url()} \n`;
+      // if (req.method() === "POST") {
+      //   buffer += "  Post data: " + req.postData();
+      // }
+    //}
   });
   return {
     logs() {

install.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# run with PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true sh install.sh if chrome is already installed
+set -e
+./synapse/install.sh
+./riot/install.sh
+npm install

package.json

@@ -9,6 +9,6 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "puppeteer": "^1.5.0"
+    "puppeteer": "^1.6.0"
   }
 }

riot/install.sh

@@ -1,13 +1,17 @@
+#!/bin/bash
 RIOT_BRANCH=master
 
-BASE_DIR=$(realpath $(dirname $0))
-pushd $BASE_DIR
+BASE_DIR=$(readlink -f $(dirname $0))
+if [ -d $BASE_DIR/riot-web ]; then
+	echo "riot is already installed"
+	exit
+fi
+
+cd $BASE_DIR
 curl -L /~https://github.com/vector-im/riot-web/archive/${RIOT_BRANCH}.zip --output riot.zip
-unzip riot.zip
+unzip -q riot.zip
 rm riot.zip
 mv riot-web-${RIOT_BRANCH} riot-web
-cp config-template/config.json riot-web/
-pushd riot-web
+cd riot-web
 npm install
 npm run build
-popd

riot/start.sh

@@ -1,8 +1,51 @@
-BASE_DIR=$(realpath $(dirname $0))
-pushd $BASE_DIR
-pushd riot-web/webapp/
-python -m SimpleHTTPServer 8080 &
-PID=$!
-popd
-echo $PID > riot.pid
-popd
+#!/bin/bash
+PORT=5000
+BASE_DIR=$(readlink -f $(dirname $0))
+PIDFILE=$BASE_DIR/riot.pid
+CONFIG_BACKUP=config.e2etests_backup.json
+
+if [ -f $PIDFILE ]; then
+	exit
+fi
+
+cd $BASE_DIR/
+echo -n "starting riot on http://localhost:$PORT ... "
+pushd riot-web/webapp/ > /dev/null
+
+# backup config file before we copy template
+if [ -f config.json ]; then
+	mv config.json $CONFIG_BACKUP
+fi
+cp $BASE_DIR/config-template/config.json .
+
+LOGFILE=$(mktemp)
+# run web server in the background, showing output on error
+(
+	python -m SimpleHTTPServer $PORT > $LOGFILE 2>&1 &
+	PID=$!
+	echo $PID > $PIDFILE
+	# wait so subshell does not exit
+	# otherwise sleep below would not work
+	wait $PID; RESULT=$?
+
+	# NOT expected SIGTERM (128 + 15)
+	# from stop.sh?
+	if [ $RESULT -ne 143 ]; then
+		echo "failed"
+		cat $LOGFILE
+		rm $PIDFILE 2> /dev/null
+	fi
+	rm $LOGFILE
+	exit $RESULT
+)&
+# to be able to return the exit code for immediate errors (like address already in use)
+# we wait for a short amount of time in the background and exit when the first
+# child process exists
+sleep 0.5 &
+# wait the first child process to exit (python or sleep)
+wait -n; RESULT=$?
+# return exit code of first child to exit
+if [ $RESULT -eq 0 ]; then
+	echo "running"
+fi
+exit $RESULT

riot/stop.sh

@@ -1,6 +1,20 @@
-BASE_DIR=$(realpath $(dirname $0))
-pushd $BASE_DIR > /dev/null
+#!/bin/bash
+BASE_DIR=$(readlink -f $(dirname $0))
 PIDFILE=riot.pid
-kill $(cat $PIDFILE)
-rm $PIDFILE
-popd > /dev/null
+CONFIG_BACKUP=config.e2etests_backup.json
+
+cd $BASE_DIR
+
+if [ -f $PIDFILE ]; then
+	echo "stopping riot server ..."
+	PID=$(cat $PIDFILE)
+	rm $PIDFILE
+	kill $PID
+
+	# revert config file
+	cd riot-web/webapp
+	rm config.json
+	if [ -f $CONFIG_BACKUP ]; then
+		mv $CONFIG_BACKUP config.json
+	fi
+fi

run.sh

@@ -1,4 +1,19 @@
-tmux \
-	new-session "sh riot/stop.sh; sh synapse/stop.sh; sh synapse/start.sh; sh riot/start.sh; read"\; \
-	split-window "sleep 5; node start.js; sh riot/stop.sh; sh synapse/stop.sh; read"\; \
-	select-layout even-vertical
+#!/bin/bash
+
+stop_servers() {
+	./riot/stop.sh
+	./synapse/stop.sh
+}
+
+handle_error() {
+	EXIT_CODE=$?
+	stop_servers
+	exit $EXIT_CODE
+}
+
+trap 'handle_error' ERR
+
+./synapse/start.sh
+./riot/start.sh
+node start.js
+stop_servers

start.js

@@ -25,12 +25,27 @@ const acceptServerNoticesInviteAndConsent = require('./tests/server-notices-cons
 
 const homeserver = 'http://localhost:8008';
 
-global.riotserver = 'http://localhost:8080';
+global.riotserver = 'http://localhost:5000';
 global.browser = null;
 
+let consoleLogs = null;
+let xhrLogs = null;
+let globalPage = null;
+
 async function runTests() {
-  global.browser = await puppeteer.launch();
+  console.log("running tests ...");
+  const options = {};
+  if (process.env.CHROME_PATH) {
+    const path = process.env.CHROME_PATH;
+    console.log(`(using external chrome/chromium at ${path}, make sure it's compatible with puppeteer)`);
+    options.executablePath = path;
+  }
+  global.browser = await puppeteer.launch(options);
   const page = await helpers.newPage();
+  globalPage = page;
+
+  consoleLogs = helpers.logConsole(page);
+  xhrLogs = helpers.logXHRRequests(page);
 
   const username = 'user-' + helpers.randomInt(10000);
   const password = 'testtest';
@@ -39,7 +54,7 @@ async function runTests() {
   process.stdout.write('done\n');
 
   const noticesName = "Server Notices";
-  process.stdout.write(`* accepting "${noticesName}" and accepting terms & conditions ...`);
+  process.stdout.write(`* accepting "${noticesName}" and accepting terms & conditions ... `);
   await acceptServerNoticesInviteAndConsent(page, noticesName);
   process.stdout.write('done\n');
 
@@ -55,8 +70,21 @@ function onSuccess() {
   console.log('all tests finished successfully');
 }
 
-function onFailure(err) {
+async function onFailure(err) {
+
+  let documentHtml = "no page";
+  if (globalPage) {
+    documentHtml = await globalPage.content();
+  }
+
   console.log('failure: ', err);
+  console.log('console.log output:');
+  console.log(consoleLogs.logs());
+  console.log('XHR requests:');
+  console.log(xhrLogs.logs());
+  console.log('document html:');
+  console.log(documentHtml);
+
   process.exit(-1);
 }
 

synapse/install.sh

@@ -1,17 +1,25 @@
+#!/bin/bash
 # config
 SYNAPSE_BRANCH=master
 INSTALLATION_NAME=consent
 SERVER_DIR=installations/$INSTALLATION_NAME
 CONFIG_TEMPLATE=consent
 PORT=8008
 # set current directory to script directory
-BASE_DIR=$(realpath $(dirname $0))
-pushd $BASE_DIR
+BASE_DIR=$(readlink -f $(dirname $0))
+
+if [ -d $BASE_DIR/$SERVER_DIR ]; then
+	echo "synapse is already installed"
+	exit
+fi
+
+cd $BASE_DIR
+
 mkdir -p installations/
 curl https://codeload.github.com/matrix-org/synapse/zip/$SYNAPSE_BRANCH --output synapse.zip
-unzip synapse.zip
+unzip -q synapse.zip
 mv synapse-$SYNAPSE_BRANCH $SERVER_DIR
-pushd $SERVER_DIR
+cd $SERVER_DIR
 virtualenv -p python2.7 env
 source env/bin/activate
 pip install --upgrade pip
@@ -29,5 +37,3 @@ sed -i "s#{{SYNAPSE_PORT}}#${PORT}/#g" homeserver.yaml
 sed -i "s#{{FORM_SECRET}}#$(uuidgen)#g" homeserver.yaml
 sed -i "s#{{REGISTRATION_SHARED_SECRET}}#$(uuidgen)#g" homeserver.yaml
 sed -i "s#{{MACAROON_SECRET_KEY}}#$(uuidgen)#g" homeserver.yaml
-popd	#back to synapse root dir
-popd	#back to wherever we were

synapse/start.sh

@@ -1,7 +1,12 @@
-BASE_DIR=$(realpath $(dirname $0))
-pushd $BASE_DIR
-pushd installations/consent
+#!/bin/bash
+BASE_DIR=$(readlink -f $(dirname $0))
+cd $BASE_DIR
+cd installations/consent
 source env/bin/activate
-./synctl start
-popd
-popd
+LOGFILE=$(mktemp)
+./synctl start 2> $LOGFILE
+EXIT_CODE=$?
+if [ $EXIT_CODE -ne 0 ]; then
+	cat $LOGFILE
+fi
+exit $EXIT_CODE

synapse/stop.sh

@@ -1,7 +1,6 @@
-BASE_DIR=$(realpath $(dirname $0))
-pushd $BASE_DIR > /dev/null
-pushd installations/consent > /dev/null
+#!/bin/bash
+BASE_DIR=$(readlink -f $(dirname $0))
+cd $BASE_DIR
+cd installations/consent
 source env/bin/activate
 ./synctl stop
-popd > /dev/null
-popd > /dev/null

tests/signup.js

@@ -19,8 +19,6 @@ const acceptTerms = require('./consent');
 const assert = require('assert');
 
 module.exports = async function signup(page, username, password, homeserver) {
-  const consoleLogs = helpers.logConsole(page);
-  const xhrLogs = helpers.logXHRRequests(page);
   await page.goto(helpers.riotUrl('/#/register'));
   //click 'Custom server' radio button
   if (homeserver) {
@@ -55,7 +53,7 @@ module.exports = async function signup(page, username, password, homeserver) {
   const error_text = await helpers.tryGetInnertext(page, '.mx_Login_error');
   assert.strictEqual(!!error_text, false);
   //submit form
-  await page.screenshot({path: "beforesubmit.png", fullPage: true});
+  //await page.screenshot({path: "beforesubmit.png", fullPage: true});
   await registerButton.click();
 
   //confirm dialog saying you cant log back in without e-mail