Add CI/CD with Travis-CI

+ Test on master, develop, release-* branches + Test when a commit is tagged as v* + Test with tox on linux (py37), on windows (py36, py37, py38) + Deploy to GitHub Release only the tagged commits + Compile executable for Ubuntu18.04, Windows10
cityjson · Jan 10, 2020 · 66aee5f · 66aee5f
1 parent 61cdae5
commit 66aee5f
Show file tree

Hide file tree

Showing 10 changed files with 124 additions and 33 deletions.
diff --git a/.gitignore b/.gitignore
@@ -51,9 +51,8 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 # Test datasets
-tests/data/balazs_config.yml
-tests/data/test.json
-tests/data/test_all.json
+tests/data/db3dnl_config.yml
+tests/data/*.json
 
 # Translations
 *.mo

diff --git a/.travis.yml b/.travis.yml
@@ -1,15 +1,98 @@
-# Config file for automatic testing at travis-ci.org
-
 language: python
 python:
-  - 3.8
   - 3.7
-  - 3.6
+cache:
+  - pip
+  - directories:
+      - $HOME/.pyenv
+# Would be great to cache Postgres here but dir names with spaces are split into two and escaping doesn't work
+#      - "/c/Program\ Files/PostgreSQL"
+
+env:
+  - EXE_NAME=cjdb DISTPATH=$TRAVIS_BUILD_DIR/dist
+os:
+  - linux
+
+branches:
+  only:
+    - master
+    - develop
+    # Run on release-* branches. Case insensitive.
+    - /^(?i:release)-.*$/
+    # A build is triggered with a tag push, in which case the branch name is
+    # the tag name. Thus in order for on.tags:true to work, the tag-branch
+    # must be safelisted
+    - /^v.*$/
+
+stages:
+  - name: test
+  - name: deploy
+    # The executable is built only with python3.7 on the master branch
+    if: branch = master AND env(TRAVIS_PYTHON_VERSION) = 3.7
+
+jobs:
+  include:
+    - name: Python 3.7.5 on Bionic Ubuntu
+      dist: bionic
+      before_install:
+        - export EXE_PKG="$EXE_NAME"_"$TRAVIS_CPU_ARCH"_ubuntu1804
+    - name: Python 3.7.5 with pyenv on Windows
+      os: windows
+      language: shell
+      before_install:
+        - export PATH="$HOME/.pyenv/pyenv-win/bin:$HOME/.pyenv/pyenv-win/shims:$HOME/.pyenv/pyenv-win/versions/3.7.5:$HOME/.pyenv/pyenv-win/versions/3.7.5/Scripts:$PATH"
+        - export EXE_PKG="$EXE_NAME"_"$TRAVIS_CPU_ARCH"_windows10
+        - if [ -z "$(ls -A $HOME/.pyenv)" ]; then
+          git clone /~https://github.com/pyenv-win/pyenv-win.git $HOME/.pyenv;
+          pyenv install -q 3.6.7;
+          pyenv install -q 3.7.5;
+          pyenv install -q 3.8.1;
+          fi
+        - pyenv rehash
+        - pyenv global 3.7.5
+        - pyenv local 3.7.5
+        - python --version
+        - python -m pip install --upgrade pip
+        - pip install pywin32
+        - choco install postgresql11
+
+install:
+  - pip install -U tox-travis
 
-# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
-install: pip install -U tox-travis
+script:
+  - tox
 
-# Command to run tests, e.g. python setup.py test
-script: tox
+before_deploy:
+  - pip install pyinstaller
+  - pip install -r requirements.txt
+  - pyinstaller --name $EXE_NAME --onefile cjio_dbexport/cli.py --distpath $DISTPATH
+  - sleep 1
+  - cat $TRAVIS_BUILD_DIR/build/cjdb/warn-cjdb.txt
+  - ls -l $DISTPATH
+  - $DISTPATH/$EXE_NAME || $DISTPATH/$EXE_NAME.exe
+  - |
+    case $TRAVIS_OS_NAME in
+      windows)
+        7z a -t7z $EXE_PKG.zip $DISTPATH/$EXE_NAME.exe
+        rm $DISTPATH/$EXE_NAME.exe
+        ;;
+      linux)
+        zip -r $EXE_PKG.zip $DISTPATH/$EXE_NAME
+        rm $DISTPATH/$EXE_NAME
+        ;;
+    esac
 
+deploy:
+  - provider: releases
+    cleanup: false
+    token:
+      secure: nt4IH6uOIpATCz8BK15FCLMAXswJnnfJmRb6umP8NSB3J4zCIZPZ0p5k8jj9zXlcMCBpTDEONIhoN72DhRUXZ4jGruNfFAu+nHAsezKEjhm8i0lpqMkHrNjYmaTn5Wk+uXXKe5D1Dw9WwBgKvjkY/ADtS53fTusCS6LnmPLUkjPIKIJbRhVyShGexmc4V9J96atn93aETX2C+oShMxDnMHLABRrXaOuxJmXyjSfl3U9L+uf6mRCTUN9f0LS3XTUIgbDWTVa3nItFg+fQ1GYgNTFbCi7GhqKLEqm1VZ9ZgFv+yMFExWaDxtYXqPIr5AwwNl8PHQc+FmsZu0nzaYvmiZTddBqYivMu47wURGV07sY/jXcyi2hf9zswWxWS3oZmDYXK5h385V0v+0ZKVYj2Ux8w9s8XBQqE7nMbpFu/9PtVcm5A9vPw61ceuzTk9WMs7E91QUvvPujXg2Vl7FuvCoMtZlxODAgXiQ+Vr37TYVfxxOrXadzLRFNvkFVjHHr/uog8yiGszEZXi6mJER5cJpOyVqqaOm4r9LKSH556YpUckoqeODD2DjVAspBAzsklJWCHQAG758H+DnYaYLDzCKLFoCahY67m/e1VIGFhxdI1W3s7dMCrJDjID2H5MgqlArshyvXp50bKnDyKuU0FQ6A9nO3RFVi5xGAJBN8l5Ro=
+    file_glob: true
+    file: cjdb_*.zip
+    on:
+      repo: balazsdukai/cjio_dbexport
+      tags: true
+    edge: true
 
+notifications:
+  email: false
diff --git a/cjio_dbexport/cli.py b/cjio_dbexport/cli.py
@@ -24,15 +24,14 @@
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 """
-import sys
 import logging
+import sys
 from pathlib import Path
 
 import click
 from cjio import cityjson
 
-from cjio_dbexport import recorder, configure, db
-from cjio_dbexport import db3dnl
+from cjio_dbexport import recorder, configure, db, db3dnl
 
 
 @click.group()
@@ -60,6 +59,7 @@ def main(ctx, verbose, quiet, configuration):
     ctx.obj['cfg'] = configure.parse_configuration(configuration)
     return 0
 
+
 @click.command('export')
 @click.option('--bbox', nargs=4, type=float,
               help='2D bbox: (minx miny maxx maxy).')
@@ -84,6 +84,5 @@ def export_cmd(ctx, bbox, filename):
 
 main.add_command(export_cmd)
 
-
 if __name__ == "__main__":
     sys.exit(main())  # pragma: no cover
diff --git a/cjio_dbexport/configure.py b/cjio_dbexport/configure.py
@@ -57,14 +57,17 @@ def verify_cotypes(cfg: Mapping) -> bool:
         'tunnelpart', 'tunnelinstallation'
     ]
     if 'cityobject_type' not in cfg:
-        raise ValueError("The configuration file must have a member 'cityobject_type'")
+        raise ValueError(
+            "The configuration file must have a member 'cityobject_type'")
     else:
         for cotype in cfg['cityobject_type']:
             _cotype = cotype.lower()
             if _cotype == 'cityobjectgroup':
                 log.error("CityObjectGroup type is not supported")
             elif _cotype in second_level:
-                f_lvl = _cotype.replace('installation','').replace('part','').replace('constructionelement', '')
+                f_lvl = _cotype.replace('installation', '').replace('part',
+                                                                    '').replace(
+                    'constructionelement', '')
                 if f_lvl not in cfg['cityobject_type']:
                     raise ValueError(f"Cannot declare 2nd-level CityObject "
                                      f"{_cotype} by itself. It must have a "

diff --git a/cjio_dbexport/db3dnl.py b/cjio_dbexport/db3dnl.py
@@ -25,18 +25,18 @@
 """
 import logging
 import re
-from typing import Mapping
 from datetime import datetime
+from typing import Mapping
 
 from cjio import cityjson
 from cjio.models import CityObject, Geometry
 from psycopg2 import sql
 
 from cjio_dbexport import db
 
-
 log = logging.getLogger(__name__)
 
+
 def build_query(conn: db.Db, features: db.Schema, bbox=None):
     """Build an SQL query for extracting CityObjects from a single table.
 
@@ -55,16 +55,17 @@ def build_query(conn: db.Db, features: db.Schema, bbox=None):
     else:
         exclude = []
     attr_select = sql.SQL(', ').join(sql.Identifier(col) for col in table_fields
-                                      if col != features.field.pk.string and
-                                      col != features.field.geometry.string and
-                                      col != features.field.cityobject_id.string and
-                                      col not in exclude)
+                                     if col != features.field.pk.string and
+                                     col != features.field.geometry.string and
+                                     col != features.field.cityobject_id.string and
+                                     col not in exclude)
     # BBOX clause
     if bbox:
         log.info(f"Exporting with BBOX {bbox}")
         epsg = 7415
-        where_bbox = sql.SQL(f"WHERE ST_Intersects({features.field.geometry.string},"
-                             f"ST_MakeEnvelope({','.join(map(str, bbox))}, {str(epsg)}))")
+        where_bbox = sql.SQL(
+            f"WHERE ST_Intersects({features.field.geometry.string},"
+            f"ST_MakeEnvelope({','.join(map(str, bbox))}, {str(epsg)}))")
     else:
         where_bbox = sql.SQL("")
 
@@ -84,15 +85,15 @@ def build_query(conn: db.Db, features: db.Schema, bbox=None):
             {pk} pk,
             {attr}
         FROM
-            {table}
+            {TABLE}
     ),
     polygons AS (
         SELECT
             {pk} pk,
             (ST_Dump({geometry})).geom,
             {coid} coid
         FROM
-            {table}
+            {TABLE}
         {where_bbox}
     ),
     boundary AS (
@@ -206,6 +207,6 @@ def parse_polygonz(wkt_polygonz):
         for ring in rings:
             pts = [tuple(map(float, pt.split()))
                    for pt in ring.split(',')]
-            yield pts[1:] # WKT repeats the first vertex
+            yield pts[1:]  # WKT repeats the first vertex
     else:
         log.error("Not a POLYGON Z")
diff --git a/cjio_dbexport/recorder.py b/cjio_dbexport/recorder.py
@@ -23,8 +23,8 @@
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 """
+from logging import basicConfig, getLogger
 from sys import stdout
-from logging import basicConfig,getLogger
 
 log = getLogger(__name__)
 

diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,5 @@
+--index-url https://pypi.python.org/simple/
+
+-e git+/~https://github.com/cityjson/cjio@develop#egg=cjio
+
+-e .
diff --git a/requirements_dev.txt b/requirements_dev.txt
@@ -1,4 +1,4 @@
-pip==19.2.3
+pip>=19.3
 bump2version==0.5.11
 wheel==0.33.6
 watchdog==0.9.0
@@ -10,4 +10,5 @@ twine==1.14.0
 Click==7.0
 pytest==4.6.5
 pytest-runner==5.1
-pyinstaller==3.5
+# Need pyinstaller 4.0
+/~https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -55,7 +55,7 @@ def cfg_open(data_dir):
 
 @pytest.fixture('session')
 def cfg_parsed(data_dir):
-    config = data_dir / 'balazs_config.yml'
+    config = data_dir / 'db3dnl_config.yml'
     with open(config, 'r') as fo:
         c = configure.parse_configuration(fo)
         yield c

diff --git a/tox.ini b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py36, py37, py38, flake8
+envlist = py36, py37, py38
 
 [travis]
 python =