From 66aee5f6c882caf8fc58eb68827ba90472d67bb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Dukai?= Date: Fri, 10 Jan 2020 21:11:23 +0100 Subject: [PATCH] Add CI/CD with Travis-CI + Test on master, develop, release-* branches + Test when a commit is tagged as v* + Test with tox on linux (py37), on windows (py36, py37, py38) + Deploy to GitHub Release only the tagged commits + Compile executable for Ubuntu18.04, Windows10 --- .gitignore | 5 +- .travis.yml | 99 +++++++++++++++++++++++++++++++++++--- cjio_dbexport/cli.py | 7 ++- cjio_dbexport/configure.py | 7 ++- cjio_dbexport/db3dnl.py | 23 ++++----- cjio_dbexport/recorder.py | 2 +- requirements.txt | 5 ++ requirements_dev.txt | 5 +- tests/conftest.py | 2 +- tox.ini | 2 +- 10 files changed, 124 insertions(+), 33 deletions(-) create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 78f6ac4..10f3546 100644 --- a/.gitignore +++ b/.gitignore @@ -51,9 +51,8 @@ coverage.xml .hypothesis/ .pytest_cache/ # Test datasets -tests/data/balazs_config.yml -tests/data/test.json -tests/data/test_all.json +tests/data/db3dnl_config.yml +tests/data/*.json # Translations *.mo diff --git a/.travis.yml b/.travis.yml index 1067cab..90896c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,98 @@ -# Config file for automatic testing at travis-ci.org - language: python python: - - 3.8 - 3.7 - - 3.6 +cache: + - pip + - directories: + - $HOME/.pyenv +# Would be great to cache Postgres here but dir names with spaces are split into two and escaping doesn't work +# - "/c/Program\ Files/PostgreSQL" + +env: + - EXE_NAME=cjdb DISTPATH=$TRAVIS_BUILD_DIR/dist +os: + - linux + +branches: + only: + - master + - develop + # Run on release-* branches. Case insensitive. + - /^(?i:release)-.*$/ + # A build is triggered with a tag push, in which case the branch name is + # the tag name. Thus in order for on.tags:true to work, the tag-branch + # must be safelisted + - /^v.*$/ + +stages: + - name: test + - name: deploy + # The executable is built only with python3.7 on the master branch + if: branch = master AND env(TRAVIS_PYTHON_VERSION) = 3.7 + +jobs: + include: + - name: Python 3.7.5 on Bionic Ubuntu + dist: bionic + before_install: + - export EXE_PKG="$EXE_NAME"_"$TRAVIS_CPU_ARCH"_ubuntu1804 + - name: Python 3.7.5 with pyenv on Windows + os: windows + language: shell + before_install: + - export PATH="$HOME/.pyenv/pyenv-win/bin:$HOME/.pyenv/pyenv-win/shims:$HOME/.pyenv/pyenv-win/versions/3.7.5:$HOME/.pyenv/pyenv-win/versions/3.7.5/Scripts:$PATH" + - export EXE_PKG="$EXE_NAME"_"$TRAVIS_CPU_ARCH"_windows10 + - if [ -z "$(ls -A $HOME/.pyenv)" ]; then + git clone /~https://github.com/pyenv-win/pyenv-win.git $HOME/.pyenv; + pyenv install -q 3.6.7; + pyenv install -q 3.7.5; + pyenv install -q 3.8.1; + fi + - pyenv rehash + - pyenv global 3.7.5 + - pyenv local 3.7.5 + - python --version + - python -m pip install --upgrade pip + - pip install pywin32 + - choco install postgresql11 + +install: + - pip install -U tox-travis -# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: pip install -U tox-travis +script: + - tox -# Command to run tests, e.g. python setup.py test -script: tox +before_deploy: + - pip install pyinstaller + - pip install -r requirements.txt + - pyinstaller --name $EXE_NAME --onefile cjio_dbexport/cli.py --distpath $DISTPATH + - sleep 1 + - cat $TRAVIS_BUILD_DIR/build/cjdb/warn-cjdb.txt + - ls -l $DISTPATH + - $DISTPATH/$EXE_NAME || $DISTPATH/$EXE_NAME.exe + - | + case $TRAVIS_OS_NAME in + windows) + 7z a -t7z $EXE_PKG.zip $DISTPATH/$EXE_NAME.exe + rm $DISTPATH/$EXE_NAME.exe + ;; + linux) + zip -r $EXE_PKG.zip $DISTPATH/$EXE_NAME + rm $DISTPATH/$EXE_NAME + ;; + esac +deploy: + - provider: releases + cleanup: false + token: + secure: nt4IH6uOIpATCz8BK15FCLMAXswJnnfJmRb6umP8NSB3J4zCIZPZ0p5k8jj9zXlcMCBpTDEONIhoN72DhRUXZ4jGruNfFAu+nHAsezKEjhm8i0lpqMkHrNjYmaTn5Wk+uXXKe5D1Dw9WwBgKvjkY/ADtS53fTusCS6LnmPLUkjPIKIJbRhVyShGexmc4V9J96atn93aETX2C+oShMxDnMHLABRrXaOuxJmXyjSfl3U9L+uf6mRCTUN9f0LS3XTUIgbDWTVa3nItFg+fQ1GYgNTFbCi7GhqKLEqm1VZ9ZgFv+yMFExWaDxtYXqPIr5AwwNl8PHQc+FmsZu0nzaYvmiZTddBqYivMu47wURGV07sY/jXcyi2hf9zswWxWS3oZmDYXK5h385V0v+0ZKVYj2Ux8w9s8XBQqE7nMbpFu/9PtVcm5A9vPw61ceuzTk9WMs7E91QUvvPujXg2Vl7FuvCoMtZlxODAgXiQ+Vr37TYVfxxOrXadzLRFNvkFVjHHr/uog8yiGszEZXi6mJER5cJpOyVqqaOm4r9LKSH556YpUckoqeODD2DjVAspBAzsklJWCHQAG758H+DnYaYLDzCKLFoCahY67m/e1VIGFhxdI1W3s7dMCrJDjID2H5MgqlArshyvXp50bKnDyKuU0FQ6A9nO3RFVi5xGAJBN8l5Ro= + file_glob: true + file: cjdb_*.zip + on: + repo: balazsdukai/cjio_dbexport + tags: true + edge: true +notifications: + email: false diff --git a/cjio_dbexport/cli.py b/cjio_dbexport/cli.py index a932416..3124908 100644 --- a/cjio_dbexport/cli.py +++ b/cjio_dbexport/cli.py @@ -24,15 +24,14 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -import sys import logging +import sys from pathlib import Path import click from cjio import cityjson -from cjio_dbexport import recorder, configure, db -from cjio_dbexport import db3dnl +from cjio_dbexport import recorder, configure, db, db3dnl @click.group() @@ -60,6 +59,7 @@ def main(ctx, verbose, quiet, configuration): ctx.obj['cfg'] = configure.parse_configuration(configuration) return 0 + @click.command('export') @click.option('--bbox', nargs=4, type=float, help='2D bbox: (minx miny maxx maxy).') @@ -84,6 +84,5 @@ def export_cmd(ctx, bbox, filename): main.add_command(export_cmd) - if __name__ == "__main__": sys.exit(main()) # pragma: no cover diff --git a/cjio_dbexport/configure.py b/cjio_dbexport/configure.py index 5d4c779..1c596b3 100644 --- a/cjio_dbexport/configure.py +++ b/cjio_dbexport/configure.py @@ -57,14 +57,17 @@ def verify_cotypes(cfg: Mapping) -> bool: 'tunnelpart', 'tunnelinstallation' ] if 'cityobject_type' not in cfg: - raise ValueError("The configuration file must have a member 'cityobject_type'") + raise ValueError( + "The configuration file must have a member 'cityobject_type'") else: for cotype in cfg['cityobject_type']: _cotype = cotype.lower() if _cotype == 'cityobjectgroup': log.error("CityObjectGroup type is not supported") elif _cotype in second_level: - f_lvl = _cotype.replace('installation','').replace('part','').replace('constructionelement', '') + f_lvl = _cotype.replace('installation', '').replace('part', + '').replace( + 'constructionelement', '') if f_lvl not in cfg['cityobject_type']: raise ValueError(f"Cannot declare 2nd-level CityObject " f"{_cotype} by itself. It must have a " diff --git a/cjio_dbexport/db3dnl.py b/cjio_dbexport/db3dnl.py index f2cd9dd..1ec9da8 100644 --- a/cjio_dbexport/db3dnl.py +++ b/cjio_dbexport/db3dnl.py @@ -25,8 +25,8 @@ """ import logging import re -from typing import Mapping from datetime import datetime +from typing import Mapping from cjio import cityjson from cjio.models import CityObject, Geometry @@ -34,9 +34,9 @@ from cjio_dbexport import db - log = logging.getLogger(__name__) + def build_query(conn: db.Db, features: db.Schema, bbox=None): """Build an SQL query for extracting CityObjects from a single table. @@ -55,16 +55,17 @@ def build_query(conn: db.Db, features: db.Schema, bbox=None): else: exclude = [] attr_select = sql.SQL(', ').join(sql.Identifier(col) for col in table_fields - if col != features.field.pk.string and - col != features.field.geometry.string and - col != features.field.cityobject_id.string and - col not in exclude) + if col != features.field.pk.string and + col != features.field.geometry.string and + col != features.field.cityobject_id.string and + col not in exclude) # BBOX clause if bbox: log.info(f"Exporting with BBOX {bbox}") epsg = 7415 - where_bbox = sql.SQL(f"WHERE ST_Intersects({features.field.geometry.string}," - f"ST_MakeEnvelope({','.join(map(str, bbox))}, {str(epsg)}))") + where_bbox = sql.SQL( + f"WHERE ST_Intersects({features.field.geometry.string}," + f"ST_MakeEnvelope({','.join(map(str, bbox))}, {str(epsg)}))") else: where_bbox = sql.SQL("") @@ -84,7 +85,7 @@ def build_query(conn: db.Db, features: db.Schema, bbox=None): {pk} pk, {attr} FROM - {table} + {TABLE} ), polygons AS ( SELECT @@ -92,7 +93,7 @@ def build_query(conn: db.Db, features: db.Schema, bbox=None): (ST_Dump({geometry})).geom, {coid} coid FROM - {table} + {TABLE} {where_bbox} ), boundary AS ( @@ -206,6 +207,6 @@ def parse_polygonz(wkt_polygonz): for ring in rings: pts = [tuple(map(float, pt.split())) for pt in ring.split(',')] - yield pts[1:] # WKT repeats the first vertex + yield pts[1:] # WKT repeats the first vertex else: log.error("Not a POLYGON Z") diff --git a/cjio_dbexport/recorder.py b/cjio_dbexport/recorder.py index 15578fd..c6c8a20 100644 --- a/cjio_dbexport/recorder.py +++ b/cjio_dbexport/recorder.py @@ -23,8 +23,8 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +from logging import basicConfig, getLogger from sys import stdout -from logging import basicConfig,getLogger log = getLogger(__name__) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5a2b0ee --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +--index-url https://pypi.python.org/simple/ + +-e git+/~https://github.com/cityjson/cjio@develop#egg=cjio + +-e . \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index c6b3dbb..8b66b24 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,4 @@ -pip==19.2.3 +pip>=19.3 bump2version==0.5.11 wheel==0.33.6 watchdog==0.9.0 @@ -10,4 +10,5 @@ twine==1.14.0 Click==7.0 pytest==4.6.5 pytest-runner==5.1 -pyinstaller==3.5 \ No newline at end of file +# Need pyinstaller 4.0 +/~https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 8190e0a..f8422ad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,7 +55,7 @@ def cfg_open(data_dir): @pytest.fixture('session') def cfg_parsed(data_dir): - config = data_dir / 'balazs_config.yml' + config = data_dir / 'db3dnl_config.yml' with open(config, 'r') as fo: c = configure.parse_configuration(fo) yield c diff --git a/tox.ini b/tox.ini index 4230e5a..36c23d4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38, flake8 +envlist = py36, py37, py38 [travis] python =