zombienet: warp-sync integration test added

.gitlab-ci.yml

@@ -33,6 +33,7 @@ stages:
   - test
   - build
   - publish
+  - zombienet
   - deploy
   - notify
 
@@ -51,6 +52,7 @@ variables:
   BUILDAH_IMAGE:                   "quay.io/buildah/stable:v1.27"
   RUSTY_CACHIER_SINGLE_BRANCH:     master
   RUSTY_CACHIER_DONT_OPERATE_ON_MAIN_BRANCH: "true"
+  ZOMBIENET_IMAGE:                 "docker.io/paritytech/zombienet:v1.2.78"
 
 default:
   retry:
@@ -183,14 +185,28 @@ default:
         - frame/contracts/**/*
         - primitives/sandbox/**/*
 
+.publish-refs:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+
 .build-refs:
+  # publish-refs + PRs
   rules:
     - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+
+.zombienet-refs:
+  extends: .build-refs
 
 .nightly-pipeline:
   rules:
@@ -222,6 +238,8 @@ include:
   - scripts/ci/gitlab/pipeline/build.yml
   # publish jobs
   - scripts/ci/gitlab/pipeline/publish.yml
+  # zombienet jobs
+  - scripts/ci/gitlab/pipeline/zombienet.yml
 
 #### stage:                        deploy
 

scripts/ci/docker/subkey.Dockerfile

@@ -3,10 +3,11 @@ FROM docker.io/library/ubuntu:20.04
 # metadata
 ARG VCS_REF
 ARG BUILD_DATE
+ARG IMAGE_NAME
 
 LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
-	io.parity.image.title="parity/subkey" \
+	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="Subkey: key generating utility for Substrate." \
 	io.parity.image.source="/~https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/subkey.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \

scripts/ci/docker/substrate.Dockerfile

@@ -3,10 +3,11 @@ FROM docker.io/library/ubuntu:20.04
 # metadata
 ARG VCS_REF
 ARG BUILD_DATE
+ARG IMAGE_NAME
 
 LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
-	io.parity.image.title="parity/substrate" \
+	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="Substrate: The platform for blockchain innovators." \
 	io.parity.image.source="/~https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \

scripts/ci/gitlab/pipeline/build.yml

@@ -86,7 +86,7 @@ build-linux-substrate:
   extends:
     - .collect-artifacts
     - .docker-env
-    - .build-refs
+    - .publish-refs
   variables:
     # this variable gets overriden by "rusty-cachier environment inject", use the value as default
     CARGO_TARGET_DIR:              "$CI_PROJECT_DIR/target"

scripts/ci/gitlab/pipeline/publish.yml

@@ -2,32 +2,33 @@
 # This file is part of .gitlab-ci.yml
 # Here are all jobs that are executed during "publish" stage
 
-.build-push-docker-image:
+.build-push-docker-image-common:
   extends:
-    - .build-refs
     - .kubernetes-env
+  stage:                           publish
   variables:
     CI_IMAGE:                      $BUILDAH_IMAGE
     GIT_STRATEGY:                  none
     DOCKERFILE:                    $PRODUCT.Dockerfile
-    IMAGE_NAME:                    docker.io/parity/$PRODUCT
+    IMAGE_NAME:                    docker.io/$IMAGE_PATH
   before_script:
     - cd ./artifacts/$PRODUCT/
     - VERSION="$(cat ./VERSION)"
     - echo "${PRODUCT} version = ${VERSION}"
     - test -z "${VERSION}" && exit 1
   script:
-    - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" ||
+    - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
         ( echo "no docker credentials provided"; exit 1 )
     - buildah bud
         --format=docker
         --build-arg VCS_REF="${CI_COMMIT_SHA}"
         --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+        --build-arg IMAGE_NAME="${IMAGE_PATH}"
         --tag "$IMAGE_NAME:$VERSION"
         --tag "$IMAGE_NAME:latest"
         --file "$DOCKERFILE" .
-    - echo "$Docker_Hub_Pass_Parity" |
-        buildah login --username "$Docker_Hub_User_Parity" --password-stdin docker.io
+    - echo "$DOCKER_PASS" |
+        buildah login --username "$DOCKER_USER" --password-stdin docker.io
     - buildah info
     - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
     - buildah push --format=v2s2 "$IMAGE_NAME:latest"
@@ -38,17 +39,49 @@
     - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}"   | tee -a ./artifacts/$PRODUCT/build.env
     - cat ./artifacts/$PRODUCT/build.env
 
+.build-push-docker-image:
+  extends:
+    - .publish-refs
+    - .build-push-docker-image-common
+  variables:
+    IMAGE_PATH:                    parity/$PRODUCT
+    DOCKER_USER:                   $Docker_Hub_User_Parity
+    DOCKER_PASS:                   $Docker_Hub_Pass_Parity
+
+
+# publish image to docker.io/paritypr, (e.g. for later use in zombienet testing)
+.build-push-image-temporary:
+  extends:
+    - .build-refs
+    - .build-push-docker-image-common
+  variables:
+    IMAGE_PATH:                    paritypr/$PRODUCT
+    DOCKER_USER:                   $PARITYPR_USER
+    DOCKER_PASS:                   $PARITYPR_PASS
+
 publish-docker-substrate:
-  stage:                           publish
   extends:                         .build-push-docker-image
   needs:
     - job:                         build-linux-substrate
       artifacts:                   true
   variables:
     PRODUCT:                       substrate
 
+publish-docker-substrate-temporary:
+  extends:                         .build-push-image-temporary
+  needs:
+    - job:                         build-linux-substrate
+      artifacts:                   true
+  variables:
+    PRODUCT:                       substrate
+  artifacts:
+    reports:
+      # this artifact is used in zombienet-tests job
+      # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
+      dotenv: ./artifacts/$PRODUCT/build.env
+    expire_in:                     24h
+
 publish-docker-subkey:
-  stage:                           publish
   extends:                         .build-push-docker-image
   needs:
     - job:                         build-subkey-linux
@@ -59,7 +92,7 @@ publish-docker-subkey:
 publish-s3-release:
   stage:                           publish
   extends:
-    - .build-refs
+    - .publish-refs
     - .kubernetes-env
   needs:
     - job:                         build-linux-substrate

scripts/ci/gitlab/pipeline/zombienet.yml

@@ -0,0 +1,34 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "zombienet" stage
+
+# common settings for all zombienet jobs
+.zombienet-common:
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}
+    - echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  needs:
+    - job:                         publish-docker-substrate-temporary
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  variables:
+    GH_DIR:                        "/~https://github.com/paritytech/substrate/tree/${CI_COMMIT_SHA}/zombienet"
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-0000-block-building:
+  extends:
+    - .zombienet-common
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}/0000_block_building"
+        --test="0000-block-building.zndsl"

zombienet/0000_block_building/0000-block-building.toml

@@ -0,0 +1,15 @@
+[settings]
+enable_tracing = false
+
+[relaychain]
+default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}"
+default_command = "substrate"
+chain = "local"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true

zombienet/0000_block_building/0000-block-building.zndsl

@@ -0,0 +1,15 @@
+Description: Block building
+Network: ./0000-block-building.toml
+Creds: config
+
+alice: is up
+bob: is up
+
+alice: reports node_roles is 4
+bob: reports node_roles is 4
+
+alice: reports peers count is at least 1
+bob: reports peers count is at least 1
+
+alice: reports block height is at least 5 within 20 seconds
+bob: reports block height is at least 5 within 20 seconds

zombienet/0001-basic-warp-sync/0001-README.md

@@ -0,0 +1,75 @@
+# Test design
+The `warp-sync` test works on predefined database which is stored in the cloud and
+fetched by the test. `alice` and `bob` nodes are spun up using this database snapshot.
+
+As `warp-sync` requires at least 3 peers, the test spawns the `charlie` full node which uses the same database snapshot.
+
+The `dave` node executed with `--sync warp` syncs database with the rest of the network.
+
+# How to prepare database
+Database was prepared using the following zombienet file (`0001-generate-warp-sync-database.toml`):
+```
+[relaychain]
+default_image = "docker.io/parity/substrate:master"
+default_command = "substrate"
+
+chain = "gen-db"
+
+chain_spec_path = "0001-chain-spec.json"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true
+```
+
+The zombienet shall be executed with the following command, and run for some period of time to allow for few grandpa eras.
+```
+./zombienet-linux spawn --dir ./db-test-gen --provider native 0001-generate-warp-sync-database.toml
+```
+
+Once the zombienet is stopped, the database snapshot
+(`{alice,bob}/data/chains/local_testnet/db/` dirs) was created using the following
+commands:
+```bash
+mkdir -p db-snapshot/{alice,bob}/data/chains/local_testnet/db/  
+cp -r db-test-gen/alice/data/chains/local_testnet/db/full db-snapshot/alice/data/chains/local_testnet/db/  
+cp -r db-test-gen/bob/data/chains/local_testnet/db/full   db-snapshot/bob/data/chains/local_testnet/db/
+```
+
+Also refer to: [zombienet#578](/~https://github.com/paritytech/zombienet/issues/578)
+
+# Chain spec
+Chain spec was simply built with:
+```
+substrate build-spec --chain=local > 0001-chain-spec.json
+```
+
+# Run the test
+Test can be run with the following command:
+```
+zombienet-linux test --dir db-snapshot --provider native 0001-test-warp-sync.zndsl
+```
+
+*NOTE*: currently blocked by: [zombienet#578](/~https://github.com/paritytech/zombienet/issues/578)
+
+
+# Save some time hack
+Substrate can be patched to reduce the grandpa session period.
+```
+diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs
+index 23fb13cfb0..89f8646291 100644
+--- a/bin/node/runtime/src/constants.rs
++++ b/bin/node/runtime/src/constants.rs
+@@ -63,7 +63,7 @@ pub mod time {
+
+    // NOTE: Currently it is not possible to change the epoch duration after the chain has started.
+    //       Attempting to do so will brick block production.
+-   pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 10 * MINUTES;
++   pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 1 * MINUTES / 2;
+    pub const EPOCH_DURATION_IN_SLOTS: u64 = {
+        const SLOT_FILL_RATE: f64 = MILLISECS_PER_BLOCK as f64 / SLOT_DURATION as f64
+```

zombienet/0001-basic-warp-sync/0001-generate-warp-sync-database.toml

@@ -0,0 +1,30 @@
+# this file is not intended to be executed in CI stage
+[relaychain]
+default_image = "docker.io/parity/substrate:latest"
+default_command = "substrate"
+
+chain = "gen-db"
+
+#built with: substrate build-spec --chain=local
+chain_spec_path = "0001-chain-spec.json"
+
+# to build the database:
+# ./zombienet-linux spawn --dir ./db-test-gen --provider native 0000x-gen-db.toml
+
+# to create snapshot for testing:
+# mkdir -p db-snapshot/{alice,bob}/data/chains/local_testnet/db/  
+# cp -r db-test-gen/alice/data/chains/local_testnet/db/full db-snapshot/alice/data/chains/local_testnet/db/  
+# cp -r db-test-gen/bob/data/chains/local_testnet/db/full   db-snapshot/bob/data/chains/local_testnet/db/
+# cp db-snapshot db
+
+# to run the test:
+# ./zombienet-linux spawn --dir db --provider native 0000y-test-db.toml
+
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true

zombienet/0001-basic-warp-sync/0001-test-warp-sync.toml

@@ -0,0 +1,34 @@
+[settings]
+enable_tracing = false
+
+[relaychain]
+default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}"
+default_command = "substrate"
+
+chain = "gen-db"
+chain_spec_path = "0001-chain-spec.json"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  # see: /~https://github.com/paritytech/zombienet/issues/578
+  # dir_snapshot="s3://bucket/0001-test-warp-sync/snapshots/node-db.tgz"
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true
+  # dir_snapshot="s3://bucket/0001-test-warp-sync/snapshots/node-db.tgz"
+
+  #we need at least 3 nodes for warp sync
+  [[relaychain.nodes]]
+  name = "charlie"
+  validator = false
+  # dir_snapshot="s3://bucket/0001-test-warp-sync/snapshots/node-db.tgz"
+
+  [[relaychain.nodes]]
+  name = "dave"
+  validator = false
+  args = ["--sync warp"]
+  # port is fixed for local debugging convenience
+  ws_port=39047