Merge branch 'main' into bencher_experiment

.daily_canary

@@ -3,4 +3,4 @@
  (  V  ) /  .  \  |    +---=---'
 /--x-m- /--n-n---xXx--/--yY------>>>----<<<>>]]{{}}---||-/\---..
 2024__
-!
+!"

.github/dependabot.yml

@@ -14,3 +14,9 @@ updates:
     directory: "/python" # Location of package manifests
     schedule:
       interval: "weekly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every week
+      interval: "weekly"

.github/workflows/codeql-analysis.yml

@@ -8,13 +8,19 @@ name: "CodeQL"
 on:
   push:
     branches: [main]
+  pull_request:
+    branches: [main]
+    paths:
+      - ".github/workflows/codeql-analysis.yml"
 
 permissions: read-all
 
 jobs:
   analyze:
     name: Analyze
-    runs-on: ubuntu-20.04
+    # Insufficient space to run on public runner, so use custom pool
+    runs-on: [self-hosted, 1ES.Pool=gha-virtual-ccf-sub]
+
     permissions:
       security-events: write
 
@@ -35,7 +41,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           queries: security-extended
@@ -64,15 +70,18 @@ jobs:
           sudo apt update
           sudo apt install -y ansible software-properties-common bsdmainutils dnsutils
           sudo ansible-playbook ccf-dev.yml --extra-vars "platform=virtual" --extra-vars "require_open_enclave=false"
+        name: Install dependencies
 
       - run: |
           mkdir build
           cd build
           cmake -DCOMPILE_TARGET=virtual -DREQUIRE_OPENENCLAVE=OFF -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=OFF -DLVI_MITIGATIONS=OFF -DCMAKE_C_COMPILER=`which clang-11` -DCMAKE_CXX_COMPILER=`which clang++-11` ..
+        name: Run CMake
 
       - run: |
           cd build
           make
+        name: Run Make
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/npm.yml

@@ -13,7 +13,7 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 20
 

.github/workflows/tlaplus.yml

@@ -54,7 +54,7 @@ jobs:
           ./tlc.sh -workers auto consistency/MCMultiNodeReadsAlt.tla -dumpTrace json MCMultiNodeReadsAlt.json
 
       - name: Upload TLC's out file as an artifact. Can be imported into the TLA+ Toolbox.
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
           name: tlc
@@ -104,7 +104,7 @@ jobs:
           ./tlc.sh -workers auto -simulate num=500 -depth 50 consistency/MultiNodeReads.tla -dumpTrace json MultiNodeReads.json
 
       - name: Upload traces in TLA and JSON format
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
           name: tlc

CMakeLists.txt

@@ -302,19 +302,6 @@ endif()
 
 install(TARGETS cchost DESTINATION bin)
 
-# Perf scenario executable
-add_executable(
-  scenario_perf_client ${CCF_DIR}/src/clients/perf/scenario_perf_client.cpp
-)
-target_link_libraries(
-  scenario_perf_client PRIVATE ${CMAKE_THREAD_LIBS_INIT} http_parser.host
-                               ccfcrypto.host
-)
-if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9)
-  target_link_libraries(scenario_perf_client PRIVATE c++fs)
-endif()
-install(TARGETS scenario_perf_client DESTINATION bin)
-
 # HTTP parser
 if(COMPILE_TARGET STREQUAL "sgx")
   add_enclave_library_c(http_parser.enclave "${HTTP_PARSER_SOURCES}")
@@ -350,8 +337,20 @@ set(CCF_JS_SOURCES
     ${CCF_DIR}/src/js/core/runtime.cpp
     ${CCF_DIR}/src/js/core/context.cpp
     ${CCF_DIR}/src/js/no_plugins.cpp
-    ${CCF_DIR}/src/js/globals/ccf.cpp
     ${CCF_DIR}/src/js/ffi_plugins.cpp
+    ${CCF_DIR}/src/js/extensions/console.cpp
+    ${CCF_DIR}/src/js/extensions/math/random.cpp
+    ${CCF_DIR}/src/js/extensions/ccf/consensus.cpp
+    ${CCF_DIR}/src/js/extensions/ccf/converters.cpp
+    ${CCF_DIR}/src/js/extensions/ccf/crypto.cpp
+    ${CCF_DIR}/src/js/extensions/ccf/gov_effects.cpp
+    ${CCF_DIR}/src/js/extensions/ccf/historical.cpp
+    ${CCF_DIR}/src/js/extensions/ccf/host.cpp
+    ${CCF_DIR}/src/js/extensions/ccf/kv.cpp
+    ${CCF_DIR}/src/js/extensions/ccf/network.cpp
+    ${CCF_DIR}/src/js/extensions/ccf/node.cpp
+    ${CCF_DIR}/src/js/extensions/ccf/rpc.cpp
+    ${CCF_DIR}/src/js/extensions/ccf/request.cpp
 )
 
 if(COMPILE_TARGET STREQUAL "sgx")
@@ -592,10 +591,9 @@ elseif(COMPILE_TARGET STREQUAL "virtual")
   set(JS_SNP_ATTESTATION_VIRTUAL js_snp_attestation.virtual)
 endif()
 
+set(JS_GENERIC_SOURCES ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp)
 if(COMPILE_TARGET STREQUAL "sgx")
-  add_enclave_library(
-    js_generic_base.enclave ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp
-  )
+  add_enclave_library(js_generic_base.enclave ${JS_GENERIC_SOURCES})
   target_link_libraries(js_generic_base.enclave PUBLIC ccf.enclave)
   add_lvi_mitigations(js_generic_base.enclave)
   install(
@@ -604,10 +602,7 @@ if(COMPILE_TARGET STREQUAL "sgx")
     DESTINATION lib
   )
 elseif(COMPILE_TARGET STREQUAL "snp")
-  add_library(
-    js_generic_base.snp STATIC
-    ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp
-  )
+  add_library(js_generic_base.snp STATIC ${JS_GENERIC_SOURCES})
   add_san(js_generic_base.snp)
   add_warning_checks(js_generic_base.snp)
   target_link_libraries(js_generic_base.snp PUBLIC ccf.snp)
@@ -623,10 +618,7 @@ elseif(COMPILE_TARGET STREQUAL "snp")
     DESTINATION lib
   )
 elseif(COMPILE_TARGET STREQUAL "virtual")
-  add_library(
-    js_generic_base.virtual STATIC
-    ${CCF_DIR}/src/apps/js_generic/js_generic_base.cpp
-  )
+  add_library(js_generic_base.virtual STATIC ${JS_GENERIC_SOURCES})
   add_san(js_generic_base.virtual)
   add_warning_checks(js_generic_base.virtual)
   target_link_libraries(js_generic_base.virtual PUBLIC ccf.virtual)
@@ -1431,6 +1423,11 @@ if(BUILD_TESTS)
                       ${CMAKE_SOURCE_DIR}/samples/apps/logging/js
     )
 
+    add_e2e_test(
+      NAME programmability
+      PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/programmability.py
+    )
+
     # This test uses large requests (so too slow for SAN)
     if(NOT SAN)
       add_e2e_test(
@@ -1584,23 +1581,6 @@ if(BUILD_TESTS)
       )
     endif()
 
-    add_perf_test(
-      NAME ls
-      PYTHON_SCRIPT ${CMAKE_CURRENT_LIST_DIR}/tests/infra/perfclient.py
-      CLIENT_BIN ./scenario_perf_client
-      ADDITIONAL_ARGS
-        --package
-        "samples/apps/logging/liblogging"
-        --scenario-file
-        ${CMAKE_CURRENT_LIST_DIR}/tests/perf_logging_scenario_100txs.json
-        --max-writes-ahead
-        1000
-        --repetitions
-        10000
-        --msg-ser-fmt
-        text
-    )
-
     add_perf_test(
       NAME pi_ls
       PYTHON_SCRIPT ${CMAKE_CURRENT_LIST_DIR}/tests/infra/piccolo_driver.py
@@ -1635,24 +1615,6 @@ if(BUILD_TESTS)
       )
     endif()
 
-    add_perf_test(
-      NAME ls_jwt
-      PYTHON_SCRIPT ${CMAKE_CURRENT_LIST_DIR}/tests/infra/perfclient.py
-      CLIENT_BIN ./scenario_perf_client
-      ADDITIONAL_ARGS
-        --package
-        "samples/apps/logging/liblogging"
-        --scenario-file
-        ${CMAKE_CURRENT_LIST_DIR}/tests/perf_logging_scenario_100txs.json
-        --max-writes-ahead
-        1000
-        --repetitions
-        1000
-        --use-jwt
-        --msg-ser-fmt
-        msgpack
-    )
-
     add_perf_test(
       NAME pi_ls_jwt
       PYTHON_SCRIPT ${CMAKE_CURRENT_LIST_DIR}/tests/infra/piccolo_driver.py
@@ -1667,58 +1629,6 @@ if(BUILD_TESTS)
         --use-jwt
     )
 
-    add_perf_test(
-      NAME ls_js
-      PYTHON_SCRIPT ${CMAKE_CURRENT_LIST_DIR}/tests/infra/perfclient.py
-      CLIENT_BIN ./scenario_perf_client
-      ADDITIONAL_ARGS
-        --js-app-bundle
-        ${CMAKE_SOURCE_DIR}/samples/apps/logging/js_perf
-        --scenario-file
-        ${CMAKE_CURRENT_LIST_DIR}/tests/perf_logging_scenario_100txs.json
-        --max-writes-ahead
-        1000
-        --repetitions
-        1000
-        --msg-ser-fmt
-        text
-    )
-
-    add_perf_test(
-      NAME ls_full_js
-      PYTHON_SCRIPT ${CMAKE_CURRENT_LIST_DIR}/tests/infra/perfclient.py
-      CLIENT_BIN ./scenario_perf_client
-      ADDITIONAL_ARGS
-        --js-app-bundle
-        ${CMAKE_SOURCE_DIR}/samples/apps/logging/js
-        --scenario-file
-        ${CMAKE_CURRENT_LIST_DIR}/tests/perf_logging_scenario_100txs.json
-        --max-writes-ahead
-        1000
-        --repetitions
-        1000
-        --msg-ser-fmt
-        text
-    )
-
-    add_perf_test(
-      NAME ls_js_jwt
-      PYTHON_SCRIPT ${CMAKE_CURRENT_LIST_DIR}/tests/infra/perfclient.py
-      CLIENT_BIN ./scenario_perf_client
-      ADDITIONAL_ARGS
-        --js-app-bundle
-        ${CMAKE_SOURCE_DIR}/samples/apps/logging/js_perf
-        --scenario-file
-        ${CMAKE_CURRENT_LIST_DIR}/tests/perf_logging_scenario_100txs.json
-        --max-writes-ahead
-        1000
-        --repetitions
-        700
-        --use-jwt
-        --msg-ser-fmt
-        text
-    )
-
     add_e2e_test(
       NAME historical_query_perf_test
       PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/historical_query_perf.py

getting_started/setup_vm/roles/ccf_build/tasks/install.yml

@@ -7,7 +7,7 @@
     update_cache: yes
   become: yes
 
-# Note: Do not install docker CLI if moby-buildx is already installed
+# Note: Do not install docker CLI if docker-ce-cli is already installed
 - name: Gather the package facts
   ansible.builtin.package_facts:
     manager: auto
@@ -17,22 +17,22 @@
     url: "https://download.docker.com/linux/ubuntu/gpg"
     state: present
   become: yes
-  when: "'moby-buildx' not in ansible_facts.packages"
+  when: "'docker-ce-cli' not in ansible_facts.packages"
 
 - name: Add docker repository
   apt_repository:
     repo: "deb https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
     state: present
     update_cache: yes
   become: yes
-  when: "'moby-buildx' not in ansible_facts.packages"
+  when: "'docker-ce-cli' not in ansible_facts.packages"
 
 - name: Install docker debs
   apt:
     name: "{{ docker_debs }}"
     update_cache: yes
   become: yes
-  when: "'moby-buildx' not in ansible_facts.packages"
+  when: "'docker-ce-cli' not in ansible_facts.packages"
 
 - name: Remove doxygen debian package
   apt:

include/ccf/bundle.h

@@ -0,0 +1,39 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "ccf/ds/json.h"
+#include "ccf/endpoint.h"
+
+#include <map>
+#include <string>
+
+namespace ccf::js
+{
+  struct Metadata
+  {
+    std::map<
+      std::string,
+      std::map<std::string, ccf::endpoints::EndpointProperties>>
+      endpoints;
+  };
+  DECLARE_JSON_TYPE(Metadata);
+  DECLARE_JSON_REQUIRED_FIELDS(Metadata, endpoints);
+
+  struct Bundle
+  {
+    std::map<std::string, std::string> modules;
+    Metadata metadata;
+  };
+
+  DECLARE_JSON_TYPE(Bundle);
+  DECLARE_JSON_REQUIRED_FIELDS(Bundle, modules, metadata);
+
+  struct BundleWrapper
+  {
+    Bundle bundle;
+  };
+
+  DECLARE_JSON_TYPE(BundleWrapper);
+  DECLARE_JSON_REQUIRED_FIELDS(BundleWrapper, bundle);
+}

include/ccf/claims_digest.h

@@ -65,4 +65,11 @@ namespace ccf
   {
     ds::json::fill_schema<ClaimsDigest::Digest>(schema);
   }
+
+  static ClaimsDigest empty_claims()
+  {
+    ClaimsDigest cd;
+    cd.set(ClaimsDigest::Digest::Representation());
+    return cd;
+  }
 }