From ade7cdeb3c5fb986ce9bb7a9c42617a32b8f4d85 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Tue, 27 Nov 2018 19:42:29 -0800 Subject: [PATCH] Add relational database store (#194) * Move to latest cava build * Add a SQL database store --- build.gradle | 2 + documentation/install/configure.md | 39 ++++++++++++++++++- gradle/check-licenses.gradle | 1 + .../consensys/orion/acceptance/NodeUtils.java | 5 ++- .../acceptance/send/SingleNodeSendTest.java | 3 +- .../receive/DualNodesSendReceiveTest.java | 15 ++++++- .../receive/SingleNodeSendReceiveTest.java | 3 +- .../java/net/consensys/orion/cmd/Orion.java | 16 +++++--- .../net/consensys/orion/config/Config.java | 6 ++- .../orion/config/TomlConfigTest.java | 3 +- 10 files changed, 78 insertions(+), 15 deletions(-) diff --git a/build.gradle b/build.gradle index d9002d2a..90f224b3 100644 --- a/build.gradle +++ b/build.gradle @@ -223,6 +223,7 @@ dependencies { // storage compile 'org.fusesource.leveldbjni:leveldbjni-all:1.8' compile 'org.mapdb:mapdb:3.0.7' + compile 'com.jolbox:bonecp:0.8.0.RELEASE' // serialization compile 'com.moandjiezana.toml:toml4j:0.7.2' @@ -236,6 +237,7 @@ dependencies { compile 'org.apache.logging.log4j:log4j-1.2-api:2.11.0' // testing + testCompile 'com.h2database:h2:1.4.197' testCompile 'org.junit.jupiter:junit-jupiter-api:5.2.0' testCompile 'org.junit.jupiter:junit-jupiter-params:5.2.0' testRuntime 'org.junit.jupiter:junit-jupiter-engine:5.2.0' diff --git a/documentation/install/configure.md b/documentation/install/configure.md index c84d4c52..1cde7471 100644 --- a/documentation/install/configure.md +++ b/documentation/install/configure.md @@ -40,4 +40,41 @@ privatekeys = ["foo.key"] ``` You can check all the available properties in the -[`sample.conf`](/~https://github.com/ConsenSys/orion/blob/master/src/main/resources/sample.conf) file. \ No newline at end of file +[`sample.conf`](/~https://github.com/ConsenSys/orion/blob/master/src/main/resources/sample.conf) file. + +### Storage + +By default, Orion relies on leveldb to store information. + +#### LevelDB + +``` +storage = "leveldb:oriondb" +``` + +#### MapDB + +Orion offers persistence using [MapDB](http://www.mapdb.org/). + +``` +storage = "mapdb:oriondb" +``` + +#### SQL + +Orion supports working with relational databases. + +* Add the SQL driver jar to the lib folder of the Orion installation +* Create a table in your database: + +| Database | Create statement | +|---|---| +| MySQL | `create table store(key varbinary, value varbinary, primary key(key))` | +| PostgresQL | `create table store(key bytea, value bytea, primary key(key))` | + +* Set storage to: + +``` +storage = "sql:jdbc:postgresql://localhost/oriondb" + +``` \ No newline at end of file diff --git a/gradle/check-licenses.gradle b/gradle/check-licenses.gradle index b1fbf354..fb351802 100644 --- a/gradle/check-licenses.gradle +++ b/gradle/check-licenses.gradle @@ -86,6 +86,7 @@ downloadLicenses { 'Apache License Version 2.0', 'Apache License, Version 2.0', 'Apache Software Licenses', + 'Apache v2', 'ASL, Version 2', 'The Apache License, Version 2.0', 'The Apache Software License, Version 2.0', diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java b/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java index dc17d6ff..b30f2441 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java @@ -61,7 +61,8 @@ public static Config nodeConfig( String privKeys, String tls, String tlsServerTrust, - String tlsClientTrust) throws IOException { + String tlsClientTrust, + String storage) throws IOException { Path workDir = tempDir.resolve("acceptance").toAbsolutePath(); Files.createDirectories(workDir); @@ -77,7 +78,7 @@ public static Config nodeConfig( + "clienturl = \"" + clientUrl + "\"\n" + "clientport = " + clientPort + "\n" + "clientnetworkinterface = \"" + clientNetworkInterface + "\"\n" - + "storage = \"leveldb:database/" + nodeName + "\"\n" + + "storage = \"" + storage + "\"\n" + "othernodes = [\"" + otherNodes + "\"]\n" + "publickeys = [" + pubKeys + "]\n" + "privatekeys = [" + privKeys + "]\n" diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java index 6fc10946..7ec79dcb 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java @@ -82,7 +82,8 @@ static void setUpSingleNode(@TempDirectory Path tempDir) throws Exception { joinPathsAsTomlListEntry(key1key, key2key), "off", "tofu", - "tofu"); + "tofu", + "leveldb:database/node1"); } @BeforeEach diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java index 496670be..d65f5104 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java @@ -37,6 +37,9 @@ import java.net.URL; import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; import java.util.concurrent.TimeUnit; import io.vertx.core.Vertx; @@ -84,6 +87,12 @@ void setUpDualNodes(@TempDirectory Path tempDir) throws Exception { Path key2pub = copyResource("key2.pub", tempDir.resolve("key2.pub")); Path key2key = copyResource("key2.key", tempDir.resolve("key2.key")); + String jdbcUrl = "jdbc:h2:" + tempDir.resolve("node2").toString(); + try (Connection conn = DriverManager.getConnection(jdbcUrl)) { + Statement st = conn.createStatement(); + st.executeUpdate("create table if not exists store(key binary, value binary, primary key(key))"); + } + firstNodeConfig = NodeUtils.nodeConfig( tempDir, firstNodeBaseUrl, @@ -98,7 +107,8 @@ void setUpDualNodes(@TempDirectory Path tempDir) throws Exception { joinPathsAsTomlListEntry(key1key), "off", "tofu", - "tofu"); + "tofu", + "leveldb:database/node1"); secondNodeConfig = NodeUtils.nodeConfig( tempDir, secondNodeBaseUrl, @@ -113,7 +123,8 @@ void setUpDualNodes(@TempDirectory Path tempDir) throws Exception { joinPathsAsTomlListEntry(key2key), "off", "tofu", - "tofu"); + "tofu", + "sql:" + jdbcUrl); vertx = vertx(); firstOrionLauncher = NodeUtils.startOrion(firstNodeConfig); firstHttpClient = vertx.createHttpClient(); diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/SingleNodeSendReceiveTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/SingleNodeSendReceiveTest.java index c2f2ef25..d9ef75ea 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/SingleNodeSendReceiveTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/SingleNodeSendReceiveTest.java @@ -80,7 +80,8 @@ static void setUpSingleNode(@TempDirectory Path tempDir) throws Exception { joinPathsAsTomlListEntry(key1key, key2key), "off", "tofu", - "tofu"); + "tofu", + "leveldb:database/node1"); } @BeforeEach diff --git a/src/main/java/net/consensys/orion/cmd/Orion.java b/src/main/java/net/consensys/orion/cmd/Orion.java index be569398..fbd42bc6 100644 --- a/src/main/java/net/consensys/orion/cmd/Orion.java +++ b/src/main/java/net/consensys/orion/cmd/Orion.java @@ -23,6 +23,7 @@ import net.consensys.cava.kv.KeyValueStore; import net.consensys.cava.kv.LevelDBKeyValueStore; import net.consensys.cava.kv.MapDBKeyValueStore; +import net.consensys.cava.kv.SQLKeyValueStore; import net.consensys.cava.net.tls.TLS; import net.consensys.cava.net.tls.VertxTrustOptions; import net.consensys.orion.config.Config; @@ -438,18 +439,23 @@ private KeyValueStore createStorage(String storage, Path storagePath) { db = storageOptions[1]; } - Path dbPath = storagePath.resolve(db); if (storage.toLowerCase().startsWith("mapdb")) { try { - return MapDBKeyValueStore.open(dbPath); + return MapDBKeyValueStore.open(storagePath.resolve(db)); } catch (IOException e) { - throw new OrionStartException("Couldn't create MapDB store: " + dbPath, e); + throw new OrionStartException("Couldn't create MapDB store: " + db, e); } } else if (storage.toLowerCase().startsWith("leveldb")) { try { - return LevelDBKeyValueStore.open(dbPath); + return LevelDBKeyValueStore.open(storagePath.resolve(db)); } catch (IOException e) { - throw new OrionStartException("Couldn't create LevelDB store: " + dbPath, e); + throw new OrionStartException("Couldn't create LevelDB store: " + db, e); + } + } else if (storage.toLowerCase().startsWith("sql")) { + try { + return SQLKeyValueStore.open(db); + } catch (IOException e) { + throw new OrionStartException("Couldn't create SQL-backed store: " + db, e); } } else { throw new OrionStartException("unsupported storage mechanism: " + storage); diff --git a/src/main/java/net/consensys/orion/config/Config.java b/src/main/java/net/consensys/orion/config/Config.java index 33c1de5c..ca9f14a3 100644 --- a/src/main/java/net/consensys/orion/config/Config.java +++ b/src/main/java/net/consensys/orion/config/Config.java @@ -243,6 +243,7 @@ public Optional passwords() { * * @@ -523,6 +524,7 @@ private static Schema configSchema() { + "\n" + " - leveldb:path - LevelDB\n" + " - mapdb:path - MapDB\n" + + " - sql:jdbcurl - SQL database\n" + " - memory - Contents are cleared when Orion exits", Config::validateStorage); @@ -663,10 +665,10 @@ private static List validateStorage( @Nullable String value) { assert (value != null); String storageType = value.split(":", 2)[0]; - if (!Arrays.asList("mapdb", "leveldb", "memory").contains(storageType)) { + if (!Arrays.asList("mapdb", "leveldb", "sql", "memory").contains(storageType)) { return singleError( position, - "Value of property '" + key + "' must have storage type of \"leveldb\", \"mapdb\", or \"memory\""); + "Value of property '" + key + "' must have storage type of \"leveldb\", \"mapdb\", \"sql\" or \"memory\""); } return noErrors(); } diff --git a/src/test/java/net/consensys/orion/config/TomlConfigTest.java b/src/test/java/net/consensys/orion/config/TomlConfigTest.java index 6a03a408..bfc1fe14 100644 --- a/src/test/java/net/consensys/orion/config/TomlConfigTest.java +++ b/src/test/java/net/consensys/orion/config/TomlConfigTest.java @@ -92,7 +92,7 @@ void invalidConfigsThrowException() { ConfigException.class, () -> Config.load(this.getClass().getClassLoader().getResourceAsStream("invalidConfigTest.toml"))); String message = "Value of property 'clienturl' is not a valid URL (line 4, column 1)\n" - + "Value of property 'storage' must have storage type of \"leveldb\", \"mapdb\", or \"memory\" (line 11, column 1)\n" + + "Value of property 'storage' must have storage type of \"leveldb\", \"mapdb\", \"sql\" or \"memory\" (line 11, column 1)\n" + "Value of property 'othernodes' is not a valid URL (line 6, column 1)\n" + "Value of property 'othernodes' is not a valid URL (line 6, column 1)\n" + "Value of property 'tlsservertrust' should be \"whitelist\", \"ca\", \"ca-or-whitelist\", \"tofu\", \"insecure-tofa\", \"ca-or-tofu\", \"insecure-ca-or-tofa\", \"insecure-no-validation\", \"insecure-record\", or \"insecure-ca-or-record\" (line 9, column 1)\n" @@ -126,6 +126,7 @@ void storageValidationTypes() { Config.load("storage=\"memory\""); Config.load("storage=\"leveldb\""); Config.load("storage=\"mapdb\""); + Config.load("storage=\"sql:url\""); assertThrows(ConfigException.class, () -> Config.load("storage=\"memoryX\"")); assertThrows(ConfigException.class, () -> Config.load("storage=\"invalidStorage\"")); }