Skip to content

Commit

Permalink
Fix timeout in monitor (0.119) (#9871)
Browse files Browse the repository at this point in the history 
Fix timeout in monitor (#9870)

* Add a `hedera.mirror.monitor.nodes.certHash` property
* Fix TLS connectivity by setting `client.setNetworkFromAddressBook()` using address book created from `/api/v1/network/nodes`

---------

Signed-off-by: Steven Sheehy <steven.sheehy@swirldslabs.com>
  • Loading branch information
@steven-sheehy
steven-sheehy authored Dec 5, 2024
1 parent 00412fd commit 653bafe
Show file tree
Hide file tree
Showing 7 changed files with 81 additions and 18 deletions.
8 changes: 5 additions & 3 deletions docs/configuration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,9 +427,11 @@ monitor.
| `hedera.mirror.monitor.mirrorNode.rest.host` | "" | The hostname of the mirror node's REST API |
| `hedera.mirror.monitor.mirrorNode.rest.port` | 443 | The port of the mirror node's REST API |
| `hedera.mirror.monitor.network` | TESTNET | Which network to connect to. Automatically populates the main node & mirror node endpoints. Can be `MAINNET`, `PREVIEWNET`, `TESTNET` or `OTHER` |
| `hedera.mirror.monitor.nodes[].accountId` | "" | The main node's account ID |
| `hedera.mirror.monitor.nodes[].host` | "" | The main node's hostname |
| `hedera.mirror.monitor.nodes[].port` | 50211 | The main node's port |
| `hedera.mirror.monitor.nodes[].accountId` | "" | The consensus node's account ID |
| `hedera.mirror.monitor.nodes[].certHash` | "" | The consensus node's certificate hash used for TLS certificate verification |
| `hedera.mirror.monitor.nodes[].host` | "" | The consensus node's hostname |
| `hedera.mirror.monitor.nodes[].nodeId` | "" | The consensus node's node ID |
| `hedera.mirror.monitor.nodes[].port` | 50211 | The consensus node's port |
| `hedera.mirror.monitor.nodeValidation.enabled` | true | Whether to validate and remove invalid or down nodes permanently before publishing |
| `hedera.mirror.monitor.nodeValidation.frequency` | 1d | The amount of time between validations of the network. If not specified, millisecond is implied as the unit. |
| `hedera.mirror.monitor.nodeValidation.maxAttempts` | 8 | The number of times the monitor should attempt to receive a healthy response from a node before marking it as unhealthy. |
Expand Down
2 changes: 1 addition & 1 deletion docs/database/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ the saved snapshot before retry.

Some tables may contain errata information to workaround known issues with the stream files. The state of the consensus
nodes was never impacted, only the externalization of these changes to the stream files that the mirror node consumes.
There were three instances of bugs in the node software that misrepresented the side-effects of certain user
The below scenarios are considered bugs in the node software that misrepresented the side-effects of certain user
transactions in the balance and record streams. These issues should only appear in mainnet.

### Account Balance File Skew
Expand Down
35 changes: 35 additions & 0 deletions hedera-mirror-monitor/src/main/java/com/hedera/mirror/monitor/NodeProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,22 @@
package com.hedera.mirror.monitor;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.net.InetAddresses;
import com.google.protobuf.ByteString;
import com.hedera.hashgraph.sdk.AccountId;
import com.hedera.hashgraph.sdk.proto.AccountID;
import com.hedera.hashgraph.sdk.proto.NodeAddress;
import com.hedera.hashgraph.sdk.proto.ServiceEndpoint;
import jakarta.validation.constraints.Max;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.NotBlank;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.List;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.SneakyThrows;
import lombok.ToString;
import org.springframework.validation.annotation.Validated;

Expand All @@ -41,6 +49,8 @@ public class NodeProperties {
@ToString.Exclude
private final List<AccountId> accountIds = List.of(AccountId.fromString(getAccountId()));

private String certHash;

@NotBlank
private String host;

Expand Down Expand Up @@ -70,4 +80,29 @@ public long getNodeId() {
}
return nodeId;
}

@SneakyThrows
public NodeAddress toNodeAddress() {
var ipAddressV4 = toIpAddressV4();
var nodeAccountId = getAccountIds().get(0);

return NodeAddress.newBuilder()
.setNodeCertHash(certHash != null ? ByteString.copyFromUtf8(certHash) : ByteString.EMPTY)
.setNodeAccountId(AccountID.parseFrom(nodeAccountId.toBytes()))
.setNodeId(nodeId)
.addServiceEndpoint(ServiceEndpoint.newBuilder()
.setDomainName(ipAddressV4.isEmpty() ? host : "")
.setIpAddressV4(ipAddressV4)
.setPort(port))
.build();
}

private ByteString toIpAddressV4() throws UnknownHostException {
if (!InetAddresses.isInetAddress(host)) {
return ByteString.EMPTY;
}

var address = InetAddress.getByName(host).getAddress();
return ByteString.copyFrom(address);
}
}
24 changes: 17 additions & 7 deletions hedera-mirror-monitor/src/main/java/com/hedera/mirror/monitor/publish/NodeSupplier.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@
import com.hedera.hashgraph.sdk.PrivateKey;
import com.hedera.hashgraph.sdk.Status;
import com.hedera.hashgraph.sdk.TransferTransaction;
import com.hedera.hashgraph.sdk.proto.NodeAddressBook;
import com.hedera.mirror.monitor.MonitorProperties;
import com.hedera.mirror.monitor.NodeProperties;
import com.hedera.mirror.monitor.subscribe.rest.RestApiClient;
import com.hedera.mirror.rest.model.NetworkNode;
import io.micrometer.common.util.StringUtils;
import jakarta.annotation.PostConstruct;
import jakarta.inject.Named;
import java.time.Duration;
Expand All @@ -40,6 +40,8 @@
import java.util.concurrent.atomic.AtomicLong;
import lombok.CustomLog;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
Expand Down Expand Up @@ -139,21 +141,29 @@ private Flux<NodeProperties> toNodeProperties(NetworkNode networkNode) {
: serviceEndpoint.getIpAddressV4();
var nodeProperties = new NodeProperties();
nodeProperties.setAccountId(networkNode.getNodeAccountId());
nodeProperties.setCertHash(StringUtils.remove(networkNode.getNodeCertHash(), "0x"));
nodeProperties.setHost(host);
nodeProperties.setNodeId(networkNode.getNodeId());
nodeProperties.setPort(serviceEndpoint.getPort());
return nodeProperties;
}));
}

private Client toClient(Map<String, AccountId> nodes) {
AccountId operatorId =
AccountId.fromString(monitorProperties.getOperator().getAccountId());
PrivateKey operatorPrivateKey =
@SneakyThrows
private Client toClient(NodeProperties node) {
var operatorId = AccountId.fromString(monitorProperties.getOperator().getAccountId());
var operatorPrivateKey =
PrivateKey.fromString(monitorProperties.getOperator().getPrivateKey());
var validationProperties = monitorProperties.getNodeValidation();

Client client = Client.forNetwork(nodes);
var network = Map.of(node.getEndpoint(), AccountId.fromString(node.getAccountId()));
var nodeAddress = node.toNodeAddress();
var nodeAddressBook =
NodeAddressBook.newBuilder().addNodeAddress(nodeAddress).build().toByteString();

var client = Client.forNetwork(Map.of());
client.setNetworkFromAddressBook(com.hedera.hashgraph.sdk.NodeAddressBook.fromBytes(nodeAddressBook));
client.setNetwork(network);
client.setMaxAttempts(validationProperties.getMaxAttempts());
client.setMaxBackoff(validationProperties.getMaxBackoff());
client.setMinBackoff(validationProperties.getMinBackoff());
Expand All @@ -174,7 +184,7 @@ boolean validateNode(NodeProperties node) {
Hbar hbar = Hbar.fromTinybars(1L);
AccountId nodeAccountId = AccountId.fromString(node.getAccountId());

try (Client client = toClient(Map.of(node.getEndpoint(), nodeAccountId))) {
try (Client client = toClient(node)) {
Status receiptStatus = new TransferTransaction()
.addHbarTransfer(nodeAccountId, hbar)
.addHbarTransfer(client.getOperatorAccountId(), hbar.negated())
Expand Down
27 changes: 20 additions & 7 deletions ...-mirror-monitor/src/main/java/com/hedera/mirror/monitor/publish/TransactionPublisher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,18 @@
import com.hedera.hashgraph.sdk.TransactionReceiptQuery;
import com.hedera.hashgraph.sdk.TransactionRecordQuery;
import com.hedera.hashgraph.sdk.TransactionResponse;
import com.hedera.hashgraph.sdk.proto.NodeAddressBook;
import com.hedera.mirror.monitor.MonitorProperties;
import com.hedera.mirror.monitor.NodeProperties;
import jakarta.inject.Named;
import java.security.SecureRandom;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.CustomLog;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;

Expand Down Expand Up @@ -141,20 +144,30 @@ private <T> Mono<T> execute(Client client, Query<T, ?> executable) {
}

private Flux<Client> getClients() {
return nodeSupplier
.refresh()
.collect(Collectors.toMap(NodeProperties::getEndpoint, p -> AccountId.fromString(p.getAccountId())))
.flatMapMany(nodeMap -> Flux.range(0, publishProperties.getClients())
.flatMap(i -> Mono.defer(() -> Mono.just(toClient(nodeMap)))));
return nodeSupplier.refresh().collect(Collectors.toList()).flatMapMany(nodes -> Flux.range(
0, publishProperties.getClients())
.flatMap(i -> Mono.defer(() -> Mono.just(toClient(nodes)))));
}

private Client toClient(Map<String, AccountId> nodes) {
@SneakyThrows
private Client toClient(List<NodeProperties> nodes) {
AccountId operatorId =
AccountId.fromString(monitorProperties.getOperator().getAccountId());
PrivateKey operatorPrivateKey =
PrivateKey.fromString(monitorProperties.getOperator().getPrivateKey());

Client client = Client.forNetwork(nodes);
// setNetworkFromAddressBook() doesn't support in-process URIs so we have to set network too
var network = nodes.stream()
.collect(Collectors.toMap(NodeProperties::getEndpoint, p -> AccountId.fromString(p.getAccountId())));
var nodeAddresses = nodes.stream().map(NodeProperties::toNodeAddress).toList();
var nodeAddressBook = NodeAddressBook.newBuilder()
.addAllNodeAddress(nodeAddresses)
.build()
.toByteString();

var client = Client.forNetwork(Map.of());
client.setNetworkFromAddressBook(com.hedera.hashgraph.sdk.NodeAddressBook.fromBytes(nodeAddressBook));
client.setNetwork(network);
client.setNodeMaxBackoff(publishProperties.getNodeMaxBackoff());
client.setOperator(operatorId, operatorPrivateKey);
return client;
Expand Down
2 changes: 2 additions & 0 deletions hedera-mirror-monitor/src/test/java/com/hedera/mirror/monitor/publish/NodeSupplierTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ class NodeSupplierTest {
@BeforeEach
void setup() throws IOException {
node = new NodeProperties("0.0.3", "in-process:" + SERVER);
node.setNodeId(0L);
networkNode = new NetworkNode();
networkNode.setNodeAccountId(node.getAccountId());
networkNode.addServiceEndpointsItem(
Expand Down Expand Up @@ -290,6 +291,7 @@ void someValidNodes() throws Exception {

try {
var node2 = new NodeProperties("0.0.4", "in-process:" + server3);
node2.setNodeId(1L);
monitorProperties.setNodes(Set.of(node, node2));

// Validate good node
Expand Down
1 change: 1 addition & 0 deletions ...ror-monitor/src/test/java/com/hedera/mirror/monitor/publish/TransactionPublisherTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ void setup() throws IOException {
publishScenarioProperties.setName("test");
publishScenarioProperties.setType(TransactionType.CRYPTO_TRANSFER);
var node = new NodeProperties("0.0.3", "in-process:" + SERVER);
node.setNodeId(0L);
monitorProperties = new MonitorProperties();
monitorProperties.setNodes(Set.of(node));
monitorProperties.getNodeValidation().setEnabled(false);
Expand Down

0 comments on commit 653bafe

Please sign in to comment.