Limited threads resiliency fix durability nonblock (#573)

* Adding unit test case for record delivery validation

* Initial prototype for notification mechanism between ShardConsumerSubscriber and FanoutPublisher. The SDK Threads are made to block wait on the ack from the ShardConsumerSubscriber

* initial non blocking prototype

* Refactoring src and test

* Added unit test cases. Addressed review comments. Handled edge cases

* Minor code changes. Note that the previous commit has blocking impl of PrefetchPublisher

* Refactored the cleanup logic

* Fix for Cloudwatch exception handling and other revioew comment fixes

* Typo fix

* Removing cloudwatch fix. Will be released in a separate commit.

* Changing RejectedTaskEvent log message for the release

* Added javadoc to RecordsDeliveryAck and optimized imports

* Adding Kinesis Internal API tag for new concrete implementations

amazon-kinesis-client/src/main/java/software/amazon/kinesis/common/DiagnosticUtils.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.common;
+
+import org.slf4j.Logger;
+import software.amazon.kinesis.annotations.KinesisClientInternalApi;
+
+import java.time.Duration;
+import java.time.Instant;
+
+import static software.amazon.kinesis.lifecycle.ShardConsumer.MAX_TIME_BETWEEN_REQUEST_RESPONSE;
+
+@KinesisClientInternalApi
+public class DiagnosticUtils {
+
+    /**
+     * Util for RecordPublisher to measure the event delivery latency of the executor service and take appropriate action.
+     * @param shardId of the shard that is having delayed delivery
+     * @param enqueueTimestamp of the event submitted to the executor service
+     * @param log Slf4j Logger from RecordPublisher to log the events
+     */
+    public static void takeDelayedDeliveryActionIfRequired(String shardId, Instant enqueueTimestamp, Logger log) {
+        final long durationBetweenEnqueueAndAckInMillis = Duration
+                .between(enqueueTimestamp, Instant.now()).toMillis();
+        if (durationBetweenEnqueueAndAckInMillis > MAX_TIME_BETWEEN_REQUEST_RESPONSE / 3) {
+            // The above condition logs the warn msg if the delivery time exceeds 11 seconds.
+            log.warn(
+                    "{}: Record delivery time to shard consumer is high at {} millis. Check the ExecutorStateEvent logs"
+                            + " to see the state of the executor service. Also check if the RecordProcessor's processing "
+                            + "time is high. ",
+                    shardId, durationBetweenEnqueueAndAckInMillis);
+        } else if (log.isDebugEnabled()) {
+            log.debug("{}: Record delivery time to shard consumer is {} millis", shardId,
+                    durationBetweenEnqueueAndAckInMillis);
+        }
+    }
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/coordinator/RejectedTaskEvent.java

@@ -26,7 +26,7 @@
 @KinesisClientInternalApi
 class RejectedTaskEvent implements DiagnosticEvent {
     private static final String MESSAGE = "Review your thread configuration to prevent task rejections. " +
-            "Until next release, KCL will not be resilient to task rejections. ";
+            "Task rejections will slow down your application and some shards may stop processing. ";
 
     private ExecutorStateEvent executorStateEvent;
     private Throwable throwable;

amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/NotifyingSubscriber.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.lifecycle;
+
+import org.reactivestreams.Subscriber;
+import org.reactivestreams.Subscription;
+import software.amazon.kinesis.retrieval.RecordsPublisher;
+import software.amazon.kinesis.retrieval.RecordsRetrieved;
+import software.amazon.kinesis.retrieval.RecordsDeliveryAck;
+
+/**
+ * Subscriber that notifies its publisher on receipt of the onNext event.
+ */
+public interface NotifyingSubscriber extends Subscriber<RecordsRetrieved> {
+
+    /**
+     * Return the actual subscriber to which the events needs to be delegated.
+     * @return Subscriber<T> to be delegated
+     */
+    Subscriber<RecordsRetrieved> getDelegateSubscriber();
+
+    /**
+     * Return the publisher to be notified
+     * @return RecordsPublisher to be notified.
+     */
+    RecordsPublisher getRecordsPublisher();
+
+    /**
+     * Construct RecordsDeliveryAck object from the incoming data and return it
+     * @param recordsRetrieved for which we need the ack.
+     * @return getRecordsDeliveryAck
+     */
+    RecordsDeliveryAck getRecordsDeliveryAck(RecordsRetrieved recordsRetrieved);
+
+    @Override
+    default void onSubscribe(Subscription subscription) {
+        getDelegateSubscriber().onSubscribe(subscription);
+    }
+
+    @Override
+    default void onNext(RecordsRetrieved recordsRetrieved) {
+        getRecordsPublisher().notify(getRecordsDeliveryAck(recordsRetrieved));
+        getDelegateSubscriber().onNext(recordsRetrieved);
+    }
+
+    @Override
+    default void onError(Throwable throwable) {
+        getDelegateSubscriber().onError(throwable);
+    }
+
+    @Override
+    default void onComplete() {
+        getDelegateSubscriber().onComplete();
+    }
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumerNotifyingSubscriber.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.lifecycle;
+
+import lombok.AllArgsConstructor;
+import org.reactivestreams.Subscriber;
+import software.amazon.kinesis.annotations.KinesisClientInternalApi;
+import software.amazon.kinesis.retrieval.RecordsPublisher;
+import software.amazon.kinesis.retrieval.RecordsRetrieved;
+import software.amazon.kinesis.retrieval.RecordsDeliveryAck;
+
+@KinesisClientInternalApi
+@AllArgsConstructor
+public class ShardConsumerNotifyingSubscriber implements NotifyingSubscriber {
+
+    private final Subscriber<RecordsRetrieved> delegate;
+
+    private final RecordsPublisher recordsPublisher;
+
+    @Override
+    public Subscriber<RecordsRetrieved> getDelegateSubscriber() {
+        return delegate;
+    }
+
+    @Override
+    public RecordsPublisher getRecordsPublisher() {
+        return recordsPublisher;
+    }
+
+    @Override
+    public RecordsDeliveryAck getRecordsDeliveryAck(RecordsRetrieved recordsRetrieved) {
+        return () -> recordsRetrieved.batchUniqueIdentifier();
+    }
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/lifecycle/ShardConsumerSubscriber.java

@@ -82,7 +82,7 @@ void startSubscriptions() {
                 recordsPublisher.restartFrom(lastAccepted);
             }
             Flowable.fromPublisher(recordsPublisher).subscribeOn(scheduler).observeOn(scheduler, true, bufferSize)
-                    .subscribe(this);
+                    .subscribe(new ShardConsumerNotifyingSubscriber(this, recordsPublisher));
         }
     }
 
@@ -216,4 +216,5 @@ public void cancel() {
             subscription.cancel();
         }
     }
+
 }

amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/BatchUniqueIdentifier.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.retrieval;
+
+import lombok.Data;
+import software.amazon.kinesis.annotations.KinesisClientInternalApi;
+
+@KinesisClientInternalApi
+@Data
+public class BatchUniqueIdentifier {
+    private final String recordBatchIdentifier;
+    private final String flowIdentifier;
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/RecordsDeliveryAck.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package software.amazon.kinesis.retrieval;
+
+/**
+ * Interface to supply all the meta information for record delivery ack.
+ */
+public interface RecordsDeliveryAck {
+
+    /**
+     * Unique record batch identifier used to ensure the durability and ordering guarantees.
+     * @return id that uniquely determines a record batch and its source.
+     */
+    BatchUniqueIdentifier batchUniqueIdentifier();
+
+}

amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/RecordsPublisher.java

@@ -18,7 +18,6 @@
 import org.reactivestreams.Publisher;
 
 import software.amazon.kinesis.common.InitialPositionInStreamExtended;
-import software.amazon.kinesis.lifecycle.events.ProcessRecordsInput;
 import software.amazon.kinesis.retrieval.kpl.ExtendedSequenceNumber;
 
 /**
@@ -47,4 +46,12 @@ public interface RecordsPublisher extends Publisher<RecordsRetrieved> {
      * Shutdowns the publisher. Once this method returns the publisher should no longer provide any records.
      */
     void shutdown();
+
+    /**
+     * Notify the publisher on receipt of a data event.
+     * @param ack acknowledgement received from the subscriber.
+     */
+    default void notify(RecordsDeliveryAck ack) {
+        throw new UnsupportedOperationException("RecordsPublisher does not support acknowledgement from Subscriber");
+    }
 }

amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/RecordsRetrieved.java

@@ -24,4 +24,13 @@ public interface RecordsRetrieved {
      * @return the processRecordsInput received
      */
     ProcessRecordsInput processRecordsInput();
+
+    /**
+     * Returns the identifier that uniquely identifies this batch.
+     *
+     * @return batchUniqueIdentifier that uniquely identifies the records batch and its source.
+     */
+    default BatchUniqueIdentifier batchUniqueIdentifier() {
+        throw new UnsupportedOperationException("Retrieval of batch unique identifier is not supported");
+    }
 }