editoast: core-client: add retry on broken pipes errors

OpenRailAssociation · Apr 23, 2024 · 8480f19 · 8480f19
1 parent b86407f
commit 8480f19
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 2 deletions.
diff --git a/editoast/openapi.yaml b/editoast/openapi.yaml
@@ -428,6 +428,25 @@ components:
       - status
       - message
       type: object
+    EditoastCoreErrorBrokenPipe:
+      properties:
+        context:
+          type: object
+        message:
+          type: string
+        status:
+          enum:
+          - 500
+          type: integer
+        type:
+          enum:
+          - editoast:coreclient:BrokenPipe
+          type: string
+      required:
+      - type
+      - status
+      - message
+      type: object
     EditoastCoreErrorCannotExtractResponseBody:
       properties:
         context:
@@ -671,6 +690,7 @@ components:
       - $ref: '#/components/schemas/EditoastAutoFixesEditoastErrorMissingErrorObject'
       - $ref: '#/components/schemas/EditoastCacheOperationErrorDuplicateIdsProvided'
       - $ref: '#/components/schemas/EditoastCacheOperationErrorObjectNotFound'
+      - $ref: '#/components/schemas/EditoastCoreErrorBrokenPipe'
       - $ref: '#/components/schemas/EditoastCoreErrorCannotExtractResponseBody'
       - $ref: '#/components/schemas/EditoastCoreErrorConnectionClosedBeforeMessageCompleted'
       - $ref: '#/components/schemas/EditoastCoreErrorConnectionResetByPeer'

diff --git a/editoast/src/core/mod.rs b/editoast/src/core/mod.rs
@@ -115,10 +115,12 @@ impl CoreClient {
                     match request.send().await.map_err(Into::<CoreError>::into) {
                         // This error occurs quite often in the CI.
                         // It's linked to this issue /~https://github.com/hyperium/hyper/issues/2136.
-                        // This is why we retry the request here
+                        // This is why we retry the request here.
+                        // We also retry on broken pipe.
                         Err(
                             CoreError::ConnectionResetByPeer
-                            | CoreError::ConnectionClosedBeforeMessageCompleted,
+                            | CoreError::ConnectionClosedBeforeMessageCompleted
+                            | CoreError::BrokenPipe,
                         ) if i_try < MAX_RETRIES => {
                             i_try += 1;
                             info!("Core request '{}: {}': Connection closed before message completed. Retry [{}/{}]", method, path, i_try, MAX_RETRIES);
@@ -314,6 +316,9 @@ enum CoreError {
     #[error("Core connection reset by peer. Should retry.")]
     #[editoast_error(status = 500)]
     ConnectionResetByPeer,
+    #[error("Core connection broken. Should retry.")]
+    #[editoast_error(status = 500)]
+    BrokenPipe,
 
     #[cfg(test)]
     #[error("The mocked response had no body configured - check out StubResponseBuilder::body if this is unexpected")]
@@ -364,6 +369,12 @@ impl From<reqwest::Error> for CoreError {
         if value.to_string().contains("Connection reset by peer") {
             return Self::ConnectionResetByPeer;
         }
+        if value
+            .to_string()
+            .contains("error writing a body to connection: Broken pipe")
+        {
+            return Self::BrokenPipe;
+        }
 
         // Convert the reqwest error
         Self::GenericCoreError {

diff --git a/front/public/locales/en/errors.json b/front/public/locales/en/errors.json
@@ -26,6 +26,7 @@
       "ObjectNotFound": "{{obj_type}} {{obj_id}} could not be found everywhere in the infrastructure cache"
     },
     "coreclient": {
+      "BrokenPipe": "Core connection broken pipe. Should retry.",
       "CannotExtractResponseBody": "Cannot extract Core response body: {{msg}}",
       "ConnectionClosedBeforeMessageCompleted": "Core connection closed before message completed. Should retry.",
       "ConnectionResetByPeer": "Core connection reset by peer. Should retry.",

diff --git a/front/public/locales/fr/errors.json b/front/public/locales/fr/errors.json
@@ -26,6 +26,7 @@
       "ObjectNotFound": "{{obj_type}} {{obj_id}} n'a pu être trouvé nulle part dans le cache de l'infrastructure"
     },
     "coreclient": {
+      "BrokenPipe": "Core: connexion interrompue. Nouvelle tentative.",
       "CannotExtractResponseBody": "Core: Impossible d'extraire le corps de la réponse : {{msg}}",
       "ConnectionClosedBeforeMessageCompleted": "Core: connexion fermée avant la fin du message. Nouvelle tentative.",
       "ConnectionResetByPeer": "Core: réinitialisation de la connexion. Nouvelle tentative.",