refactor(backend): Make parsing background task (#304)

* refactor(backend): Make parsing background task

* lint

* change test_api to fit with new import_from_github

* fix add_task to background

* delete useless lines

* fix(backend):tempfile created in background task

* refactor(backend):clean code

* fix(backend): parser works for upload

* refactor(backend,taxonomy-editor-frontend): Handle background task on frontend

* refactor(backend): Make parsing background task

* lint

* change test_api to fit with new import_from_github

* fix add_task to background

* delete useless lines

* fix(backend):tempfile created in background task

* refactor(backend):clean code

* fix(backend): parser works for upload

* refactor(backend):Simplify import and upload functions

* refactor(backend): lint

* refactor(backend): add an error node when parsing raises an exception

* fix(backend):create errors node in exception

* fix(backend): base_uri works and project status is set

* refactor(backend,taxonomy-editor-frontend): Handle background task on frontend

* refactor(backend): remove duplicated code with upload taxonomy

* refactor(backend): fix checks

* refactor(backend):add context manager to get taxonomy file

* feature(frontend): add warning message when taxonomy has parsing errors

---------

Co-authored-by: alice.juan <alice.juan@student-cs.fr>

backend/editor/api.py

@@ -3,8 +3,6 @@
 """
 import logging
 import os
-import shutil
-import tempfile
 
 # Required imports
 # ------------------------------------------------------------------------------------#
@@ -82,7 +80,7 @@ async def shutdown():
     """
     Shutdown database
     """
-    graph_db.shutdown_db()
+    await graph_db.shutdown_db()
 
 
 @app.middleware("http")
@@ -122,6 +120,8 @@ class StatusFilter(str, Enum):
 
     OPEN = "OPEN"
     CLOSED = "CLOSED"
+    LOADING = "LOADING"
+    FAILED = "FAILED"
 
 
 @app.exception_handler(RequestValidationError)
@@ -372,28 +372,35 @@ async def export_to_github(
 
 
 @app.post("/{taxonomy_name}/{branch}/import")
-async def import_from_github(request: Request, branch: str, taxonomy_name: str):
+async def import_from_github(
+    request: Request, branch: str, taxonomy_name: str, background_tasks: BackgroundTasks
+):
     """
     Get taxonomy from Product Opener GitHub repository
     """
     incoming_data = await request.json()
     description = incoming_data["description"]
 
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
+
     if not taxonomy.is_valid_branch_name():
         raise HTTPException(status_code=422, detail="branch_name: Enter a valid branch name!")
     if await taxonomy.does_project_exist():
         raise HTTPException(status_code=409, detail="Project already exists!")
     if not await taxonomy.is_branch_unique():
         raise HTTPException(status_code=409, detail="branch_name: Branch name should be unique!")
 
-    result = await taxonomy.import_from_github(description)
-    return result
+    status = await taxonomy.import_from_github(description, background_tasks)
+    return status
 
 
 @app.post("/{taxonomy_name}/{branch}/upload")
 async def upload_taxonomy(
-    branch: str, taxonomy_name: str, file: UploadFile, description: str = Form(...)
+    branch: str,
+    taxonomy_name: str,
+    file: UploadFile,
+    background_tasks: BackgroundTasks,
+    description: str = Form(...),
 ):
     """
     Upload taxonomy file to be parsed
@@ -406,11 +413,7 @@ async def upload_taxonomy(
     if not await taxonomy.is_branch_unique():
         raise HTTPException(status_code=409, detail="branch_name: Branch name should be unique!")
 
-    with tempfile.TemporaryDirectory(prefix="taxonomy-") as tmpdir:
-        filepath = f"{tmpdir}/{file.filename}"
-        with open(filepath, "wb") as f:
-            shutil.copyfileobj(file.file, f)
-        result = await taxonomy.upload_taxonomy(filepath, description)
+    result = await taxonomy.import_from_github(description, background_tasks, file)
 
     return result
 

backend/editor/entries.py

@@ -1,14 +1,18 @@
 """
 Database helper functions for API
 """
+import contextlib
 import re
+import shutil
 import tempfile
 import urllib.request  # Sending requests
 
+from fastapi import BackgroundTasks, UploadFile
 from openfoodfacts_taxonomy_parser import normalizer  # Normalizing tags
 from openfoodfacts_taxonomy_parser import parser  # Parser for taxonomies
 from openfoodfacts_taxonomy_parser import unparser  # Unparser for taxonomies
 
+from . import settings
 from .exceptions import GithubBranchExistsError  # Custom exceptions
 from .exceptions import (
     GithubUploadError,
@@ -82,61 +86,59 @@ async def create_node(self, label, entry, main_language_code):
         result = await get_current_transaction().run(" ".join(query), params)
         return (await result.data())[0]["n.id"]
 
-    async def parse_taxonomy(self, filename):
-        """
-        Helper function to call the Open Food Facts Python Taxonomy Parser
-        """
-        # Close current transaction to use the session variable in parser
-        await get_current_transaction().commit()
-
-        with SyncTransactionCtx() as session:
-            # Create parser object and pass current session to it
-            parser_object = parser.Parser(session)
-            try:
-                # Parse taxonomy with given file name and branch name
-                parser_object(filename, self.branch_name, self.taxonomy_name)
-                return True
-            except Exception as e:
-                raise TaxonomyParsingError() from e
+    @contextlib.contextmanager
+    def get_taxonomy_file(self, uploadfile=None):
+        if uploadfile is None:  # taxonomy is imported
+            base_url = (
+                "https://raw.githubusercontent.com/" + settings.repo_uri + "/main/taxonomies/"
+            )
+            filename = f"{self.taxonomy_name}.txt"
+            base_url += filename
+        else:  # taxonomy is uploaded
+            filename = uploadfile.filename
+
+        with tempfile.TemporaryDirectory(prefix="taxonomy-") as tmpdir:
+            filepath = f"{tmpdir}/{filename}"
+            if uploadfile is None:
+                # Downloads and creates taxonomy file in current working directory
+                urllib.request.urlretrieve(base_url, filepath)
+            else:
+                with open(filepath, "wb") as f:
+                    shutil.copyfileobj(uploadfile.file, f)
+            yield filepath
 
-    async def import_from_github(self, description):
+    def parse_taxonomy(self, uploadfile=None):
         """
-        Helper function to import a taxonomy from GitHub
+        Helper function to call the Open Food Facts Python Taxonomy Parser
         """
-        base_url = (
-            "https://raw.githubusercontent.com/openfoodfacts/openfoodfacts-server"
-            "/main/taxonomies/"
-        )
-        filename = self.taxonomy_name + ".txt"
-        base_url += filename
         try:
-            with tempfile.TemporaryDirectory(prefix="taxonomy-") as tmpdir:
-                # File to save the downloaded taxonomy
-                filepath = f"{tmpdir}/{filename}"
-
-                # Downloads and creates taxonomy file in current working directory
-                urllib.request.urlretrieve(base_url, filepath)
-
-                status = await self.parse_taxonomy(filepath)  # Parse the taxonomy
-
-            async with TransactionCtx():
-                await self.create_project(description)  # Creates a "project node" in neo4j
-
-            return status
+            with self.get_taxonomy_file(uploadfile) as filepath:
+                with SyncTransactionCtx() as session:
+                    # Create parser object and pass current session to it
+                    parser_object = parser.Parser(session)
+                    try:
+                        # Parse taxonomy with given file name and branch name
+                        parser_object(filepath, self.branch_name, self.taxonomy_name)
+                        self.set_project_status(session, status="OPEN")
+                        return True
+                    except Exception as e:
+                        # outer exception handler will put project status to FAILED
+                        raise TaxonomyParsingError() from e
         except Exception as e:
+            # add an error node so we can display it with errors in the app
+            parser_object.create_parsing_errors_node(self.taxonomy_name, self.branch_name)
+            self.set_project_status(session, status="FAILED")
             raise TaxonomyImportError() from e
 
-    async def upload_taxonomy(self, filepath, description):
+    async def import_from_github(
+        self, description, background_tasks: BackgroundTasks, uploadfile: UploadFile = None
+    ):
         """
-        Helper function to upload a taxonomy file and create a project node
+        Helper function to import a taxonomy from GitHub
         """
-        try:
-            status = await self.parse_taxonomy(filepath)
-            async with TransactionCtx():
-                await self.create_project(description)
-            return status
-        except Exception as e:
-            raise TaxonomyImportError() from e
+        await self.create_project(description)
+        background_tasks.add_task(self.parse_taxonomy, uploadfile)
+        return True
 
     def dump_taxonomy(self):
         """
@@ -169,9 +171,9 @@ async def github_export(self):
 
         filepath = self.dump_taxonomy()
         # Create a new transaction context
-        async with TransactionCtx():
+        async with TransactionCtx() as (_, session):
             result = await self.export_to_github(filepath)
-            await self.set_project_status(status="CLOSED")
+            self.set_project_status(session, status="CLOSED")
         return result
 
     async def export_to_github(self, filename):
@@ -244,11 +246,11 @@ async def create_project(self, description):
             "taxonomy_name": self.taxonomy_name,
             "branch_name": self.branch_name,
             "description": description,
-            "status": "OPEN",
+            "status": "LOADING",
         }
         await get_current_transaction().run(query, params)
 
-    async def set_project_status(self, status):
+    def set_project_status(self, session, status):
         """
         Helper function to update a Taxonomy Editor project status
         """
@@ -258,7 +260,8 @@ async def set_project_status(self, status):
             SET n.status = $status
         """
         params = {"project_name": self.project_name, "status": status}
-        await get_current_transaction().run(query, params)
+        with session.begin_transaction() as tx:
+            tx.run(query, params)
 
     async def list_projects(self, status=None):
         """

backend/editor/graph_db.py

@@ -50,11 +50,11 @@ def initialize_db():
     driver = neo4j.AsyncGraphDatabase.driver(uri)
 
 
-def shutdown_db():
+async def shutdown_db():
     """
     Close session and driver of Neo4J database
     """
-    driver.close()
+    await driver.close()
 
 
 def get_current_transaction():

taxonomy-editor-frontend/src/components/Alerts.tsx

@@ -0,0 +1,67 @@
+import React from "react";
+import Alert from "@mui/material/Alert";
+import AlertTitle from "@mui/material/AlertTitle";
+import useFetch from "./useFetch";
+
+interface CustomAlertProps {
+  severity: "error" | "warning" | "info" | "success";
+  title?: string;
+  message: string;
+}
+
+interface WarningParsingErrorsProps {
+  baseUrl: string;
+}
+
+interface ParsingErrorsType {
+  errors: string[];
+}
+
+const CustomAlert: React.FC<CustomAlertProps> = ({
+  severity,
+  title,
+  message,
+}) => {
+  return (
+    <Alert severity={severity}>
+      {title && <AlertTitle>{title}</AlertTitle>}
+      {message}
+    </Alert>
+  );
+};
+
+// warning users the taxonomy had parsing errors, so should not edit it
+export const WarningParsingErrors: React.FC<WarningParsingErrorsProps> = ({
+  baseUrl,
+}) => {
+  console.log("in WarningParsingErrors");
+  const { data: parsingErrors, isPending: isPendingParsingErrors } =
+    useFetch<ParsingErrorsType>(`${baseUrl}parsing_errors`);
+  if (!isPendingParsingErrors) {
+    return (
+      <>
+        {parsingErrors && parsingErrors.errors.length !== 0 && (
+          <CustomAlert
+            severity="warning"
+            title="Parsing errors"
+            message="This taxonomy has encountered parsing errors, preventing further editing. 
+            Please review the errors on the dedicated Errors page for resolution, ensuring the 
+            taxonomy can be edited once the issues are addressed."
+          />
+        )}
+      </>
+    );
+  } else {
+    return null;
+  }
+};
+
+export default WarningParsingErrors;
+
+<CustomAlert
+  severity="warning"
+  title="Parsing errors"
+  message="This taxonomy has encountered parsing errors, preventing further editing. 
+          Please review the errors on the dedicated error page for resolution, ensuring the 
+          taxonomy can be edited once the issues are addressed."
+/>;