feat: Delete projects when their Github PR is merged (#385)

* feat: add scheduled project cleean up task

* style: run lint

* test: update tests

* chore: bump githubkit

* refactor: run delete_merged_project concurrently

* chore: add comments

backend/editor/api.py

@@ -1,6 +1,7 @@
 """
 Taxonomy Editor Backend API
 """
+import contextlib
 import logging
 
 # Required imports
@@ -29,7 +30,7 @@
 from . import graph_db
 
 # Controller imports
-from .controllers.project_controller import edit_project
+from .controllers import project_controller
 from .entries import TaxonomyGraph
 
 # Custom exceptions
@@ -38,6 +39,7 @@
 # Data model imports
 from .models.node_models import Footer, Header
 from .models.project_models import ProjectEdit, ProjectStatus
+from .scheduler import scheduler_lifespan
 
 # -----------------------------------------------------------------------------------#
 
@@ -49,7 +51,16 @@
 
 log = logging.getLogger(__name__)
 
-app = FastAPI(title="Open Food Facts Taxonomy Editor API")
+
+# Setup FastAPI app lifespan
+@contextlib.asynccontextmanager
+async def app_lifespan(app: FastAPI):
+    async with graph_db.database_lifespan():
+        with scheduler_lifespan():
+            yield
+
+
+app = FastAPI(title="Open Food Facts Taxonomy Editor API", lifespan=app_lifespan)
 
 # Allow anyone to call the API from their own apps
 app.add_middleware(
@@ -69,22 +80,6 @@
 )
 
 
-@app.on_event("startup")
-async def startup():
-    """
-    Initialize database
-    """
-    graph_db.initialize_db()
-
-
-@app.on_event("shutdown")
-async def shutdown():
-    """
-    Shutdown database
-    """
-    await graph_db.shutdown_db()
-
-
 @app.middleware("http")
 async def initialize_neo4j_transactions(request: Request, call_next):
     async with graph_db.TransactionCtx():
@@ -167,7 +162,9 @@ async def set_project_status(
     Set the status of a Taxonomy Editor project
     """
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
-    result = await edit_project(taxonomy.project_name, ProjectEdit(status=status))
+    result = await project_controller.edit_project(
+        taxonomy.project_name, ProjectEdit(status=status)
+    )
     return result
 
 
@@ -505,11 +502,10 @@ async def delete_node(request: Request, branch: str, taxonomy_name: str):
     await taxonomy.delete_node(taxonomy.get_label(id), id)
 
 
-@app.delete("/{taxonomy_name}/{branch}/delete")
-async def delete_project(response: Response, branch: str, taxonomy_name: str):
+@app.delete("/{taxonomy_name}/{branch}", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_project(branch: str, taxonomy_name: str):
     """
     Delete a project
     """
     taxonomy = TaxonomyGraph(branch, taxonomy_name)
-    result_data = await taxonomy.delete_taxonomy_project(branch, taxonomy_name)
-    return {"message": "Deleted {} projects".format(result_data)}
+    await project_controller.delete_project(taxonomy.project_name)

backend/editor/controllers/node_controller.py

@@ -0,0 +1,14 @@
+from ..graph_db import get_current_transaction
+
+
+async def delete_project_nodes(project_id: str):
+    """
+    Remove all nodes for project.
+    This includes entries, stopwords, synonyms and errors
+    """
+
+    query = f"""
+    MATCH (n:{project_id})
+    DETACH DELETE n
+    """
+    await get_current_transaction().run(query)

backend/editor/controllers/project_controller.py

@@ -1,5 +1,6 @@
 from ..graph_db import get_current_transaction
-from ..models.project_models import Project, ProjectCreate, ProjectEdit
+from ..models.project_models import Project, ProjectCreate, ProjectEdit, ProjectStatus
+from .node_controller import delete_project_nodes
 
 
 async def get_project(project_id: str) -> Project:
@@ -15,6 +16,19 @@ async def get_project(project_id: str) -> Project:
     return Project(**(await result.single())["p"])
 
 
+async def get_projects_by_status(status: ProjectStatus) -> list[Project]:
+    """
+    Get projects by status
+    """
+    query = """
+    MATCH (p:PROJECT {status: $status})
+    RETURN p
+    """
+    params = {"status": status}
+    result = await get_current_transaction().run(query, params)
+    return [Project(**record["p"]) async for record in result]
+
+
 async def create_project(project: ProjectCreate):
     """
     Create project
@@ -39,3 +53,16 @@ async def edit_project(project_id: str, project_edit: ProjectEdit):
         "project_edit": project_edit.model_dump(exclude_unset=True),
     }
     await get_current_transaction().run(query, params)
+
+
+async def delete_project(project_id: str):
+    """
+    Delete project, its nodes and relationships
+    """
+    query = """
+    MATCH (p:PROJECT {id: $project_id})
+    DETACH DELETE p
+    """
+    params = {"project_id": project_id}
+    await get_current_transaction().run(query, params)
+    await delete_project_nodes(project_id)

backend/editor/entries.py

@@ -686,19 +686,3 @@ async def full_text_search(self, text):
         _result = await get_current_transaction().run(query, params)
         result = [record["node"] for record in await _result.data()]
         return result
-
-    async def delete_taxonomy_project(self, branch, taxonomy_name):
-        """
-        Delete taxonomy projects
-        """
-
-        delete_query = """
-            MATCH (n:PROJECT {taxonomy_name: $taxonomy_name, branch_name: $branch_name})
-            DELETE n
-        """
-        result = await get_current_transaction().run(
-            delete_query, taxonomy_name=taxonomy_name, branch_name=branch
-        )
-        summary = await result.consume()
-        count = summary.counters.nodes_deleted
-        return count

backend/editor/github_functions.py

@@ -1,6 +1,7 @@
 """
 Github helper functions for the Taxonomy Editor API
 """
+
 import base64
 from functools import cached_property
 from textwrap import dedent
@@ -135,3 +136,19 @@ async def create_pr(self, description) -> PullRequest:
                 *self.repo_info, title=title, body=body, head=self.branch_name, base="main"
             )
         ).parsed_data
+
+    async def is_pr_merged(self, pr_number: int) -> bool:
+        """
+        Check if a pull request is merged
+        """
+        try:
+            await self.connection.rest.pulls.async_check_if_merged(
+                *self.repo_info, pull_number=pr_number
+            )
+            return True
+        except RequestFailed as e:
+            # The API returns 404 if pull request has not been merged
+            if e.response.status_code == 404:
+                return False
+            # re-raise in case of unexpected status code
+            raise e

backend/editor/graph_db.py

@@ -41,20 +41,18 @@ async def TransactionCtx():
         session.set(None)
 
 
-def initialize_db():
+@contextlib.asynccontextmanager
+async def database_lifespan():
     """
-    Initialize Neo4J database
+    Context manager for Neo4J database
     """
     global driver
     uri = settings.uri
     driver = neo4j.AsyncGraphDatabase.driver(uri)
-
-
-async def shutdown_db():
-    """
-    Close session and driver of Neo4J database
-    """
-    await driver.close()
+    try:
+        yield
+    finally:
+        await driver.close()
 
 
 def get_current_transaction():

backend/editor/scheduler.py

@@ -0,0 +1,44 @@
+import asyncio
+import contextlib
+import logging
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
+from .controllers.project_controller import delete_project, get_projects_by_status
+from .github_functions import GithubOperations
+from .graph_db import TransactionCtx
+from .models.project_models import Project, ProjectStatus
+
+log = logging.getLogger(__name__)
+
+
+async def delete_merged_projects():
+    async with TransactionCtx():
+        exported_projects = await get_projects_by_status(ProjectStatus.EXPORTED)
+        results = await asyncio.gather(
+            *map(delete_merged_project, exported_projects), return_exceptions=True
+        )
+        for exception_result in filter(lambda x: x is not None, results):
+            log.warn(exception_result)
+
+
+async def delete_merged_project(exported_project: Project):
+    pr_number = exported_project.github_pr_url and exported_project.github_pr_url.rsplit("/", 1)[-1]
+    if not pr_number:
+        log.warning(f"PR number not found for project {exported_project.id}")
+        return
+
+    github_object = GithubOperations(exported_project.taxonomy_name, exported_project.branch_name)
+    if await github_object.is_pr_merged(int(pr_number)):
+        await delete_project(exported_project.id)
+
+
+@contextlib.contextmanager
+def scheduler_lifespan():
+    scheduler = AsyncIOScheduler()
+    try:
+        scheduler.add_job(delete_merged_projects, "interval", hours=24)
+        scheduler.start()
+        yield
+    finally:
+        scheduler.shutdown()

backend/pyproject.toml

@@ -49,6 +49,7 @@ uvicorn = { extras = ["standard"], version = "^0.23.2" }
 neo4j = "^5.14.0"
 openfoodfacts_taxonomy_parser = { path = "../parser", develop = true }
 python-multipart = "^0.0.6"
+apscheduler = "^3.10.4"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.10.1"