Adiciona pre-commit, corrige e formata código

.pre-commit-config.yaml

@@ -0,0 +1,30 @@
+repos:
+  - repo: /~https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    - id: check-case-conflict
+      fail_fast: true
+    - id: check-merge-conflict
+      fail_fast: true
+    - id: debug-statements
+      fail_fast: true
+    - id: detect-aws-credentials
+      fail_fast: true
+      args: ["--allow-missing-credentials"]
+    - id: detect-private-key
+      fail_fast: true
+
+  - repo: /~https://github.com/astral-sh/ruff-pre-commit
+    rev: 'v0.6.2'
+    hooks:
+    - id: ruff
+      fail_fast: true
+      args: ["--fix", "--ignore", "E501", "--select", "I"]
+    - id: ruff-format
+
+  - repo: /~https://github.com/Lucas-C/pre-commit-hooks-safety
+    rev: v1.3.3
+    hooks:
+    - id: python-safety-dependencies-check
+      fail_fast: true
+      files: requirements\/[a-z]+\.txt

data_extraction/__init__.py

@@ -1,2 +1,8 @@
 from .interfaces import TextExtractorInterface
 from .text_extraction import ApacheTikaTextExtractor, create_apache_tika_text_extraction
+
+__all__ = [
+    "ApacheTikaTextExtractor",
+    "create_apache_tika_text_extraction",
+    "TextExtractorInterface",
+]

data_extraction/interfaces.py

@@ -1,5 +1,6 @@
 import abc
 
+
 class TextExtractorInterface(abc.ABC):
     @abc.abstractmethod
     def extract_text(self, filepath: str) -> str:

data_extraction/text_extraction.py

@@ -1,8 +1,7 @@
 import logging
-import magic
 import os
-import subprocess
 
+import magic
 import requests
 
 from .interfaces import TextExtractorInterface

database/__init__.py

@@ -1,2 +1,8 @@
 from .interfaces import DatabaseInterface
 from .postgresql import PostgreSQL, create_database_interface
+
+__all__ = [
+    "create_database_interface",
+    "DatabaseInterface",
+    "PostgreSQL",
+]

database/interfaces.py

@@ -1,5 +1,5 @@
-from typing import Dict, Iterable, Tuple
 import abc
+from typing import Dict, Iterable, Tuple
 
 
 class DatabaseInterface(abc.ABC):

database/postgresql.py

@@ -1,6 +1,6 @@
-from typing import Dict, Iterable, Tuple
-import os
 import logging
+import os
+from typing import Dict, Iterable, Tuple
 
 import psycopg2
 
@@ -59,16 +59,16 @@ def select(self, command: str) -> Iterable[Tuple]:
             logging.debug(f"Finished query: {cursor.query}")
 
     def insert(self, command: str, data: Dict = {}):
-        logging.debug(f"Inserting:")
+        logging.debug("Inserting:")
         self._commit_changes(command, data)
-        logging.debug(f"Finished inserting")
+        logging.debug("Finished inserting")
 
     def update(self, command: str, data: Dict = {}):
-        logging.debug(f"Updating:")
+        logging.debug("Updating:")
         self._commit_changes(command, data)
-        logging.debug(f"Finished updating")
+        logging.debug("Finished updating")
 
     def delete(self, command: str, data: Dict = {}):
-        logging.debug(f"Deleting:")
+        logging.debug("Deleting:")
         self._commit_changes(command, data)
-        logging.debug(f"Finished deleting")
+        logging.debug("Finished deleting")

index/__init__.py

@@ -1,2 +1,7 @@
 from .interfaces import IndexInterface
 from .opensearch import create_index_interface
+
+__all__ = [
+    "create_index_interface",
+    "IndexInterface",
+]

index/interfaces.py

@@ -1,5 +1,5 @@
-from typing import Dict, Iterable
 import abc
+from typing import Dict, Iterable
 
 
 class IndexInterface(abc.ABC):
@@ -40,5 +40,3 @@ def paginated_search(
         """
         Searches the index with the provided query, with pagination
         """
-
-

index/opensearch.py

@@ -1,14 +1,23 @@
-from typing import Dict, Iterable, List, Union
 import os
+from typing import Dict, Iterable, List, Union
 
 import opensearchpy
 
 from .interfaces import IndexInterface
 
 
 class OpenSearchInterface(IndexInterface):
-    def __init__(self, hosts: List, user: str, password: str, timeout: int = 30, default_index: str = ""):
-        self._search_engine = opensearchpy.OpenSearch(hosts=hosts, http_auth=(user, password))
+    def __init__(
+        self,
+        hosts: List,
+        user: str,
+        password: str,
+        timeout: int = 30,
+        default_index: str = "",
+    ):
+        self._search_engine = opensearchpy.OpenSearch(
+            hosts=hosts, http_auth=(user, password)
+        )
         self._timeout = timeout
         self._default_index = default_index
 
@@ -51,7 +60,9 @@ def index_document(
         refresh: bool = False,
     ) -> None:
         index = self.get_index_name(index)
-        self._search_engine.index(index=index, body=document, id=document_id, refresh=refresh)
+        self._search_engine.index(
+            index=index, body=document, id=document_id, refresh=refresh
+        )
 
     def search(self, query: Dict, index: str = "") -> Dict:
         index = self.get_index_name(index)
@@ -60,7 +71,9 @@ def search(self, query: Dict, index: str = "") -> Dict:
 
     def analyze(self, text: str, field: str, index: str = "") -> Dict:
         index = self.get_index_name(index)
-        result = self._search_engine.indices.analyze(body={"text": text, "field":field}, index=index)
+        result = self._search_engine.indices.analyze(
+            body={"text": text, "field": field}, index=index
+        )
         return result
 
     def paginated_search(
@@ -96,17 +109,25 @@ def get_opensearch_host():
 def get_opensearch_index():
     return os.environ["OPENSEARCH_INDEX"]
 
+
 def get_opensearch_user():
     return os.environ["OPENSEARCH_USER"]
 
+
 def get_opensearch_password():
     return os.environ["OPENSEARCH_PASSWORD"]
 
+
 def create_index_interface() -> IndexInterface:
     hosts = get_opensearch_host()
     if not isinstance(hosts, str) or len(hosts) == 0:
         raise Exception("Missing index hosts")
     default_index_name = get_opensearch_index()
     if not isinstance(default_index_name, str) or len(default_index_name) == 0:
         raise Exception("Invalid index name")
-    return OpenSearchInterface([hosts], get_opensearch_user(), get_opensearch_password(), default_index=default_index_name)
+    return OpenSearchInterface(
+        [hosts],
+        get_opensearch_user(),
+        get_opensearch_password(),
+        default_index=default_index_name,
+    )

main/__init__.py

@@ -1,4 +1,9 @@
 from .__main__ import (
-    is_debug_enabled,
     enable_debug_if_necessary,
+    is_debug_enabled,
 )
+
+__all__ = [
+    "is_debug_enabled",
+    "enable_debug_if_necessary",
+]

main/__main__.py

@@ -1,11 +1,11 @@
-from os import environ
 import argparse
 import logging
+from os import environ
 
 from data_extraction import create_apache_tika_text_extraction
 from database import create_database_interface
-from storage import create_storage_interface
 from index import create_index_interface
+from storage import create_storage_interface
 from tasks import run_task
 
 
@@ -37,12 +37,24 @@ def gazette_texts_pipeline():
 
     run_task("create_gazettes_index", index)
     territories = run_task("get_territories", database)
-    gazettes_to_be_processed = run_task("get_gazettes_to_be_processed", execution_mode, database)
-    indexed_gazette_ids = run_task("extract_text_from_gazettes", gazettes_to_be_processed, territories, database, storage, index, text_extractor)
-
+    gazettes_to_be_processed = run_task(
+        "get_gazettes_to_be_processed", execution_mode, database
+    )
+    indexed_gazette_ids = run_task(
+        "extract_text_from_gazettes",
+        gazettes_to_be_processed,
+        territories,
+        database,
+        storage,
+        index,
+        text_extractor,
+    )
+
     for theme in themes:
         run_task("create_themed_excerpts_index", theme, index)
-        themed_excerpt_ids = run_task("extract_themed_excerpts_from_gazettes", theme, indexed_gazette_ids, index)
+        themed_excerpt_ids = run_task(
+            "extract_themed_excerpts_from_gazettes", theme, indexed_gazette_ids, index
+        )
         run_task("embedding_rerank_excerpts", theme, themed_excerpt_ids, index)
         run_task("tag_entities_in_excerpts", theme, themed_excerpt_ids, index)
 

requirements-dev.txt

@@ -0,0 +1 @@
+pre-commit==3.5.0

segmentation/__init__.py

@@ -2,4 +2,4 @@
 
 __all__ = [
     "get_segmenter",
-]
+]

segmentation/base/__init__.py

@@ -1,7 +1,7 @@
-from .gazette_segment import GazetteSegment
 from .association_segmenter import AssociationSegmenter
+from .gazette_segment import GazetteSegment
 
 __all__ = [
     "GazetteSegment",
     "AssociationSegmenter",
-]
+]

segmentation/base/association_segmenter.py

@@ -1,18 +1,23 @@
 from typing import Any, Dict, Iterable, List, Union
+
 from segmentation.base import GazetteSegment
 
 
 class AssociationSegmenter:
     def __init__(self, territories: Iterable[Dict[str, Any]]):
         self.territories = territories
 
-    def get_gazette_segments(self, *args, **kwargs) -> List[Union[GazetteSegment, Dict]]:
+    def get_gazette_segments(
+        self, *args, **kwargs
+    ) -> List[Union[GazetteSegment, Dict]]:
         """
         Returns a list of GazetteSegment
         """
         raise NotImplementedError
 
-    def split_text_by_territory(self, *args, **kwargs) -> Union[Dict[str, str], List[str]]:
+    def split_text_by_territory(
+        self, *args, **kwargs
+    ) -> Union[Dict[str, str], List[str]]:
         """
         Segment a association text by territory
         and returns a list of text segments
@@ -24,4 +29,3 @@ def build_segment(self, *args, **kwargs) -> GazetteSegment:
         Returns a GazetteSegment
         """
         raise NotImplementedError
-

segmentation/base/gazette_segment.py

@@ -1,5 +1,5 @@
-from datetime import date, datetime
 from dataclasses import dataclass
+from datetime import date, datetime
 
 
 @dataclass
@@ -8,6 +8,7 @@ class GazetteSegment:
     Dataclass to represent a gazette segment of a association
     related to a city
     """
+
     id: str
     territory_name: str
     source_text: str
@@ -24,4 +25,4 @@ class GazetteSegment:
     state_code: str
     territory_id: str
     file_raw_txt: str
-    url: str
+    url: str

segmentation/factory.py

@@ -1,13 +1,14 @@
 from typing import Any, Dict, Iterable
 
-from segmentation.base import AssociationSegmenter
 from segmentation import segmenters
-
+from segmentation.base import AssociationSegmenter
 
 _segmenter_instances = {}
 
 
-def get_segmenter(territory_id: str, territories: Iterable[Dict[str, Any]]) -> AssociationSegmenter:
+def get_segmenter(
+    territory_id: str, territories: Iterable[Dict[str, Any]]
+) -> AssociationSegmenter:
     """
     Factory method to return a AssociationSegmenter
 

segmentation/segmenters/__init__.py

@@ -2,4 +2,4 @@
 
 __all__ = [
     "ALAssociacaoMunicipiosSegmenter",
-]
+]

segmentation/segmenters/al_associacao_municipios.py

@@ -1,7 +1,7 @@
-import re
 import logging
-
+import re
 from typing import Any, Dict, List
+
 from segmentation.base import AssociationSegmenter, GazetteSegment
 from tasks.utils import batched, get_checksum, get_territory_data, get_territory_slug
 
@@ -64,15 +64,17 @@ def build_segment(
         )
         territory_data = get_territory_data(territory_slug, self.territories)
 
-        return GazetteSegment(**{
-            **gazette,
-            # segment specific values
-            "processed": True,
-            "file_checksum": get_checksum(segment_text),
-            "source_text": segment_text.strip(),
-            "territory_name": territory_data["territory_name"],
-            "territory_id": territory_data["id"],
-        })
+        return GazetteSegment(
+            **{
+                **gazette,
+                # segment specific values
+                "processed": True,
+                "file_checksum": get_checksum(segment_text),
+                "source_text": segment_text.strip(),
+                "territory_name": territory_data["territory_name"],
+                "territory_id": territory_data["id"],
+            }
+        )
 
     def _normalize_territory_name(self, territory_name: str) -> str:
         clean_name = territory_name.strip().replace("\n", "")

storage/__init__.py

@@ -1,2 +1,8 @@
 from .digital_ocean_spaces import DigitalOceanSpaces, create_storage_interface
 from .interfaces import StorageInterface
+
+__all__ = [
+    "create_storage_interface",
+    "DigitalOceanSpaces",
+    "StorageInterface",
+]