Implement storage- and relationship-aware cleanup

[Swatinem]

This implements a generic mechanism to resolve the model relationship graph based on Django metadata. That graph is then sorted in topological order, and all models are being deleted. Special manual deletion code is written for all the models that have some files in storage, in order to make sure that files are also being cleaned up properly.

---

This is work towards codecov/engineering-team#1127

[codecov-notifications]

❌ 1 Tests Failed:

Tests completed	Failed	Passed	Skipped
1779	1	1778	4

View the top 1 failed tests by shortest run time

tasks/tests/unit/test_flush_repo.py::test_flush_repo_little_bit_of_everything

Stack Traces | 0.132s run time

self = <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>
sql = 'INSERT INTO "branches" ("branch", "repoid", "authors", "head", "base", "updatestamp") VALUES (%s, %s, %s::integer[], %s, %s, %s)'
params = ('President.', 5, None, '0ea5733d0b9d5754f6b06a2f0d5882e1e3c48cf3', None, datetime.datetime(2025, 1, 17, 12, 22, 7, 500648, tzinfo=datetime.timezone.utc))
ignored_wrapper_args = (False, {'connection': <DatabaseWrapper vendor='postgresql' alias='default'>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>})

    def _execute(self, sql, params, *ignored_wrapper_args):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            if params is None:
                # params default might be backend specific.
                return self.cursor.execute(sql)
            else:
>               return self.cursor.execute(sql, params)
E               psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "branches_repoid_branch"
E               DETAIL:  Key (repoid, branch)=(5, President.) already exists.

.../local/lib/python3.13.../db/backends/utils.py:89: UniqueViolation

The above exception was the direct cause of the following exception:

mocker = <pytest_mock.plugin.MockFixture object at 0x7f36181422d0>
mock_storage = <shared.storage.memory.MemoryStorageService object at 0x7f3611f45e50>

    @pytest.mark.django_db
    def test_flush_repo_little_bit_of_everything(mocker, mock_storage):
        repo = RepositoryFactory()
        archive_service = ArchiveService(repo)
    
        for i in range(8):
            # NOTE: `CommitWithReportFactory` exists, but its only usable from `api`,
            # because of unresolved imports
            commit = CommitFactory(repository=repo)
            commit_report = CommitReportFactory(commit=commit)
            upload = UploadFactory(report=commit_report, storage_path=f"upload{i}")
    
            archive_service.write_chunks(commit.commitid, f"chunks_data{i}")
            archive_service.write_file(upload.storage_path, f"upload_data{i}")
    
            ba_report = CommitReportFactory(commit=commit, report_type="bundle_analysis")
            ba_upload = UploadFactory(report=ba_report, storage_path=f"ba_upload{i}")
            ba_report_path = StoragePaths.bundle_report.path(
                repo_key=archive_service.storage_hash, report_key=ba_report.external_id
            )
            archive_service.storage.write_file(
                "bundle-analysis", ba_report_path, f"ba_report_data{i}"
            )
            archive_service.storage.write_file(
                "bundle-analysis", ba_upload.storage_path, f"ba_upload_data{i}"
            )
    
        for i in range(17):
            PullFactory(repository=repo, pullid=i + 100)
    
        for i in range(23):
>           BranchFactory(repository=repo)

.../tests/unit/test_flush_repo.py:118: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../local/lib/python3.13............/site-packages/factory/base.py:40: in __call__
    return cls.create(**kwargs)
.../local/lib/python3.13............/site-packages/factory/base.py:528: in create
    return cls._generate(enums.CREATE_STRATEGY, kwargs)
.../local/lib/python3.13....../site-packages/factory/django.py:117: in _generate
    return super()._generate(strategy, params)
.../local/lib/python3.13............/site-packages/factory/base.py:465: in _generate
    return step.build()
.../local/lib/python3.13.../site-packages/factory/builder.py:262: in build
    instance = self.factory_meta.instantiate(
.../local/lib/python3.13............/site-packages/factory/base.py:317: in instantiate
    return self.factory._create(model, *args, **kwargs)
.../local/lib/python3.13....../site-packages/factory/django.py:166: in _create
    return manager.create(*args, **kwargs)
.../local/lib/python3.13.../db/models/manager.py:87: in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
.../local/lib/python3.13.../db/models/query.py:658: in create
    obj.save(force_insert=True, using=self.db)
.../local/lib/python3.13.../db/models/base.py:814: in save
    self.save_base(
.../local/lib/python3.13.../db/models/base.py:877: in save_base
    updated = self._save_table(
.../local/lib/python3.13.../db/models/base.py:1020: in _save_table
    results = self._do_insert(
.../local/lib/python3.13.../site-packages/django_prometheus/models.py:43: in _do_insert
    return super()._do_insert(*args, **kwargs)
.../local/lib/python3.13.../db/models/base.py:1061: in _do_insert
    return manager._insert(
.../local/lib/python3.13.../db/models/manager.py:87: in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
.../local/lib/python3.13.../db/models/query.py:1805: in _insert
    return query.get_compiler(using=using).execute_sql(returning_fields)
.../local/lib/python3.13.../models/sql/compiler.py:1822: in execute_sql
    cursor.execute(sql, params)
.../local/lib/python3.13.../db/backends/utils.py:67: in execute
    return self._execute_with_wrappers(
.../local/lib/python3.13.../db/backends/utils.py:80: in _execute_with_wrappers
    return executor(sql, params, many, context)
.../local/lib/python3.13.../db/backends/utils.py:84: in _execute
    with self.db.wrap_database_errors:
.../local/lib/python3.13.../django/db/utils.py:91: in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>
sql = 'INSERT INTO "branches" ("branch", "repoid", "authors", "head", "base", "updatestamp") VALUES (%s, %s, %s::integer[], %s, %s, %s)'
params = ('President.', 5, None, '0ea5733d0b9d5754f6b06a2f0d5882e1e3c48cf3', None, datetime.datetime(2025, 1, 17, 12, 22, 7, 500648, tzinfo=datetime.timezone.utc))
ignored_wrapper_args = (False, {'connection': <DatabaseWrapper vendor='postgresql' alias='default'>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>})

    def _execute(self, sql, params, *ignored_wrapper_args):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            if params is None:
                # params default might be backend specific.
                return self.cursor.execute(sql)
            else:
>               return self.cursor.execute(sql, params)
E               django.db.utils.IntegrityError: duplicate key value violates unique constraint "branches_repoid_branch"
E               DETAIL:  Key (repoid, branch)=(5, President.) already exists.

.../local/lib/python3.13.../db/backends/utils.py:89: IntegrityError

To view more test analytics, go to the Test Analytics Dashboard
📢 Thoughts on this report? Let us know!

[github-actions]

✅ All tests successful. No failed tests were found.

📣 Thoughts on this report? Let Codecov know! | Powered by Codecov

[codecov]

Codecov Report

Attention: Patch coverage is 88.69779% with 46 lines in your changes missing coverage. Please review.

Project coverage is 97.64%. Comparing base (2b5bffc) to head (1d61cc3).

❌ We are unable to process any of the uploaded JUnit XML files. Please ensure your files are in the right format.

Files with missing lines	Patch %	Lines
services/cleanup/models.py	64.04%	32 Missing ⚠️
tasks/tests/unit/test_flush_repo.py	85.71%	9 Missing ⚠️
services/cleanup/relations.py	95.65%	3 Missing ⚠️
services/cleanup/repository.py	92.85%	2 Missing ⚠️

Additional details and impacted files

@@            Coverage Diff             @@
##             main     #973      +/-   ##
==========================================
- Coverage   97.75%   97.64%   -0.12%     
==========================================
  Files         451      458       +7     
  Lines       36812    36887      +75     
==========================================
+ Hits        35986    36017      +31     
- Misses        826      870      +44     

Flag	Coverage Δ
integration	42.61% <34.88%> (+0.13%)	⬆️
unit	90.12% <88.69%> (-0.10%)	⬇️

Flags with carried forward coverage won't be shown. Click here to find out more.

☔ View full report in Codecov by Sentry.
📢 Have feedback on the report? Share it here.

⚠️ Impact Analysis from Codecov is deprecated and will be sunset on Jan 31 2025. See more

[github-actions]

This PR includes changes to shared. Please review them here: codecov/shared@609e56d...c1c9cb2

[codecov-qa]

❌ 1 Tests Failed:

Tests completed	Failed	Passed	Skipped
1779	1	1778	4

View the top 1 failed tests by shortest run time

tasks/tests/unit/test_flush_repo.py::test_flush_repo_little_bit_of_everything

Stack Traces | 0.132s run time

self = <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>
sql = 'INSERT INTO "branches" ("branch", "repoid", "authors", "head", "base", "updatestamp") VALUES (%s, %s, %s::integer[], %s, %s, %s)'
params = ('President.', 5, None, '0ea5733d0b9d5754f6b06a2f0d5882e1e3c48cf3', None, datetime.datetime(2025, 1, 17, 12, 22, 7, 500648, tzinfo=datetime.timezone.utc))
ignored_wrapper_args = (False, {'connection': <DatabaseWrapper vendor='postgresql' alias='default'>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>})

    def _execute(self, sql, params, *ignored_wrapper_args):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            if params is None:
                # params default might be backend specific.
                return self.cursor.execute(sql)
            else:
>               return self.cursor.execute(sql, params)
E               psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "branches_repoid_branch"
E               DETAIL:  Key (repoid, branch)=(5, President.) already exists.

.../local/lib/python3.13.../db/backends/utils.py:89: UniqueViolation

The above exception was the direct cause of the following exception:

mocker = <pytest_mock.plugin.MockFixture object at 0x7f36181422d0>
mock_storage = <shared.storage.memory.MemoryStorageService object at 0x7f3611f45e50>

    @pytest.mark.django_db
    def test_flush_repo_little_bit_of_everything(mocker, mock_storage):
        repo = RepositoryFactory()
        archive_service = ArchiveService(repo)
    
        for i in range(8):
            # NOTE: `CommitWithReportFactory` exists, but its only usable from `api`,
            # because of unresolved imports
            commit = CommitFactory(repository=repo)
            commit_report = CommitReportFactory(commit=commit)
            upload = UploadFactory(report=commit_report, storage_path=f"upload{i}")
    
            archive_service.write_chunks(commit.commitid, f"chunks_data{i}")
            archive_service.write_file(upload.storage_path, f"upload_data{i}")
    
            ba_report = CommitReportFactory(commit=commit, report_type="bundle_analysis")
            ba_upload = UploadFactory(report=ba_report, storage_path=f"ba_upload{i}")
            ba_report_path = StoragePaths.bundle_report.path(
                repo_key=archive_service.storage_hash, report_key=ba_report.external_id
            )
            archive_service.storage.write_file(
                "bundle-analysis", ba_report_path, f"ba_report_data{i}"
            )
            archive_service.storage.write_file(
                "bundle-analysis", ba_upload.storage_path, f"ba_upload_data{i}"
            )
    
        for i in range(17):
            PullFactory(repository=repo, pullid=i + 100)
    
        for i in range(23):
>           BranchFactory(repository=repo)

.../tests/unit/test_flush_repo.py:118: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../local/lib/python3.13............/site-packages/factory/base.py:40: in __call__
    return cls.create(**kwargs)
.../local/lib/python3.13............/site-packages/factory/base.py:528: in create
    return cls._generate(enums.CREATE_STRATEGY, kwargs)
.../local/lib/python3.13....../site-packages/factory/django.py:117: in _generate
    return super()._generate(strategy, params)
.../local/lib/python3.13............/site-packages/factory/base.py:465: in _generate
    return step.build()
.../local/lib/python3.13.../site-packages/factory/builder.py:262: in build
    instance = self.factory_meta.instantiate(
.../local/lib/python3.13............/site-packages/factory/base.py:317: in instantiate
    return self.factory._create(model, *args, **kwargs)
.../local/lib/python3.13....../site-packages/factory/django.py:166: in _create
    return manager.create(*args, **kwargs)
.../local/lib/python3.13.../db/models/manager.py:87: in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
.../local/lib/python3.13.../db/models/query.py:658: in create
    obj.save(force_insert=True, using=self.db)
.../local/lib/python3.13.../db/models/base.py:814: in save
    self.save_base(
.../local/lib/python3.13.../db/models/base.py:877: in save_base
    updated = self._save_table(
.../local/lib/python3.13.../db/models/base.py:1020: in _save_table
    results = self._do_insert(
.../local/lib/python3.13.../site-packages/django_prometheus/models.py:43: in _do_insert
    return super()._do_insert(*args, **kwargs)
.../local/lib/python3.13.../db/models/base.py:1061: in _do_insert
    return manager._insert(
.../local/lib/python3.13.../db/models/manager.py:87: in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
.../local/lib/python3.13.../db/models/query.py:1805: in _insert
    return query.get_compiler(using=using).execute_sql(returning_fields)
.../local/lib/python3.13.../models/sql/compiler.py:1822: in execute_sql
    cursor.execute(sql, params)
.../local/lib/python3.13.../db/backends/utils.py:67: in execute
    return self._execute_with_wrappers(
.../local/lib/python3.13.../db/backends/utils.py:80: in _execute_with_wrappers
    return executor(sql, params, many, context)
.../local/lib/python3.13.../db/backends/utils.py:84: in _execute
    with self.db.wrap_database_errors:
.../local/lib/python3.13.../django/db/utils.py:91: in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>
sql = 'INSERT INTO "branches" ("branch", "repoid", "authors", "head", "base", "updatestamp") VALUES (%s, %s, %s::integer[], %s, %s, %s)'
params = ('President.', 5, None, '0ea5733d0b9d5754f6b06a2f0d5882e1e3c48cf3', None, datetime.datetime(2025, 1, 17, 12, 22, 7, 500648, tzinfo=datetime.timezone.utc))
ignored_wrapper_args = (False, {'connection': <DatabaseWrapper vendor='postgresql' alias='default'>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>})

    def _execute(self, sql, params, *ignored_wrapper_args):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            if params is None:
                # params default might be backend specific.
                return self.cursor.execute(sql)
            else:
>               return self.cursor.execute(sql, params)
E               django.db.utils.IntegrityError: duplicate key value violates unique constraint "branches_repoid_branch"
E               DETAIL:  Key (repoid, branch)=(5, President.) already exists.

.../local/lib/python3.13.../db/backends/utils.py:89: IntegrityError

To view more test analytics, go to the Test Analytics Dashboard
📢 Thoughts on this report? Let us know!

[codecov-public-qa]

❌ 1 Tests Failed:

Tests completed	Failed	Passed	Skipped
1779	1	1778	4

View the top 1 failed tests by shortest run time

tasks/tests/unit/test_flush_repo.py::::test_flush_repo_little_bit_of_everything

Stack Traces | 0.132s run time

self = <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>
sql = 'INSERT INTO "branches" ("branch", "repoid", "authors", "head", "base", "updatestamp") VALUES (%s, %s, %s::integer[], %s, %s, %s)'
params = ('President.', 5, None, '0ea5733d0b9d5754f6b06a2f0d5882e1e3c48cf3', None, datetime.datetime(2025, 1, 17, 12, 22, 7, 500648, tzinfo=datetime.timezone.utc))
ignored_wrapper_args = (False, {'connection': <DatabaseWrapper vendor='postgresql' alias='default'>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>})

    def _execute(self, sql, params, *ignored_wrapper_args):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            if params is None:
                # params default might be backend specific.
                return self.cursor.execute(sql)
            else:
>               return self.cursor.execute(sql, params)
E               psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "branches_repoid_branch"
E               DETAIL:  Key (repoid, branch)=(5, President.) already exists.

.../local/lib/python3.13.../db/backends/utils.py:89: UniqueViolation

The above exception was the direct cause of the following exception:

mocker = <pytest_mock.plugin.MockFixture object at 0x7f36181422d0>
mock_storage = <shared.storage.memory.MemoryStorageService object at 0x7f3611f45e50>

    @pytest.mark.django_db
    def test_flush_repo_little_bit_of_everything(mocker, mock_storage):
        repo = RepositoryFactory()
        archive_service = ArchiveService(repo)
    
        for i in range(8):
            # NOTE: `CommitWithReportFactory` exists, but its only usable from `api`,
            # because of unresolved imports
            commit = CommitFactory(repository=repo)
            commit_report = CommitReportFactory(commit=commit)
            upload = UploadFactory(report=commit_report, storage_path=f"upload{i}")
    
            archive_service.write_chunks(commit.commitid, f"chunks_data{i}")
            archive_service.write_file(upload.storage_path, f"upload_data{i}")
    
            ba_report = CommitReportFactory(commit=commit, report_type="bundle_analysis")
            ba_upload = UploadFactory(report=ba_report, storage_path=f"ba_upload{i}")
            ba_report_path = StoragePaths.bundle_report.path(
                repo_key=archive_service.storage_hash, report_key=ba_report.external_id
            )
            archive_service.storage.write_file(
                "bundle-analysis", ba_report_path, f"ba_report_data{i}"
            )
            archive_service.storage.write_file(
                "bundle-analysis", ba_upload.storage_path, f"ba_upload_data{i}"
            )
    
        for i in range(17):
            PullFactory(repository=repo, pullid=i + 100)
    
        for i in range(23):
>           BranchFactory(repository=repo)

.../tests/unit/test_flush_repo.py:118: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../local/lib/python3.13............/site-packages/factory/base.py:40: in __call__
    return cls.create(**kwargs)
.../local/lib/python3.13............/site-packages/factory/base.py:528: in create
    return cls._generate(enums.CREATE_STRATEGY, kwargs)
.../local/lib/python3.13....../site-packages/factory/django.py:117: in _generate
    return super()._generate(strategy, params)
.../local/lib/python3.13............/site-packages/factory/base.py:465: in _generate
    return step.build()
.../local/lib/python3.13.../site-packages/factory/builder.py:262: in build
    instance = self.factory_meta.instantiate(
.../local/lib/python3.13............/site-packages/factory/base.py:317: in instantiate
    return self.factory._create(model, *args, **kwargs)
.../local/lib/python3.13....../site-packages/factory/django.py:166: in _create
    return manager.create(*args, **kwargs)
.../local/lib/python3.13.../db/models/manager.py:87: in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
.../local/lib/python3.13.../db/models/query.py:658: in create
    obj.save(force_insert=True, using=self.db)
.../local/lib/python3.13.../db/models/base.py:814: in save
    self.save_base(
.../local/lib/python3.13.../db/models/base.py:877: in save_base
    updated = self._save_table(
.../local/lib/python3.13.../db/models/base.py:1020: in _save_table
    results = self._do_insert(
.../local/lib/python3.13.../site-packages/django_prometheus/models.py:43: in _do_insert
    return super()._do_insert(*args, **kwargs)
.../local/lib/python3.13.../db/models/base.py:1061: in _do_insert
    return manager._insert(
.../local/lib/python3.13.../db/models/manager.py:87: in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
.../local/lib/python3.13.../db/models/query.py:1805: in _insert
    return query.get_compiler(using=using).execute_sql(returning_fields)
.../local/lib/python3.13.../models/sql/compiler.py:1822: in execute_sql
    cursor.execute(sql, params)
.../local/lib/python3.13.../db/backends/utils.py:67: in execute
    return self._execute_with_wrappers(
.../local/lib/python3.13.../db/backends/utils.py:80: in _execute_with_wrappers
    return executor(sql, params, many, context)
.../local/lib/python3.13.../db/backends/utils.py:84: in _execute
    with self.db.wrap_database_errors:
.../local/lib/python3.13.../django/db/utils.py:91: in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>
sql = 'INSERT INTO "branches" ("branch", "repoid", "authors", "head", "base", "updatestamp") VALUES (%s, %s, %s::integer[], %s, %s, %s)'
params = ('President.', 5, None, '0ea5733d0b9d5754f6b06a2f0d5882e1e3c48cf3', None, datetime.datetime(2025, 1, 17, 12, 22, 7, 500648, tzinfo=datetime.timezone.utc))
ignored_wrapper_args = (False, {'connection': <DatabaseWrapper vendor='postgresql' alias='default'>, 'cursor': <django.db.backends.utils.CursorWrapper object at 0x7f36110c6870>})

    def _execute(self, sql, params, *ignored_wrapper_args):
        self.db.validate_no_broken_transaction()
        with self.db.wrap_database_errors:
            if params is None:
                # params default might be backend specific.
                return self.cursor.execute(sql)
            else:
>               return self.cursor.execute(sql, params)
E               django.db.utils.IntegrityError: duplicate key value violates unique constraint "branches_repoid_branch"
E               DETAIL:  Key (repoid, branch)=(5, President.) already exists.

.../local/lib/python3.13.../db/backends/utils.py:89: IntegrityError

To view more test analytics, go to the Test Analytics Dashboard
📢 Thoughts on this report? Let us know!