Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add Pull Request model and integrate with sync_repository #845

Open
wants to merge 26 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
0aff313
feat: Add Pull Request model and integrate with sync_repository
space-techy Feb 15, 2025
aa72a12
Merge branch 'main' into feature/github-pull-request-tracking
space-techy Feb 16, 2025
1bffa45
Merge branch 'main' into feature/github-pull-request-tracking
space-techy Feb 16, 2025
2a74139
Merge branch 'OWASP:main' into feature/github-pull-request-tracking
space-techy Feb 17, 2025
022555a
Merge branch 'OWASP:main' into feature/github-pull-request-tracking
space-techy Feb 17, 2025
180616d
Merge branch 'OWASP:main' into feature/github-pull-request-tracking
space-techy Feb 18, 2025
ad3fe10
Merge branch 'OWASP:main' into feature/github-pull-request-tracking
space-techy Feb 19, 2025
a1b81d9
Merge branch 'OWASP:main' into feature/github-pull-request-tracking
space-techy Feb 20, 2025
8d1e84c
Add Pull Request Model and Sync repo
space-techy Feb 20, 2025
b601a62
Add Pull Request Mode and Improve Repository Model and Sync Repo
space-techy Feb 20, 2025
093b6bc
Merge branch 'main' into feature/github-pull-request-tracking
space-techy Feb 21, 2025
4b1d97f
resolve conflict with issue
space-techy Feb 24, 2025
3989495
resolve conflict with issue
space-techy Feb 24, 2025
c787aa1
Merge branch 'OWASP:main' into feature/github-pull-request-tracking
space-techy Feb 24, 2025
4d03eb0
Update code
arkid15r Feb 25, 2025
df52c43
Merge branch 'OWASP:main' into feature/github-pull-request-tracking
space-techy Feb 25, 2025
bf4eaf7
Merge branch 'OWASP:main' into feature/github-pull-request-tracking
space-techy Feb 25, 2025
0bbd86e
remove issue part
space-techy Feb 25, 2025
ef422c7
Fix conflict
space-techy Feb 25, 2025
5268b4c
Fix conflict
space-techy Feb 25, 2025
03a0fa3
Merge branch 'OWASP:main' into feature/github-pull-request-tracking
space-techy Feb 25, 2025
0963487
Merge branch 'OWASP:main' into feature/github-pull-request-tracking
space-techy Feb 27, 2025
c609b75
Merge branch 'OWASP:main' into feature/github-pull-request-tracking
space-techy Feb 28, 2025
946bf40
Add Generic Issue and PR model
space-techy Feb 28, 2025
b43cbfe
Removed local migration
space-techy Feb 28, 2025
4bd0594
Add Generic Model and Migrations
space-techy Feb 28, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions backend/apps/github/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from apps.github.models.issue import Issue
from apps.github.models.label import Label
from apps.github.models.organization import Organization
from apps.github.models.pull_request import PullRequest
from apps.github.models.release import Release
from apps.github.models.repository import Repository
from apps.github.models.repository_contributor import RepositoryContributor
Expand All @@ -16,6 +17,38 @@ class LabelAdmin(admin.ModelAdmin):
search_fields = ("name", "description")


class PullRequestAdmin(admin.ModelAdmin):
autocomplete_fields = (
"assignees",
"author",
"labels",
"repository",
)
list_display = (
"repository",
"title",
"state",
"custom_field_github_url",
"created_at",
"updated_at",
)
list_filter = (
"state",
"merged_at",
)
search_fields = (
"author__login",
"repository__name",
"title",
)

def custom_field_github_url(self, obj):
"""Pull Request GitHub URL."""
return mark_safe(f"<a href='{obj.url}' target='_blank'>↗️</a>") # noqa: S308

custom_field_github_url.short_description = "GitHub 🔗"


class IssueAdmin(admin.ModelAdmin):
autocomplete_fields = (
"repository",
Expand Down Expand Up @@ -114,6 +147,7 @@ class UserAdmin(admin.ModelAdmin):
search_fields = ("login", "name")


admin.site.register(PullRequest, PullRequestAdmin)
admin.site.register(Issue, IssueAdmin)
admin.site.register(Label, LabelAdmin)
admin.site.register(Organization, OrganizationAdmin)
Expand Down
54 changes: 54 additions & 0 deletions backend/apps/github/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from apps.github.models.issue import Issue
from apps.github.models.label import Label
from apps.github.models.organization import Organization
from apps.github.models.pull_request import PullRequest
from apps.github.models.release import Release
from apps.github.models.repository import Repository
from apps.github.models.repository_contributor import RepositoryContributor
Expand Down Expand Up @@ -83,6 +84,59 @@ def sync_repository(gh_repository, organization=None, user=None):
else:
logger.info("Skipping issues sync for %s", repository.name)

if not repository.is_archived and repository.project:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suggest applying the same logic we use for issues sync (see above). It fetches all states and has a cut off date if it's not the very first sync.

Copy link
Collaborator Author

@space-techy space-techy Feb 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are talking about that since argument passed to **kwargs in issue?
If you are then since is not available in PR!

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's try fetching PRs sorted by updated and break the loop when we see a date earlier than our latest_updated_at (if any)?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can sort by the created-at field to retrieve the earliest date after the first fetch is complete. For subsequent runs, we can use this date. If the closed pull request’s date is older than the stored date, we can simply skip it.

# Fetch both open and closed PRs from GitHub
kwargs = {
"direction": "desc",
"sort": "created",
"state": "open",
}

latest_pull_request = repository.latest_pull_request
if latest_pull_request:
gh_first_pr = PullRequest.objects.order_by("created_at").first().created_at
kwargs["state"] = "all"

gh_pull_requests = gh_repository.get_pulls(**kwargs)

for gh_pull_request in gh_pull_requests:
if latest_pull_request and gh_pull_request.state == "closed":
# Skipping closed PR before first sync
if gh_first_pr > gh_pull_request.created_at:
break
# Check if this PR already exists in the database and is open
existing_open_pr = PullRequest.objects.filter(
repository=repository, state="open", number=gh_pull_request.number
).first()

if not existing_open_pr:
continue # Skip closed PRs from previous syncs

# Extract author details
author = (
User.update_data(gh_pull_request.user)
if gh_pull_request.user and gh_pull_request.user.type != "Bot"
else None
)

# Update PR data
pull_request = PullRequest.update_data(
gh_pull_request, author=author, repository=repository
)

# Clear and update assignees
pull_request.assignees.clear()
for gh_pull_request_assignee in gh_pull_request.assignees:
pull_request.assignees.add(User.update_data(gh_pull_request_assignee))

# Clear and update labels
pull_request.labels.clear()
for gh_pull_request_label in gh_pull_request.labels:
try:
pull_request.labels.add(Label.update_data(gh_pull_request_label))
except UnknownObjectException:
logger.info("Couldn't get GitHub pull request label %s", pull_request.url)

# GitHub repository releases.
releases = []
if not is_owasp_site_repository:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
# Generated by Django 5.1.6 on 2025-02-28 20:27

import django.db.models.deletion
import django.db.models.manager
from django.db import migrations, models

import apps.github.models.mixins.issue


class Migration(migrations.Migration):
dependencies = [
("github", "0016_user_is_bot"),
]

operations = [
migrations.AlterModelManagers(
name="issue",
managers=[
("open_issues", django.db.models.manager.Manager()),
],
),
migrations.AlterField(
model_name="issue",
name="sequence_id",
field=models.PositiveBigIntegerField(default=0, verbose_name="ID"),
),
migrations.AlterField(
model_name="issue",
name="state",
field=models.CharField(
choices=[("open", "Open"), ("closed", "Closed")],
default="open",
max_length=6,
verbose_name="State",
),
),
migrations.CreateModel(
name="PullRequest",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("nest_created_at", models.DateTimeField(auto_now_add=True)),
("nest_updated_at", models.DateTimeField(auto_now=True)),
("node_id", models.CharField(unique=True, verbose_name="Node ID")),
("title", models.CharField(max_length=1000, verbose_name="Title")),
("body", models.TextField(default="", verbose_name="Body")),
(
"state",
models.CharField(
choices=[("open", "Open"), ("closed", "Closed")],
default="open",
max_length=6,
verbose_name="State",
),
),
("url", models.URLField(default="", max_length=500, verbose_name="URL")),
("number", models.PositiveBigIntegerField(default=0, verbose_name="Number")),
("sequence_id", models.PositiveBigIntegerField(default=0, verbose_name="ID")),
(
"closed_at",
models.DateTimeField(blank=True, null=True, verbose_name="Closed at"),
),
("created_at", models.DateTimeField(verbose_name="Created at")),
("updated_at", models.DateTimeField(db_index=True, verbose_name="Updated at")),
(
"merged_at",
models.DateTimeField(blank=True, null=True, verbose_name="Merged at"),
),
(
"assignees",
models.ManyToManyField(
blank=True,
related_name="assigned_pull_requests",
to="github.user",
verbose_name="Assignees",
),
),
(
"author",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="created_pull_requests",
to="github.user",
verbose_name="Author",
),
),
(
"labels",
models.ManyToManyField(
blank=True,
related_name="pull_request_labels",
to="github.label",
verbose_name="Labels",
),
),
(
"repository",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="pull_requests",
to="github.repository",
),
),
],
options={
"verbose_name_plural": "Pull Requests",
"db_table": "github_pull_requests",
"ordering": ("-updated_at", "-state"),
},
bases=(apps.github.models.mixins.issue.IssueIndexMixin, models.Model),
managers=[
("open_pull_requests", django.db.models.manager.Manager()),
],
),
]
2 changes: 2 additions & 0 deletions backend/apps/github/models/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
"""Github app."""

from .pull_request import PullRequest
47 changes: 47 additions & 0 deletions backend/apps/github/models/generic_issue_model.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
"""Github app Generic model."""

from django.db import models

from apps.common.models import BulkSaveModel, TimestampedModel
from apps.github.models.common import NodeModel
from apps.github.models.mixins import IssueIndexMixin


class GenericIssueModel(BulkSaveModel, IssueIndexMixin, NodeModel, TimestampedModel):
"""Generic Issue And PR model."""

objects = models.Manager()

class Meta:
abstract = True

class State(models.TextChoices):
OPEN = "open", "Open"
CLOSED = "closed", "Closed"

title = models.CharField(verbose_name="Title", max_length=1000)
body = models.TextField(verbose_name="Body", default="")

state = models.CharField(verbose_name="State", max_length=6, choices=State, default=State.OPEN)
url = models.URLField(verbose_name="URL", max_length=500, default="")

number = models.PositiveBigIntegerField(verbose_name="Number", default=0)
sequence_id = models.PositiveBigIntegerField(verbose_name="ID", default=0)

closed_at = models.DateTimeField(verbose_name="Closed at", blank=True, null=True)
created_at = models.DateTimeField(verbose_name="Created at")
updated_at = models.DateTimeField(verbose_name="Updated at", db_index=True)

def __str__(self):
"""Issue human readable representation."""
return f"{self.title} by {self.author}"

@property
def project(self):
"""Return project."""
return self.repository.project

@property
def repository_id(self):
"""Return repository ID."""
return self.repository.id
41 changes: 5 additions & 36 deletions backend/apps/github/models/issue.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,45 +5,32 @@
from django.db import models

from apps.common.index import IndexBase
from apps.common.models import BulkSaveModel, TimestampedModel
from apps.common.models import BulkSaveModel
from apps.common.open_ai import OpenAi
from apps.core.models.prompt import Prompt
from apps.github.models.common import NodeModel
from apps.github.models.managers.issue import OpenIssueManager
from apps.github.models.mixins import IssueIndexMixin

from .generic_issue_model import GenericIssueModel

class Issue(BulkSaveModel, IssueIndexMixin, NodeModel, TimestampedModel):

class Issue(GenericIssueModel):
"""Issue model."""

objects = models.Manager()
open_issues = OpenIssueManager()

class Meta:
db_table = "github_issues"
ordering = ("-updated_at", "-state")
verbose_name_plural = "Issues"

class State(models.TextChoices):
OPEN = "open", "Open"
CLOSED = "closed", "Closed"

title = models.CharField(verbose_name="Title", max_length=1000)
body = models.TextField(verbose_name="Body", default="")

summary = models.TextField(
verbose_name="Summary", default="", blank=True
) # AI generated summary
hint = models.TextField(verbose_name="Hint", default="", blank=True) # AI generated hint
state = models.CharField(
verbose_name="State", max_length=20, choices=State, default=State.OPEN
)

state_reason = models.CharField(
verbose_name="State reason", max_length=200, default="", blank=True
)
url = models.URLField(verbose_name="URL", max_length=500, default="")
number = models.PositiveBigIntegerField(verbose_name="Number", default=0)
sequence_id = models.PositiveBigIntegerField(verbose_name="Issue ID", default=0)

is_locked = models.BooleanField(verbose_name="Is locked", default=False)
lock_reason = models.CharField(
Expand All @@ -52,10 +39,6 @@ class State(models.TextChoices):

comments_count = models.PositiveIntegerField(verbose_name="Comments", default=0)

closed_at = models.DateTimeField(verbose_name="Closed at", blank=True, null=True)
created_at = models.DateTimeField(verbose_name="Created at")
updated_at = models.DateTimeField(verbose_name="Updated at", db_index=True)

# FKs.
author = models.ForeignKey(
"github.User",
Expand Down Expand Up @@ -87,20 +70,6 @@ class State(models.TextChoices):
blank=True,
)

def __str__(self):
"""Issue human readable representation."""
return f"{self.title} by {self.author}"

@property
def project(self):
"""Return project."""
return self.repository.project

@property
def repository_id(self):
"""Return repository ID."""
return self.repository.id

def from_github(self, gh_issue, author=None, repository=None):
"""Update instance based on GitHub issue data."""
field_mapping = {
Expand Down
Loading