from django.conf import settings
from django.contrib.auth.models import User
from django.db import models, transaction
from django.db.models import Q
from django.shortcuts import get_object_or_404
from datetime import datetime, timedelta, timezone
from pgcommitfest.mailqueue.util import send_template_mail
from pgcommitfest.userprofile.models import UserProfile
from .util import DiffableModel
# We have few enough of these, and it's really the only thing we
# need to extend from the user model, so just create a separate
# class.
class Committer(models.Model):
user = models.OneToOneField(
User, null=False, blank=False, primary_key=True, on_delete=models.CASCADE
)
active = models.BooleanField(null=False, blank=False, default=True)
def __str__(self):
return str(self.user)
@property
def fullname(self):
return "%s %s (%s)" % (
self.user.first_name,
self.user.last_name,
self.user.username,
)
class Meta:
ordering = ("user__last_name", "user__first_name")
class CommitFest(models.Model):
STATUS_OPEN = 2
STATUS_INPROGRESS = 3
STATUS_CLOSED = 4
_STATUS_CHOICES = (
(STATUS_OPEN, "Open"),
(STATUS_INPROGRESS, "In Progress"),
(STATUS_CLOSED, "Closed"),
)
_STATUS_LABELS = (
(STATUS_OPEN, "info"),
(STATUS_INPROGRESS, "success"),
(STATUS_CLOSED, "danger"),
)
name = models.CharField(max_length=100, blank=False, null=False, unique=True)
status = models.IntegerField(
null=False, blank=False, default=2, choices=_STATUS_CHOICES
)
startdate = models.DateField(blank=False, null=False)
enddate = models.DateField(blank=False, null=False)
draft = models.BooleanField(blank=False, null=False, default=False)
@property
def statusstring(self):
return [v for k, v in self._STATUS_CHOICES if k == self.status][0]
@property
def periodstring(self):
return "{0} {1}".format(self.startdate, self.enddate)
@property
def last_open_date(self):
return self.startdate - timedelta(days=1)
@property
def dev_cycle(self) -> int:
if self.startdate.month in [1, 3]:
return self.startdate.year - 2007
else:
return self.startdate.year - 2006
@property
def title(self):
return "Commitfest %s" % self.name
@property
def is_closed(self):
return self.status == self.STATUS_CLOSED
@property
def is_open(self):
return self.status == self.STATUS_OPEN
@property
def is_open_regular(self):
return self.is_open and not self.draft
@property
def is_open_draft(self):
return self.is_open and self.draft
@property
def is_in_progress(self):
return self.status == self.STATUS_INPROGRESS
def to_json(self):
return {
"id": self.id,
"name": self.name,
"status": self.statusstring,
"draft": self.draft,
"startdate": self.startdate.isoformat(),
"enddate": self.enddate.isoformat(),
}
def _should_auto_move_patch(self, patch, current_date):
"""Determine if a patch should be automatically moved to the next commitfest.
A patch qualifies for auto-move if it both:
1. Has had email activity within the configured number of days
2. Hasn't been failing CI for longer than the configured threshold
"""
activity_cutoff = current_date - timedelta(
days=settings.AUTO_MOVE_EMAIL_ACTIVITY_DAYS
)
failing_cutoff = current_date - timedelta(
days=settings.AUTO_MOVE_MAX_FAILING_DAYS
)
# Check for recent email activity
if not patch.lastmail or patch.lastmail < activity_cutoff:
return False
# Check if CI has been failing too long
try:
cfbot_branch = patch.cfbot_branch
if (
cfbot_branch.failing_since
and cfbot_branch.failing_since < failing_cutoff
):
return False
except CfbotBranch.DoesNotExist:
# IF no CFBot data exists, the patch is probably very new (i.e. no
# CI run has ever taken place for it yet). So we auto-move it in
# that case.
pass
return True
def auto_move_active_patches(self):
"""Automatically move active patches to the next commitfest.
A patch is moved if it has recent email activity and hasn't been
failing CI for too long.
"""
current_date = datetime.now()
# Get the next open commitfest (must exist, raises IndexError otherwise)
# For draft CFs, find the next draft CF
# For regular CFs, find the next regular CF by start date
if self.draft:
next_cf = CommitFest.objects.filter(
status=CommitFest.STATUS_OPEN,
draft=True,
startdate__gt=self.enddate,
).order_by("startdate")[0]
else:
next_cf = CommitFest.objects.filter(
status=CommitFest.STATUS_OPEN,
draft=False,
startdate__gt=self.enddate,
).order_by("startdate")[0]
# Get all patches with open status in this commitfest
open_pocs = self.patchoncommitfest_set.filter(
status__in=PatchOnCommitFest.OPEN_STATUSES
).select_related("patch")
for poc in open_pocs:
if self._should_auto_move_patch(poc.patch, current_date):
poc.patch.move(self, next_cf, by_user=None, by_cfbot=True)
def send_closure_notifications(self):
"""Send email notifications to authors of patches that are still open."""
# Get patches that still need action (not moved, not closed)
open_pocs = list(
self.patchoncommitfest_set.filter(
status__in=PatchOnCommitFest.OPEN_STATUSES
)
.select_related("patch")
.prefetch_related("patch__authors")
)
if not open_pocs:
return
# Collect unique authors and their patches
authors_patches = {}
for poc in open_pocs:
for author in poc.patch.authors.all():
if author not in authors_patches:
authors_patches[author] = []
authors_patches[author].append(poc)
# Send email to each author who has notifications enabled
for author, patches in authors_patches.items():
try:
profile = author.userprofile
if not profile.notify_all_author:
continue
notifyemail = profile.notifyemail
except UserProfile.DoesNotExist:
# No profile means use default (notify=True), so continue
notifyemail = None
email = notifyemail.email if notifyemail else author.email
send_template_mail(
settings.NOTIFICATION_FROM,
None,
email,
f"Commitfest {self.name} has closed and you have unmoved patches",
"mail/commitfest_closure.txt",
{
"user": author,
"commitfest": self,
"patches": patches,
},
)
@staticmethod
def _are_relevant_commitfests_up_to_date(cfs, current_date):
inprogress_cf = cfs["in_progress"]
if inprogress_cf and inprogress_cf.enddate < current_date:
return False
if cfs["open"].startdate self.modified:
self.modified = newmod
def update_lastmail(self):
# Update the lastmail field, based on the newest email in any of
# the threads attached to it.
threads = list(self.mailthread_set.all())
if len(threads) == 0:
self.lastmail = None
else:
self.lastmail = max(threads, key=lambda t: t.latestmessage).latestmessage
def move(
self, from_cf, to_cf, by_user, allow_move_to_in_progress=False, by_cfbot=False
):
"""Returns the new PatchOnCommitFest object, or raises UserInputError"""
current_poc = self.current_patch_on_commitfest()
if from_cf.id != current_poc.commitfest.id:
raise UserInputError("Patch not in source commitfest.")
if from_cf.id == to_cf.id:
raise UserInputError("Source and target commitfest are the same.")
if current_poc.status not in (
PatchOnCommitFest.STATUS_REVIEW,
PatchOnCommitFest.STATUS_AUTHOR,
PatchOnCommitFest.STATUS_COMMITTER,
):
raise UserInputError(
f"Patch in state {current_poc.statusstring} cannot be moved."
)
if to_cf.is_in_progress:
if not allow_move_to_in_progress:
raise UserInputError("Patch can only be moved to an open commitfest")
elif not to_cf.is_open:
raise UserInputError("Patch can only be moved to an open commitfest")
old_status = current_poc.status
current_poc.set_status(PatchOnCommitFest.STATUS_MOVED)
new_poc, _ = PatchOnCommitFest.objects.update_or_create(
patch=current_poc.patch,
commitfest=to_cf,
defaults=dict(
status=old_status,
enterdate=datetime.now(),
leavedate=None,
),
)
new_poc.save()
self.set_modified()
self.save()
PatchHistory(
patch=self,
by=by_user,
by_cfbot=by_cfbot,
what=f"Moved from CF {from_cf} to CF {to_cf}",
).save_and_notify()
return new_poc
def __str__(self):
return self.name
class Meta:
verbose_name_plural = "patches"
class PatchOnCommitFest(models.Model):
# NOTE! This is also matched by the commitfest_patchstatus table,
# but we hardcoded it in here simply for performance reasons since
# the data should be entirely static. (Yes, that's something we
# might re-evaluate in the future)
STATUS_REVIEW = 1
STATUS_AUTHOR = 2
STATUS_COMMITTER = 3
STATUS_COMMITTED = 4
STATUS_MOVED = 5
STATUS_REJECTED = 6
STATUS_RETURNED = 7
STATUS_WITHDRAWN = 8
_STATUS_CHOICES = (
(STATUS_REVIEW, "Needs review"),
(STATUS_AUTHOR, "Waiting on Author"),
(STATUS_COMMITTER, "Ready for Committer"),
(STATUS_COMMITTED, "Committed"),
(STATUS_MOVED, "Moved to different CF"),
(STATUS_REJECTED, "Rejected"),
(STATUS_RETURNED, "Returned with feedback"),
(STATUS_WITHDRAWN, "Withdrawn"),
)
_STATUS_LABELS = (
(STATUS_REVIEW, "secondary"),
(STATUS_AUTHOR, "primary"),
(STATUS_COMMITTER, "info"),
(STATUS_COMMITTED, "success"),
(STATUS_MOVED, "warning"),
(STATUS_REJECTED, "danger"),
(STATUS_RETURNED, "danger"),
(STATUS_WITHDRAWN, "danger"),
)
OPEN_STATUSES = [STATUS_REVIEW, STATUS_AUTHOR, STATUS_COMMITTER]
@classmethod
def OPEN_STATUS_CHOICES(cls):
return [x for x in cls._STATUS_CHOICES if x[0] in cls.OPEN_STATUSES]
patch = models.ForeignKey(Patch, blank=False, null=False, on_delete=models.CASCADE)
commitfest = models.ForeignKey(
CommitFest, blank=False, null=False, on_delete=models.CASCADE
)
enterdate = models.DateTimeField(blank=False, null=False)
leavedate = models.DateTimeField(blank=True, null=True)
status = models.IntegerField(
blank=False, null=False, default=STATUS_REVIEW, choices=_STATUS_CHOICES
)
@property
def is_closed(self):
return self.status not in self.OPEN_STATUSES
@property
def is_open(self):
return not self.is_closed
@property
def is_committed(self):
return self.status == self.STATUS_COMMITTED
@property
def needs_committer(self):
return self.status == self.STATUS_COMMITTER
@property
def statusstring(self):
return [v for k, v in self._STATUS_CHOICES if k == self.status][0]
@classmethod
def current_for_patch(cls, patch_id):
return get_object_or_404(
cls, Q(patch_id=patch_id) & ~Q(status=cls.STATUS_MOVED)
)
def set_status(self, status):
self.status = status
if not self.leavedate and not self.is_open:
# If the patch was not closed before, we need to set the leavedate
# now.
self.leavedate = datetime.now()
elif self.is_open:
self.leavedate = None
self.patch.set_modified()
self.patch.save()
self.save()
class Meta:
unique_together = (
(
"patch",
"commitfest",
),
)
ordering = ("-commitfest__startdate",)
class PatchHistory(models.Model):
patch = models.ForeignKey(Patch, blank=False, null=False, on_delete=models.CASCADE)
date = models.DateTimeField(
blank=False, null=False, auto_now_add=True, db_index=True
)
by = models.ForeignKey(User, blank=True, null=True, on_delete=models.CASCADE)
by_cfbot = models.BooleanField(null=False, blank=False, default=False)
what = models.CharField(max_length=500, null=False, blank=False)
@property
def by_string(self):
if self.by_cfbot:
return "CFbot"
return "%s %s (%s)" % (self.by.first_name, self.by.last_name, self.by.username)
def __str__(self):
return "%s - %s" % (self.patch.name, self.date)
class Meta:
ordering = ("-date",)
constraints = [
models.CheckConstraint(
check=(models.Q(by_cfbot=True) & models.Q(by__isnull=True))
| (models.Q(by_cfbot=False) & models.Q(by__isnull=False)),
name="check_by",
),
]
def save_and_notify(
self,
prevcommitter=None,
prevreviewers=None,
prevauthors=None,
authors_only=False,
):
# Save this model, and then trigger notifications if there are any. There are
# many different things that can trigger notifications, so try them all.
self.save()
recipients = []
if not authors_only:
recipients.extend(self.patch.subscribers.all())
# Current or previous committer wants all notifications
try:
if (
self.patch.committer
and self.patch.committer.user.userprofile.notify_all_committer
):
recipients.append(self.patch.committer.user)
except UserProfile.DoesNotExist:
pass
try:
if (
prevcommitter
and prevcommitter.user.userprofile.notify_all_committer
):
recipients.append(prevcommitter.user)
except UserProfile.DoesNotExist:
pass
# Current or previous reviewers wants all notifications
recipients.extend(
self.patch.reviewers.filter(userprofile__notify_all_reviewer=True)
)
if prevreviewers:
# prevreviewers is a list
recipients.extend(
User.objects.filter(
id__in=[p.id for p in prevreviewers],
userprofile__notify_all_reviewer=True,
)
)
# Current or previous authors wants all notifications (default is True,
# so include users without a profile)
recipients.extend(
self.patch.authors.filter(
Q(userprofile__isnull=True) | Q(userprofile__notify_all_author=True)
)
)
for u in set(recipients):
if u != self.by: # Don't notify for changes we make ourselves
PendingNotification(history=self, user=u).save()
class MailThread(models.Model):
# This class tracks mail threads from the main postgresql.org
# mailinglist archives. For each thread, we store *one* messageid.
# Using this messageid we can always query the archives for more
# detailed information, which is done dynamically as the page
# is loaded.
# For threads in an active or future commitfest, we also poll
# the archives to fetch "updated entries" at (ir)regular intervals
# so we can keep track of when there was last a change on the
# thread in question.
messageid = models.CharField(max_length=1000, null=False, blank=False, unique=True)
subject = models.CharField(max_length=500, null=False, blank=False)
firstmessage = models.DateTimeField(null=False, blank=False)
firstauthor = models.CharField(max_length=500, null=False, blank=False)
latestmessage = models.DateTimeField(null=False, blank=False)
latestauthor = models.CharField(max_length=500, null=False, blank=False)
latestsubject = models.CharField(max_length=500, null=False, blank=False)
latestmsgid = models.CharField(max_length=1000, null=False, blank=False)
def __str__(self):
return self.subject
class Meta:
ordering = ("firstmessage",)
class MailThreadAttachment(models.Model):
mailthread = models.ForeignKey(
MailThread, null=False, blank=False, on_delete=models.CASCADE
)
messageid = models.CharField(max_length=1000, null=False, blank=False)
attachmentid = models.IntegerField(null=False, blank=False)
filename = models.CharField(max_length=1000, null=False, blank=True)
date = models.DateTimeField(null=False, blank=False)
author = models.CharField(max_length=500, null=False, blank=False)
ispatch = models.BooleanField(null=True)
class Meta:
ordering = ("-date",)
unique_together = (
(
"mailthread",
"messageid",
),
)
class MailThreadAnnotation(models.Model):
mailthread = models.ForeignKey(
MailThread, null=False, blank=False, on_delete=models.CASCADE
)
date = models.DateTimeField(null=False, blank=False, auto_now_add=True)
user = models.ForeignKey(User, null=False, blank=False, on_delete=models.CASCADE)
msgid = models.CharField(max_length=1000, null=False, blank=False)
annotationtext = models.TextField(null=False, blank=False, max_length=2000)
mailsubject = models.CharField(max_length=500, null=False, blank=False)
maildate = models.DateTimeField(null=False, blank=False)
mailauthor = models.CharField(max_length=500, null=False, blank=False)
@property
def user_string(self):
return "%s %s (%s)" % (
self.user.first_name,
self.user.last_name,
self.user.username,
)
class Meta:
ordering = ("date",)
class PatchStatus(models.Model):
status = models.IntegerField(null=False, blank=False, primary_key=True)
statusstring = models.TextField(max_length=50, null=False, blank=False)
sortkey = models.IntegerField(null=False, blank=False, default=10)
class PendingNotification(models.Model):
history = models.ForeignKey(
PatchHistory, blank=False, null=False, on_delete=models.CASCADE
)
user = models.ForeignKey(User, blank=False, null=False, on_delete=models.CASCADE)
class CfbotBranch(models.Model):
STATUS_CHOICES = [
("testing", "Testing"),
("finished", "Finished"),
("failed", "Failed"),
("timeout", "Timeout"),
]
patch = models.OneToOneField(
Patch, on_delete=models.CASCADE, related_name="cfbot_branch", primary_key=True
)
branch_id = models.IntegerField(null=False)
branch_name = models.TextField(null=False)
commit_id = models.TextField(null=True, blank=True)
apply_url = models.TextField(null=False)
# URL of the CI build for this branch. Sent by cfbot so we don't have to
# know which CI provider it uses. It's null until cfbot hears about the
# branch's first build, and "timeout" branches might never get one.
build_url = models.TextField(null=True, blank=True)
# Actually a postgres enum column
status = models.TextField(choices=STATUS_CHOICES, null=False)
needs_rebase_since = models.DateTimeField(null=True, blank=True)
failing_since = models.DateTimeField(null=True, blank=True)
created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True)
version = models.TextField(null=True, blank=True)
patch_count = models.IntegerField(null=True, blank=True)
first_additions = models.IntegerField(null=True, blank=True)
first_deletions = models.IntegerField(null=True, blank=True)
all_additions = models.IntegerField(null=True, blank=True)
all_deletions = models.IntegerField(null=True, blank=True)
def save(self, *args, **kwargs):
"""Only used by the admin panel to save empty commit id as NULL
The actual cfbot webhook doesn't use the django ORM to save the data.
"""
if not self.commit_id:
self.commit_id = None
super(CfbotBranch, self).save(*args, **kwargs)
class CfbotTask(models.Model):
STATUS_CHOICES = [
("CREATED", "Created"),
("PAUSED", "Paused"),
("NEEDS_APPROVAL", "Needs Approval"),
("TRIGGERED", "Triggered"),
("EXECUTING", "Executing"),
("FAILED", "Failed"),
("COMPLETED", "Completed"),
("SCHEDULED", "Scheduled"),
("ABORTED", "Aborted"),
("ERRORED", "Errored"),
]
# This id is only used by Django. Using text type for primary keys, has
# historically caused problems.
id = models.BigAutoField(primary_key=True)
# This is the id used by the external CI system. Currently with CirrusCI
# this is an integer, and thus we could probably store it as such. But
# given that we might need to change CI providers at some point, and that
# CI provider might use e.g. UUIDs, we prefer to consider the format of the
# ID opaque and store it as text.
task_id = models.TextField()
# URL of the task in the CI system. Sent by cfbot so we don't have to know
# which CI provider it uses.
task_url = models.TextField(null=False)
task_name = models.TextField(null=False)
patch = models.ForeignKey(
Patch, on_delete=models.CASCADE, related_name="cfbot_tasks"
)
branch_id = models.IntegerField(null=False)
position = models.IntegerField(null=False)
# Actually a postgres enum column
status = models.TextField(choices=STATUS_CHOICES, null=False)
created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True)
class Meta:
constraints = [
models.UniqueConstraint(
fields=["branch_id", "position"],
name="commitfest_cfbottask_branch_position_unique",
)
]