Rename ContainerInstance to ContainerRun for #739.

Add container analysis page that lets you choose a container app and creates a container run. Doesn't select inputs or actually launch anything yet.
Generate migration for remaining tables.

kive/container/ajax.py

@@ -8,10 +8,11 @@
 from rest_framework import permissions
 from rest_framework.decorators import action
 from rest_framework.response import Response
+from rest_framework.viewsets import ReadOnlyModelViewSet
 
-from container.models import ContainerFamily, Container, ContainerApp
+from container.models import ContainerFamily, Container, ContainerApp, ContainerRun
 from container.serializers import ContainerFamilySerializer, ContainerSerializer, \
-    ContainerAppSerializer
+    ContainerAppSerializer, ContainerFamilyChoiceSerializer, ContainerRunSerializer
 from kive.ajax import CleanCreateModelMixin, RemovableModelViewSet, \
     SearchableModelMixin, IsDeveloperOrGrantedReadOnly, StandardPagination
 from metadata.models import AccessControl
@@ -81,6 +82,47 @@ def containers(self, request, pk=None):
                                 context={"request": request}).data)
 
 
+class ContainerChoiceViewSet(ReadOnlyModelViewSet, SearchableModelMixin):
+    """ Container choices are container / app combinations grouped by family.
+
+    Query parameters:
+
+    * is_granted - true For administrators, this limits the list to only include
+        records that the user has been explicitly granted access to. For other
+        users, this has no effect.
+    * filters[n][key]=x&filters[n][val]=y - Apply different filters to the
+        search. n starts at 0 and increases by 1 for each added filter.
+        Some filters just have a key and ignore the val value. The possible
+        filters are listed below.
+    * filters[n][key]=smart&filters[n][val]=match - name or description from
+        family, container, or app contains the value (case insensitive)
+    * filters[n][key]=family&filters[n][val]=match - family name contains the
+        value (case insensitive)
+    * filters[n][key]=family_desc&filters[n][val]=match - family description
+        contains the value (case insensitive)
+    * filters[n][key]=container&filters[n][val]=match - container name contains
+        the value (case insensitive)
+    * filters[n][key]=container_desc&filters[n][val]=match - container
+        description contains the value (case insensitive)
+    * filters[n][key]=app&filters[n][val]=match - app name contains the
+        value (case insensitive)
+    * filters[n][key]=app_desc&filters[n][val]=match - app description
+        contains the value (case insensitive)
+    """
+    queryset = ContainerFamily.objects.prefetch_related('containers__apps')
+    serializer_class = ContainerFamilyChoiceSerializer
+    permission_classes = (permissions.IsAuthenticated, IsDeveloperOrGrantedReadOnly)
+    pagination_class = StandardPagination
+    filters = dict(
+        smart=lambda queryset, value: queryset.filter(
+            Q(name__icontains=value) |
+            Q(description__icontains=value)),
+        family=lambda queryset, value: queryset.filter(
+            name__icontains=value),
+        family_desc=lambda queryset, value: queryset.filter(
+            description__icontains=value))
+
+
 class ContainerViewSet(CleanCreateModelMixin,
                        RemovableModelViewSet,
                        SearchableModelMixin):
@@ -108,7 +150,8 @@ class ContainerViewSet(CleanCreateModelMixin,
     * filters[n][key]=user&filters[n][val]=match - username of creator contains
         the value (case insensitive)
     """
-    queryset = Container.objects.all()
+    queryset = Container.objects.annotate(
+        num_apps=Count('apps'))
     serializer_class = ContainerSerializer
     permission_classes = (permissions.IsAuthenticated, IsDeveloperOrGrantedReadOnly)
     pagination_class = StandardPagination
@@ -187,3 +230,66 @@ def filter_granted(self, queryset):
         granted_containers = Container.filter_by_user(self.request.user)
 
         return queryset.filter(container_id__in=granted_containers)
+
+
+class ContainerRunPermission(permissions.BasePermission):
+    """
+    Custom permission for Container Runs.
+
+    All users should be allowed to create Runs.  Users should be allowed to
+    rerun any Run visible to them.  However, Runs may only be stopped by
+    administrators or their owner.
+    """
+    def has_permission(self, request, view):
+        return (request.method in permissions.SAFE_METHODS or
+                request.method == "POST" or
+                request.method == "PATCH" or
+                admin_check(request.user))
+
+    def has_object_permission(self, request, view, obj):
+        if admin_check(request.user):
+            return True
+        if not obj.can_be_accessed(request.user):
+            return False
+        if request.method == "PATCH":
+            return obj.user == request.user
+        return request.method in permissions.SAFE_METHODS
+
+
+class ContainerRunViewSet(CleanCreateModelMixin,
+                          RemovableModelViewSet,
+                          SearchableModelMixin):
+    """ A container run is a running Singularity container app.
+
+    Query parameters:
+
+    * is_granted - true For administrators, this limits the list to only include
+        records that the user has been explicitly granted access to. For other
+        users, this has no effect.
+    * filters[n][key]=x&filters[n][val]=y - Apply different filters to the
+        search. n starts at 0 and increases by 1 for each added filter.
+        Some filters just have a key and ignore the val value. The possible
+        filters are listed below.
+    * filters[n][key]=smart&filters[n][val]=match - name or description
+        contains the value (case insensitive)
+    * filters[n][key]=name&filters[n][val]=match - name contains the value (case
+        insensitive)
+    * filters[n][key]=description&filters[n][val]=match - description contains
+        the value (case insensitive)
+    * filters[n][key]=user&filters[n][val]=match - username of creator contains
+        the value (case insensitive)
+    """
+    queryset = ContainerRun.objects.all()
+    serializer_class = ContainerRunSerializer
+    permission_classes = (permissions.IsAuthenticated, ContainerRunPermission)
+    pagination_class = StandardPagination
+    filters = dict(
+        smart=lambda queryset, value: queryset.filter(
+            Q(name__icontains=value) |
+            Q(description__icontains=value)),
+        name=lambda queryset, value: queryset.filter(
+            name__icontains=value),
+        description=lambda queryset, value: queryset.filter(
+            description__icontains=value),
+        user=lambda queryset, value: queryset.filter(
+            user__username__icontains=value))

kive/container/forms.py

@@ -1,7 +1,7 @@
 from django import forms
 from django.forms.widgets import TextInput
 
-from container.models import ContainerFamily, Container, ContainerApp
+from container.models import ContainerFamily, Container, ContainerApp, ContainerRun
 from metadata.forms import PermissionsForm
 
 
@@ -41,5 +41,14 @@ class ContainerAppForm(forms.ModelForm):
 
     class Meta(object):
         model = ContainerApp
-        fields = ['name', 'description', 'inputs', 'outputs']
+        exclude = ['container']
         widgets = dict(description=forms.Textarea(attrs=dict(cols=50, rows=10)))
+
+
+class ContainerRunForm(forms.ModelForm):
+    class Meta(object):
+        model = ContainerRun
+        exclude = ['user', 'state']
+
+    def save(self, commit=True):
+        return super(ContainerRunForm, self).save(commit)

kive/container/migrations/0005_container_run.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.15 on 2018-11-26 23:25
+from __future__ import unicode_literals
+
+from django.conf import settings
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('librarian', '0106_dataset_name_not_blank'),
+        ('auth', '0008_alter_user_username_max_length'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('container', '0004_containerapp_containerargument'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Batch',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(blank=True, max_length=60, verbose_name=b'Name of this batch of container runs')),
+                ('description', models.TextField(blank=True, max_length=1000)),
+                ('groups_allowed', models.ManyToManyField(blank=True, help_text='What groups have access?', related_name='container_batch_has_access_to', to='auth.Group')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('users_allowed', models.ManyToManyField(blank=True, help_text='Which users have access?', related_name='container_batch_has_access_to', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='ContainerDataset',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(help_text=b'Local file name, also used to sort multiple inputs for a single argument.', max_length=60)),
+                ('created', models.DateTimeField(auto_now_add=True, help_text=b'When this was added to Kive.')),
+                ('argument', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='datasets', to='container.ContainerArgument')),
+                ('dataset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='containers', to='librarian.Dataset')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ContainerLog',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('type', models.CharField(choices=[(b'O', b'stdout'), (b'E', b'stderr')], max_length=1)),
+                ('short_text', models.CharField(blank=True, help_text=b"Holds the log text if it's shorter than the max length.", max_length=2000)),
+                ('long_text', models.FileField(help_text=b"Holds the log text if it's longer than the max length.", upload_to=b'')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ContainerRun',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('start_time', models.DateTimeField(blank=True, help_text='Starting time', null=True, verbose_name='start time')),
+                ('end_time', models.DateTimeField(blank=True, help_text='Ending time', null=True, verbose_name='end time')),
+                ('name', models.CharField(blank=True, max_length=60)),
+                ('description', models.CharField(blank=True, max_length=1000)),
+                ('state', models.CharField(choices=[(b'N', b'New'), (b'P', b'Pending'), (b'R', b'Running'), (b'S', b'Saving'), (b'C', b'Complete'), (b'F', b'Failed'), (b'X', b'Cancelled')], default=b'N', max_length=1)),
+                ('sandbox_path', models.CharField(blank=True, max_length=4096)),
+                ('return_code', models.IntegerField(blank=True, null=True)),
+                ('is_redacted', models.BooleanField(default=False, help_text=b'True if the outputs or logs were redacted for sensitive data')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='containerapp',
+            name='memory',
+            field=models.PositiveIntegerField(default=6000, help_text=b'Megabytes of memory Slurm will allocate for this app (0 allocates all memory)', verbose_name=b'Memory required (MB)'),
+        ),
+        migrations.AddField(
+            model_name='containerapp',
+            name='threads',
+            field=models.PositiveIntegerField(default=1, help_text=b'How many threads does this app use during execution?', validators=[django.core.validators.MinValueValidator(1)], verbose_name=b'Number of threads'),
+        ),
+        migrations.AddField(
+            model_name='containerrun',
+            name='app',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='runs', to='container.ContainerApp'),
+        ),
+        migrations.AddField(
+            model_name='containerrun',
+            name='batch',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='runs', to='container.Batch'),
+        ),
+        migrations.AddField(
+            model_name='containerrun',
+            name='groups_allowed',
+            field=models.ManyToManyField(blank=True, help_text='What groups have access?', related_name='container_containerrun_has_access_to', to='auth.Group'),
+        ),
+        migrations.AddField(
+            model_name='containerrun',
+            name='stopped_by',
+            field=models.ForeignKey(blank=True, help_text=b'User that stopped this run', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='container_runs_stopped', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='containerrun',
+            name='user',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='containerrun',
+            name='users_allowed',
+            field=models.ManyToManyField(blank=True, help_text='Which users have access?', related_name='container_containerrun_has_access_to', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='containerlog',
+            name='run',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='logs', to='container.ContainerRun'),
+        ),
+        migrations.AddField(
+            model_name='containerdataset',
+            name='run',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='datasets', to='container.ContainerRun'),
+        ),
+    ]

kive/container/models.py

@@ -188,6 +188,17 @@ class ContainerApp(models.Model):
     class Meta:
         ordering = ('name',)
 
+    @property
+    def display_name(self):
+        name = self.container.tag
+        if self.name:
+            # noinspection PyTypeChecker
+            name += ' / ' + self.name
+        return name
+
+    def __str__(self):
+        return self.display_name
+
     @property
     def inputs(self):
         return self._format_arguments(ContainerArgument.INPUT)
@@ -303,15 +314,15 @@ def delete_container_file(instance, **_kwargs):
 
 class Batch(AccessControl):
     name = models.CharField(
-        "Name of this batch of container instances",
+        "Name of this batch of container runs",
         max_length=maxlengths.MAX_NAME_LENGTH,
         blank=True)
     description = models.TextField(
         max_length=maxlengths.MAX_DESCRIPTION_LENGTH,
         blank=True)
 
 
-class ContainerInstance(Stopwatch, AccessControl):
+class ContainerRun(Stopwatch, AccessControl):
     NEW = 'N'
     PENDING = 'P'
     RUNNING = 'R'
@@ -326,9 +337,9 @@ class ContainerInstance(Stopwatch, AccessControl):
               (COMPLETE, 'Complete'),
               (FAILED, 'Failed'),
               (CANCELLED, 'Cancelled'))
-    container = models.ForeignKey(Container, related_name="instances")
-    batch = models.ForeignKey(Batch, related_name="instances")
-    name = models.CharField(max_length=maxlengths.MAX_NAME_LENGTH)
+    app = models.ForeignKey(ContainerApp, related_name="runs")
+    batch = models.ForeignKey(Batch, related_name="runs", blank=True, null=True)
+    name = models.CharField(max_length=maxlengths.MAX_NAME_LENGTH, blank=True)
     description = models.CharField(max_length=maxlengths.MAX_DESCRIPTION_LENGTH,
                                    blank=True)
     state = models.CharField(max_length=1, choices=STATES, default=NEW)
@@ -337,18 +348,34 @@ class ContainerInstance(Stopwatch, AccessControl):
         blank=True)
     return_code = models.IntegerField(blank=True, null=True)
     stopped_by = models.ForeignKey(User,
-                                   help_text="User that stopped this instance",
+                                   help_text="User that stopped this run",
                                    null=True,
                                    blank=True,
-                                   related_name="instances_stopped")
+                                   related_name="container_runs_stopped")
     is_redacted = models.BooleanField(
         default=False,
         help_text="True if the outputs or logs were redacted for sensitive data")
 
+    class Meta(object):
+        ordering = ('-start_time',)
+
+    @transaction.atomic
+    def build_removal_plan(self, removal_accumulator=None):
+        """ Make a manifest of objects to remove when removing this. """
+        removal_plan = removal_accumulator or empty_removal_plan()
+        assert self not in removal_plan["ContainerRuns"]
+        removal_plan["ContainerRuns"].add(self)
+
+        return removal_plan
+
+    @transaction.atomic
+    def remove(self):
+        removal_plan = self.build_removal_plan()
+        remove_helper(removal_plan)
+
 
 class ContainerDataset(models.Model):
-    container_instance = models.ForeignKey(ContainerInstance,
-                                           related_name="datasets")
+    run = models.ForeignKey(ContainerRun, related_name="datasets")
     argument = models.ForeignKey(ContainerArgument, related_name="datasets")
     dataset = models.ForeignKey(Dataset, related_name="containers")
     name = models.CharField(
@@ -366,8 +393,7 @@ class ContainerLog(models.Model):
     TYPES = ((STDOUT, 'stdout'),
              (STDERR, 'stderr'))
     type = models.CharField(max_length=1, choices=TYPES)
-    container_instance = models.ForeignKey(ContainerInstance,
-                                           related_name="logs")
+    run = models.ForeignKey(ContainerRun, related_name="logs")
     short_text = models.CharField(
         max_length=2000,
         blank=True,