Display stdout and stderr from container runs for #739.

kive/container/management/commands/runcontainer.py

@@ -1,15 +1,17 @@
 from __future__ import print_function
+
+import errno
 import logging
 import os
 import shutil
 from subprocess import call
 from tempfile import mkdtemp
 
-import errno
 from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
 
-from container.models import ContainerRun, ContainerArgument
+from container.models import ContainerRun, ContainerArgument, ContainerLog
 from librarian.models import Dataset
 
 logger = logging.getLogger(__name__)
@@ -50,6 +52,11 @@ def update_state(self, run_id, old_state, new_state):
 
     def create_sandbox(self, run):
         sandbox_root = os.path.join(settings.MEDIA_ROOT, settings.SANDBOX_PATH)
+        try:
+            os.mkdir(sandbox_root)
+        except OSError as ex:
+            if ex.errno != errno.EEXIST:
+                raise
         prefix = 'user{}_run{}_'.format(run.user.username, run.pk)
         run.sandbox_path = mkdtemp(prefix=prefix, dir=sandbox_root)
 
@@ -97,5 +104,16 @@ def save_outputs(self, run):
                                              user=run.user)
             run.datasets.create(dataset=dataset,
                                 argument=argument)
-
-        run.state = ContainerRun.COMPLETE
+        logs_path = os.path.join(run.sandbox_path, 'logs')
+        for file_name, log_type in (('stdout.txt', ContainerLog.STDOUT),
+                                    ('stderr.txt', ContainerLog.STDERR)):
+            # noinspection PyUnresolvedReferences,PyProtectedMember
+            chunk_size = ContainerLog._meta.get_field('short_text').max_length
+            with open(os.path.join(logs_path, file_name)) as f:
+                chunk = f.read(chunk_size)
+            run.logs.create(type=log_type, short_text=chunk)
+
+        run.state = (ContainerRun.COMPLETE
+                     if run.return_code == 0
+                     else ContainerRun.FAILED)
+        run.end_time = timezone.now()

kive/container/models.py

@@ -472,3 +472,6 @@ class ContainerLog(models.Model):
         help_text="Holds the log text if it's shorter than the max length.")
     long_text = models.FileField(
         help_text="Holds the log text if it's longer than the max length.")
+
+    def get_absolute_url(self):
+        return reverse('container_log_detail', kwargs=dict(pk=self.pk))

kive/container/templates/container/containerlog_detail.html

@@ -0,0 +1,39 @@
+<!-- Django template -->
+
+{% extends "portal/base.html" %}
+
+{% block title %}Container Log{% endblock %}
+
+{% block javascript %}
+    <script src="/static/portal/jquery-2.0.3.min.js"></script>
+    <script src="/static/portal/noxss.js"></script>
+    <script src="/static/portal/helptext.js"></script>
+{% endblock %}
+
+{% block stylesheets %}
+    <link rel="stylesheet" href="/static/librarian/dataset_view.css"/>
+{% endblock %}
+
+{% block widget_media %}
+{{ form.media }}
+{% endblock %}
+
+{% block content %}
+
+<a href="{{ object.run.get_absolute_url }}" rel="prev">return to Container Run</a>
+
+<h2>Container Log -
+{% if object.type == 'E' %}
+    stderr
+{% else %}
+    stdout
+{% endif %}
+</h2>
+
+<div class="dataset_contents">
+<pre>
+{{ object.short_text }}
+</pre>
+</div>
+
+{% endblock %}

kive/container/templates/container/containerrun_form.html

@@ -156,11 +156,11 @@ <h4>Groups allowed</h4>
         <th>Size</th>
         <th>Date</th>
     </tr>
-{% for run_dataset in object.datasets.all %}
-    <tr><td>{{ run_dataset.argument.type }}</td>
-        <td><a href="{{run_dataset.dataset.get_view_url}}">{{ run_dataset.argument.name }}</a></td>
-        <td>{{ run_dataset.dataset.get_formatted_filesize }}</td>
-        <td>{{ run_dataset.dataset.date_created }}</td>
+{% for data_entry in data_entries %}
+    <tr><td>{{ data_entry.type }}</td>
+        <td><a href="{{ data_entry.url }}">{{ data_entry.name }}</a></td>
+        <td>{{ data_entry.size }}</td>
+        <td>{{ data_entry.created }}</td>
     </tr>
 {% endfor %}
 </table>

kive/container/views.py

@@ -12,13 +12,14 @@
 from django.urls import reverse
 from django.utils.decorators import method_decorator
 from django.views.generic.base import TemplateView
+from django.views.generic.detail import DetailView
 from django.views.generic.edit import CreateView, UpdateView, ModelFormMixin
 import rest_framework.reverse
 
 from container.forms import ContainerFamilyForm, ContainerForm, \
     ContainerUpdateForm, ContainerAppForm, ContainerRunForm, BatchForm
 from container.models import ContainerFamily, Container, ContainerApp, \
-    ContainerRun, ContainerArgument
+    ContainerRun, ContainerArgument, ContainerLog
 from file_access_utils import compute_md5
 from portal.views import developer_check, AdminViewMixin
 
@@ -249,7 +250,33 @@ def get_context_data(self, **kwargs):
         context['is_dev'] = developer_check(self.request.user)
         state_names = dict(ContainerRun.STATES)
         context['state_name'] = state_names.get(self.object.state)
+        data_entries = []
+        type_names = dict(ContainerArgument.TYPES)
+        input_count = 0
+        for run_dataset in self.object.datasets.all():
+            data_entries.append(dict(
+                type=type_names[run_dataset.argument.type],
+                url=run_dataset.dataset.get_view_url,
+                name=run_dataset.argument.name,
+                size=run_dataset.dataset.get_formatted_filesize,
+                created=run_dataset.dataset.date_created))
+            if run_dataset.argument.type == ContainerArgument.INPUT:
+                input_count += 1
+        log_names = dict(ContainerLog.TYPES)
+        for log in self.object.logs.order_by('type'):
+            data_entries.insert(input_count, dict(
+                type='Log',
+                url=log.get_absolute_url(),
+                name=log_names[log.type],
+                size=len(log.short_text),
+                created=self.object.end_time))
+        context['data_entries'] = data_entries
         return context
 
     def get_success_url(self):
         return reverse('container_runs')
+
+
+@method_decorator(login_required, name='dispatch')
+class ContainerLogDetail(DetailView):
+    model = ContainerLog

kive/fleet/tests_slurmlib.py

@@ -16,7 +16,7 @@
 import fleet.slurmlib as slurmlib
 from django.conf import settings
 
-from django.test import TestCase
+from django.test import TestCase, skipIfDBFeature
 
 # NOTE: Here, select which SlurmScheduler to test.
 # we select the DummySlurmScheduler by default, so that the automatic tests
@@ -87,6 +87,7 @@ def get_accounting_info(jhandles=None, sched_cls=None):
     return curstates
 
 
+@skipIfDBFeature('is_mocked')  # Doesn't use the database, but this test is slow.
 class SlurmDummyTests(TestCase):
     def setUp(self):
         self.addTypeEqualityFunc(str, self.assertMultiLineEqual)

kive/kive/urls.py

@@ -108,6 +108,9 @@
     url(r'^container_runs/(?P<pk>\d+)/$',
         container.views.ContainerRunUpdate.as_view(),
         name='container_run_detail'),
+    url(r'^container_logs/(?P<pk>\d+)/$',
+        container.views.ContainerLogDetail.as_view(),
+        name='container_log_detail'),
 
     url(r'^datatypes$', metadata.views.datatypes, name='datatypes'),
     url(r'^datatypes/(?P<id>\d+)/$', metadata.views.datatype_detail, name='datatype_detail'),