Launch singularity with app name for #739.

Also set Slurm options like memory and priority.

kive/container/management/commands/runcontainer.py

@@ -72,28 +72,36 @@ def create_sandbox(self, run):
         run.state = ContainerRun.RUNNING
 
     def run_container(self, run):
-        container_path = run.app.container.file.path
         logs_path = os.path.join(run.sandbox_path, 'logs')
         stdout_path = os.path.join(logs_path, 'stdout.txt')
         stderr_path = os.path.join(logs_path, 'stderr.txt')
+        command = self.build_command(run)
+        with open(stdout_path, 'w') as stdout, open(stderr_path, 'w') as stderr:
+            run.return_code = call(command, stdout=stdout, stderr=stderr)
+
+        run.state = ContainerRun.SAVING
+
+    def build_command(self, run):
+        container_path = run.app.container.file.path
         input_path = os.path.join(run.sandbox_path, 'input')
         output_path = os.path.join(run.sandbox_path, 'output')
         command = ['singularity',
                    'run',
+                   '--contain',
                    '-B',
                    '{}:/mnt/input,{}:/mnt/output'.format(input_path,
-                                                         output_path),
-                   container_path]
+                                                         output_path)]
+        if run.app.name:
+            command.append('--app')
+            command.append(run.app.name)
+        command.append(container_path)
         for argument in run.app.arguments.all():
             if argument.type == ContainerArgument.INPUT:
                 folder = '/mnt/input'
             else:
                 folder = '/mnt/output'
             command.append(os.path.join(folder, argument.name))
-        with open(stdout_path, 'w') as stdout, open(stderr_path, 'w') as stderr:
-            run.return_code = call(command, stdout=stdout, stderr=stderr)
-
-        run.state = ContainerRun.SAVING
+        return command
 
     def save_outputs(self, run):
         output_path = os.path.join(run.sandbox_path, 'output')

kive/container/models.py

@@ -418,14 +418,27 @@ def schedule(self):
             if ex.errno != errno.EEXIST:
                 raise
 
+        check_call(self.build_slurm_command(sandbox_root,
+                                            settings.SLURM_QUEUES))
+
+    def build_slurm_command(self, sandbox_root, slurm_queues=None):
         sandbox_prefix = os.path.join(sandbox_root,
                                       self.get_sandbox_prefix())
         slurm_prefix = sandbox_prefix + '_job%J_node%N_'
-
-        check_call(['sbatch',
-                    '--output', slurm_prefix + 'stdout.txt',
-                    '--error', slurm_prefix + 'stderr.txt',
-                    'manage.py', 'runcontainer', str(self.pk)])
+        job_name = 'r{} {}'.format(self.pk,
+                                   self.app.name or
+                                   self.app.container.family.name)
+        command = ['sbatch',
+                   '-J', job_name,
+                   '--output', slurm_prefix + 'stdout.txt',
+                   '--error', slurm_prefix + 'stderr.txt',
+                   '-c', str(self.app.threads),
+                   '--mem', str(self.app.memory)]
+        if slurm_queues is not None:
+            kive_name, slurm_name = slurm_queues[self.priority]
+            command.extend(['-p', slurm_name])
+        command.extend(['manage.py', 'runcontainer', str(self.pk)])
+        return command
 
     def get_sandbox_prefix(self):
         return 'user{}_run{}_'.format(self.user.username, self.pk)

kive/container/templates/container/containerapp_form.html

@@ -17,7 +17,7 @@
 
 {% block content %}
 
-<a href="/container_update/{{ container_id }}" rel="prev">return to family</a>
+<a href="/container_update/{{ container_id }}" rel="prev">return to container</a>
 
 <h2>Container App</h2>
 
@@ -40,4 +40,6 @@ <h2>Container App</h2>
     <input type="submit" value="Submit" />
 </form>
 
+<p><a href="/container_inputs?app={{ object.id }}">Launch this app</a></p>
+
 {% endblock %}

kive/container/templates/container/containerinput_list.html

@@ -159,7 +159,7 @@ <h4>Filter results</h4>
 </div>
 
 <div id="scroll_content">
-    <h4 id="app_name">{{app.container.family}} {{ app }}</h4>
+    <h4 id="app_name">{{ app }}</h4>
     <table id="dataset_input_table">
 
         <thead>

kive/container/tests.py

@@ -2,18 +2,21 @@
 from __future__ import unicode_literals
 
 import os
+from datetime import datetime
 
 from django.contrib.auth.models import User
 from django.core.management import call_command
 from django.core.management.base import CommandError
 from django.test import TestCase, skipIfDBFeature
 from django.test.client import Client
 from django.urls import reverse, resolve
+from django.utils.timezone import make_aware, utc
 from rest_framework.reverse import reverse as rest_reverse
 from rest_framework import status
 from rest_framework.test import force_authenticate
 
-from container.models import ContainerFamily, ContainerApp, Container, ContainerRun, ContainerArgument, Batch
+from container.models import ContainerFamily, ContainerApp, Container, \
+    ContainerRun, ContainerArgument, Batch, ContainerLog
 from kive.tests import BaseTestCases, install_fixture_files
 from librarian.models import Dataset
 
@@ -193,6 +196,58 @@ def test_removal(self):
         self.assertEquals(end_count, start_count - 1)
 
 
+@skipIfDBFeature('is_mocked')
+class ContainerRunTests(TestCase):
+    fixtures = ['container_run']
+
+    def test_no_outputs(self):
+        run = ContainerRun.objects.get(id=1)
+        expected_entries = [dict(created=make_aware(datetime(2000, 1, 1), utc),
+                                 name='names_csv',
+                                 size='30\xa0bytes',
+                                 type='Input',
+                                 url='/dataset_view/1')]
+        client = Client()
+        client.force_login(run.user)
+        response = client.get(reverse('container_run_detail',
+                                      kwargs=dict(pk=run.pk)))
+
+        self.assertEqual('New', response.context['state_name'])
+        self.assertListEqual(expected_entries, response.context['data_entries'])
+
+    def test_outputs(self):
+        run = ContainerRun.objects.get(id=1)
+        run.state = ContainerRun.COMPLETE
+        run.end_time = make_aware(datetime(2000, 1, 2), utc)
+        run.save()
+        dataset = Dataset.objects.first()
+        argument = run.app.arguments.get(name='greetings_csv')
+        run.datasets.create(argument=argument, dataset=dataset)
+        log = run.logs.create(short_text='Job completed.', type=ContainerLog.STDERR)
+        expected_entries = [dict(created=make_aware(datetime(2000, 1, 1), utc),
+                                 name='names_csv',
+                                 size='30\xa0bytes',
+                                 type='Input',
+                                 url='/dataset_view/1'),
+                            dict(created=make_aware(datetime(2000, 1, 2), utc),
+                                 name='stderr',
+                                 size='14\xa0bytes',
+                                 type='Log',
+                                 url='/container_logs/{}/'.format(log.id)),
+                            dict(created=make_aware(datetime(2000, 1, 1), utc),
+                                 name='greetings_csv',
+                                 size='30\xa0bytes',
+                                 type='Output',
+                                 url='/dataset_view/1')]
+        client = Client()
+        client.force_login(run.user)
+        response = client.get(reverse('container_run_detail',
+                                      kwargs=dict(pk=run.pk)))
+
+        self.assertEqual('Complete', response.context['state_name'])
+        self.assertListEqual(expected_entries, response.context['data_entries'])
+
+
 @skipIfDBFeature('is_mocked')
 class BatchApiTests(BaseTestCases.ApiTestCase):
     def setUp(self):

kive/container/tests_mock.py

@@ -1,4 +1,5 @@
 import os
+from argparse import Namespace
 from io import BytesIO
 
 from django.contrib.auth.models import User
@@ -10,8 +11,9 @@
 from rest_framework.test import force_authenticate
 
 from container.ajax import ContainerAppViewSet
+from container.management.commands import runcontainer
 from container.models import Container, ContainerFamily, ContainerApp, \
-    ContainerArgument, ContainerFileFormField
+    ContainerArgument, ContainerFileFormField, ContainerRun
 from kive.tests import BaseTestCases
 from metadata.models import KiveUser
 
@@ -385,3 +387,146 @@ def test_filter_unknown(self):
 
         self.assertEquals({u'detail': u'Unknown filter key: bogus'},
                           response.data)
+
+
+@mocked_relations(ContainerRun, ContainerApp, ContainerArgument)
+class ContainerRunMockTests(TestCase):
+    def test_slurm_command_default_app(self):
+        run = ContainerRun(pk=99)
+        run.user = User(username='bob')
+        run.app = ContainerApp()
+        run.app.container = Container()
+        run.app.container.family = ContainerFamily(name='my container')
+        sandbox_root = '/Sandboxes'
+        expected_command = [
+            'sbatch',
+            '-J', 'r99 my container',
+            '--output', '/Sandboxes/userbob_run99__job%J_node%N_stdout.txt',
+            '--error', '/Sandboxes/userbob_run99__job%J_node%N_stderr.txt',
+            '-c', '1',
+            '--mem', '6000',
+            'manage.py',
+            'runcontainer',
+            '99']
+
+        command = run.build_slurm_command(sandbox_root)
+
+        self.assertListEqual(expected_command, command)
+
+    def test_slurm_command_named_app(self):
+        run = ContainerRun(pk=99)
+        run.user = User(username='bob')
+        run.app = ContainerApp(name='my_app')
+        run.app.container = Container()
+        run.app.container.family = ContainerFamily(name='my container')
+        sandbox_root = '/Sandboxes'
+        expected_command = [
+            'sbatch',
+            '-J', 'r99 my_app',
+            '--output', '/Sandboxes/userbob_run99__job%J_node%N_stdout.txt',
+            '--error', '/Sandboxes/userbob_run99__job%J_node%N_stderr.txt',
+            '-c', '1',
+            '--mem', '6000',
+            'manage.py',
+            'runcontainer',
+            '99']
+
+        command = run.build_slurm_command(sandbox_root)
+
+        self.assertListEqual(expected_command, command)
+
+    def test_slurm_command_custom_memory(self):
+        run = ContainerRun(pk=99)
+        run.user = User(username='bob')
+        run.app = ContainerApp(threads=3, memory=100)
+        run.app.container = Container()
+        run.app.container.family = ContainerFamily(name='my container')
+        sandbox_root = '/Sandboxes'
+        expected_command = [
+            'sbatch',
+            '-J', 'r99 my container',
+            '--output', '/Sandboxes/userbob_run99__job%J_node%N_stdout.txt',
+            '--error', '/Sandboxes/userbob_run99__job%J_node%N_stderr.txt',
+            '-c', '3',
+            '--mem', '100',
+            'manage.py',
+            'runcontainer',
+            '99']
+
+        command = run.build_slurm_command(sandbox_root)
+
+        self.assertListEqual(expected_command, command)
+
+    def test_slurm_command_priority(self):
+        run = ContainerRun(pk=99)
+        run.user = User(username='bob')
+        run.app = ContainerApp()
+        run.app.container = Container()
+        run.app.container.family = ContainerFamily(name='my container')
+        slurm_queues = (('low', 'kive-low'),
+                        ('medium', 'kive-medium'),
+                        ('high', 'kive-high'))
+        run.priority = 2
+        sandbox_root = '/Sandboxes'
+        expected_command = [
+            'sbatch',
+            '-J', 'r99 my container',
+            '--output', '/Sandboxes/userbob_run99__job%J_node%N_stdout.txt',
+            '--error', '/Sandboxes/userbob_run99__job%J_node%N_stderr.txt',
+            '-c', '1',
+            '--mem', '6000',
+            '-p', 'kive-high',
+            'manage.py',
+            'runcontainer',
+            '99']
+
+        command = run.build_slurm_command(sandbox_root, slurm_queues)
+
+        self.assertListEqual(expected_command, command)
+
+
+@mocked_relations(ContainerRun, ContainerApp, ContainerArgument)
+class RunContainerMockTests(TestCase):
+    def build_run(self):
+        run = ContainerRun()
+        run.app = ContainerApp()
+        run.app.container = Container()
+        run.app.container.file = Namespace(path='/tmp/foo.simg')
+        run.sandbox_path = '/tmp/box23'
+        run.app.arguments.create(type=ContainerArgument.INPUT, name='in_csv')
+        run.app.arguments.create(type=ContainerArgument.OUTPUT, name='out_csv')
+        return run
+
+    def test_default_app(self):
+        run = self.build_run()
+        handler = runcontainer.Command()
+        expected_command = [
+            'singularity',
+            'run',
+            '--contain',
+            '-B', '/tmp/box23/input:/mnt/input,/tmp/box23/output:/mnt/output',
+            '/tmp/foo.simg',
+            '/mnt/input/in_csv',
+            '/mnt/output/out_csv']
+
+        command = handler.build_command(run)
+
+        self.assertListEqual(expected_command, command)
+
+    def test_named_app(self):
+        run = self.build_run()
+        run.app.name = 'other_app'
+        handler = runcontainer.Command()
+        expected_command = [
+            'singularity',
+            'run',
+            '--contain',
+            '-B', '/tmp/box23/input:/mnt/input,/tmp/box23/output:/mnt/output',
+            '--app', 'other_app',
+            '/tmp/foo.simg',
+            '/mnt/input/in_csv',
+            '/mnt/output/out_csv']
+
+        command = handler.build_command(run)
+
+        self.assertListEqual(expected_command, command)

kive/container/views.py

@@ -256,9 +256,9 @@ def get_context_data(self, **kwargs):
         for run_dataset in self.object.datasets.all():
             data_entries.append(dict(
                 type=type_names[run_dataset.argument.type],
-                url=run_dataset.dataset.get_view_url,
+                url=run_dataset.dataset.get_view_url(),
                 name=run_dataset.argument.name,
-                size=run_dataset.dataset.get_formatted_filesize,
+                size=run_dataset.dataset.get_formatted_filesize(),
                 created=run_dataset.dataset.date_created))
             if run_dataset.argument.type == ContainerArgument.INPUT:
                 input_count += 1
@@ -268,7 +268,7 @@ def get_context_data(self, **kwargs):
                 type='Log',
                 url=log.get_absolute_url(),
                 name=log_names[log.type],
-                size=len(log.short_text),
+                size=filesizeformat(len(log.short_text)),
                 created=self.object.end_time))
         context['data_entries'] = data_entries
         return context

kive/kive/settings_default.py

@@ -323,7 +323,7 @@
 
 # A list, ordered from lowest-priority to highest-priority, of Slurm queues to
 # be used by Kive.  Fill these in with the names of the queues as you have them
-# defined on your system.  The tuples contain the name Slurm will use for the
+# defined on your system.  The tuples contain the name Kive will use for the
 # queue as well as the actual Slurm name.
 SLURM_QUEUES = [
     ("Low priority", "LOW_PRIO"),