Added the Container object including adding custom containers, and prepared...

Added the Container object including adding custom containers, and prepared for the Compute object. Partially refactored the task view. Minor fixes.

images/webapp/code/rosetta/base_app/admin.py

 from django.contrib import admin
 
-from .models import Profile, LoginToken, Task
+from .models import Profile, LoginToken, Task, Container
 
 admin.site.register(Profile)
 admin.site.register(LoginToken)
 admin.site.register(Task)
+admin.site.register(Container)

images/webapp/code/rosetta/base_app/management/commands/base_app_populate.py

 from django.core.management.base import BaseCommand
 from django.contrib.auth.models import User
-from ...models import Profile
+from ...models import Profile, Container
 
 class Command(BaseCommand):
     help = 'Adds the admin superuser with \'a\' password.'
 
     def handle(self, *args, **options):
 
+        # Admin
         try:
             User.objects.get(username='admin')
             print('Not creating admin user as it already exist')
@@ -14,7 +15,8 @@ class Command(BaseCommand):
             print('Creating admin user with default password')
             admin = User.objects.create_superuser('admin', 'admin@example.com', 'admin')
             Profile.objects.create(user=admin)
-
+        
+        # Testuser
         try:
             User.objects.get(username='testuser')
             print('Not creating test user as it already exist')
@@ -22,4 +24,38 @@ class Command(BaseCommand):
             print('Creating test user with default password')
             testuser = User.objects.create_user('testuser', 'testuser@rosetta.platform', 'testpass')
             Profile.objects.create(user=testuser, authtoken='129aac94-284a-4476-953c-ffa4349b4a50')
-            
\ No newline at end of file
+
+        # public containers
+        public_containers = Container.objects.filter(user=None)
+        if public_containers:
+            print('Not creating public containers as they already exist')
+        else:
+            print('Creating public containers...')
+            
+            # MetaDesktop Docker
+            Container.objects.create(user          = None,
+                                     image         = 'rosetta/metadesktop',
+                                     type          = 'docker',
+                                     registry      = 'docker_local',
+                                     service_ports = '8590')
+
+            # Astrocook
+            Container.objects.create(user          = None,
+                                     image         = 'sarusso/astrocook:b2b819e',
+                                     type          = 'docker',
+                                     registry      = 'docker_local',
+                                     service_ports = '8590')
+
+
+
+
+
+
+
+
+
+
+
+
+
+

images/webapp/code/rosetta/base_app/models.py

@@ -55,7 +55,6 @@ class Task(models.Model):
     tid       = models.CharField('Task ID', max_length=64, blank=False, null=False)
     uuid      = models.CharField('Task UUID', max_length=36, blank=False, null=False)
     name      = models.CharField('Task name', max_length=36, blank=False, null=False)
-    container = models.CharField('Task container', max_length=36, blank=False, null=False)
     status    = models.CharField('Task status', max_length=36, blank=True, null=True)
     created   = models.DateTimeField('Created on', default=timezone.now)
     compute   = models.CharField('Task compute', max_length=36, blank=True, null=True)
@@ -64,6 +63,9 @@ class Task(models.Model):
     ip        = models.CharField('Task ip address', max_length=36, blank=True, null=True)
     tunnel_port = models.IntegerField('Task tunnel port', blank=True, null=True)
 
+    # Links
+    container    = models.ForeignKey('Container', on_delete=models.CASCADE, related_name='+')
+
     def save(self, *args, **kwargs):
         
         try:
@@ -106,7 +108,21 @@ class Task(models.Model):
         return self.uuid.split('-')[0]
 
 
+#=========================
+#  Containers
+#=========================
+class Container(models.Model):
 
+    #uuid           = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    user           = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True)
+    # If a container has no user, it will be available to anyone. Can be created, edited and deleted only by admins.
+    image         = models.CharField('Container image', max_length=255, blank=False, null=False)
+    type          = models.CharField('Container type', max_length=36, blank=False, null=False)
+    registry      = models.CharField('Container registry', max_length=255, blank=False, null=False)
+    service_ports = models.CharField('Container service ports', max_length=36, blank=True, null=True)
+    #private       = models.BooleanField('Container is private and needs auth to be pulled from the registry')
 
+    def __str__(self):
+        return str('Container of type "{}" with image "{}" from registry "{}" of user "{}"'.format(self.type, self.image, self.registry, self.user))
 
 

images/webapp/code/rosetta/base_app/templates/add_container.html

+{% load static %} 
+{% include "header.html" %}
+{% include "navigation.html" with main_path='/main/' %}
+
+<br/>
+<br/>
+
+<div class="container">
+  <div class="dashboard">
+    <div class="span8 offset2">
+      <h1>Add container</h1> 
+      <hr>
+
+      {% if not data.added %}
+
+          <h3>Configure the new container.</h3> 
+          
+          <br/>
+          
+          <form action="#" method="POST">
+          {% csrf_token %}
+
+          <table class="dashboard" style="max-width:430px">
+    
+           <tr>
+            <td><b>Type</b></td><td>
+              <select name="container_type" >
+              <option value="docker" selected>Docker</option>
+              <option value="singularity">Singularity</option>
+              </select>
+            </td>
+           </tr>
+
+           <tr>
+            <td><b>Registry</b></td><td>
+              <select name="container_registry" >
+              <option value="docker_local" selected>Local</option>
+              <option value="docker_hub">Docker Hub</option>
+              <option value="singularityhub">Singularity Hub</option>
+              </select>
+            </td>
+           </tr>
+
+           <tr>
+            <td><b>Container image</b></td>
+            <td>
+             <input type="text" name="container_image" value="" placeholder="" size="23" required />
+            </td>
+           </tr>
+
+           <tr>
+            <td><b>Service port(s)</b></td>
+            <td>
+             <input type="text" name="container_service_ports" value="" placeholder="" size="5" required />
+            </td>
+           </tr>
+
+           <tr>
+           <td colspan=2 align=center style="padding:20px">
+           <input type="submit" value="Add">
+           </td>
+           </tr>
+          </table>
+          </form>
+
+          <br/>          
+          <br/>
+          <br/>
+          <br/>
+          <br/>
+          <br/>
+          <br/>
+          <br/>
+          <br/>
+          
+
+
+
+ 
+      {% else %}
+        Ok, Container added. Go back to your <a href="/tasks">tasks list</a>.
+        
+
+      {% endif %} 
+  
+      <br/>
+      <br/>
+      <br/>
+      <br/>
+      <br/>
+      <br/>
+      
+    </div>
+  </div>
+</div>
+
+{% include "footer.html" %}
+
+
+
+
+
+

images/webapp/code/rosetta/base_app/templates/containers.html

+{% load static %} 
+{% include "header.html" %}
+{% include "navigation.html" with main_path='/main/' %}
+
+<br/>
+<br/>
+
+<div class="container">
+  <div class="dashboard">
+    <div class="span8 offset2">
+      <h1>Containers List</h1>
+      <hr/>
+
+             
+      {% for container in data.platform_containers %}
+      <table class="dashboard">
+
+       <tr>
+        <td><b>Container image</b></td>
+        <td>{{ container.image }} (platform)</td>
+       </tr>
+
+       <tr>
+        <td><b>Container type</b></td>
+        <td>{{ container.type }}</td>
+       </tr>
+
+       <tr>
+        <td><b>Container registry</b></td>
+        <td>{{ container.registry }}</td>
+       </tr>
+
+       <tr>
+        <td><b>Container service ports</b></td>
+        <td>{{ container.service_ports}}</td>
+       </tr>
+
+      </table>
+      <br />
+      {% endfor %}
+
+
+      {% for container in data.user_containers %}
+      <table class="dashboard">
+
+       <tr>
+        <td><b>Container image</b></td>
+        <td>{{ container.image }} (user)</td>
+       </tr>
+
+       <tr>
+        <td><b>Container type</b></td>
+        <td>{{ container.type }}</td>
+       </tr>
+
+       <tr>
+        <td><b>Container registry</b></td>
+        <td>{{ container.registry }}</td>
+       </tr>
+
+       <tr>
+        <td><b>Container service ports</b></td>
+        <td>{{ container.service_ports}}</td>
+       </tr>
+
+      </table>
+      <br />
+      {% endfor %}    
+      
+      <br />
+      <a href="/create_container">Add new...</a>
+      <br/>
+      <br/>
+      <br/>
+      <br/>
+      <br/>
+      <br/>
+      
+    </div>
+  </div>
+</div>
+
+{% include "footer.html" %}
+
+
+
+
+

images/webapp/code/rosetta/base_app/templates/create_task.html

@@ -11,7 +11,7 @@
       <h1>New Task</h1> 
       <hr>
 
-      {% if not data.name %}
+      {% if not data.created %}
 
           <h3>Choose a name and a type for your new Task.</h3> 
           
@@ -20,12 +20,12 @@
           <form action="/create_task/" method="POST">
           {% csrf_token %}
 
-          <table class="dashboard" style="max-width:430px">
+          <table class="dashboard" style="max-width:600px">
     
            <tr>
             <td><b>Task name </b></td>
             <td>
-             <input type="text" name="name" value="" placeholder="" size="23" required />
+             <input type="text" name="task_name" value="" placeholder="" size="23" required />
             </td>
            </tr>
 
@@ -46,21 +46,30 @@
 
            <tr>
             <td><b>Task container</b></td><td>
-              <select name="container" >
-              <option value="metadesktop" selected>Meta Desktop</option>
+              <select name="task_container_id" >
+              <!-- <option value="metadesktop" selected>Meta Desktop</option>
               <option value="astroccok">Astrocook</option>
-              <option value="gadgetviewer">Gadget Viewer</option>
-              </select>          
+              <option value="gadgetviewer">Gadget Viewer</option> -->
+              {% for container in data.platform_containers %}
+              <option value="{{container.id}}">{{container.image}} (platform)</option> -->
+              {% endfor %}
+              {% for container in data.user_containers %}
+              <option value="{{container.id}}">{{container.image}} (user)</option> -->
+              {% endfor %}              
+              
+              </select>
+              &nbsp; | <a href="/add_container">Add new...</a>          
             </td>
            </tr>
            
            <tr>
             <td><b>Computing resource</b></td><td>
-              <select name="compute" >
+              <select name="task_compute" >
               <option value="local" selected>Local</option>
               <option value="demoremote">Demo remote</option>
               <option value="demoslurm">Demo Slurm cluster</option>
-              </select>          
+              </select>
+              &nbsp; | <a href="/add_compute">Add new...</a>
             </td>
            </tr>
                       

images/webapp/code/rosetta/base_app/templates/navigation.html

@@ -22,6 +22,9 @@
 
             {% if user.is_authenticated %}
 
+            <li>
+                <a href="/containers" onclick = $("#menu-close").click(); >Containers</a>
+            </li>
             <li>
                 <a href="/tasks" onclick = $("#menu-close").click(); >Tasks</a>
             </li>

images/webapp/code/rosetta/base_app/templates/tasks.html

@@ -32,7 +32,7 @@
 
            <tr>
             <td><b>Task container</b></td>
-            <td>{{ task.container }}</td>
+            <td>{{ task.container.image }}</td>
            </tr>
 
            <tr>
@@ -45,10 +45,10 @@
             <td>{{ task.created }}</td>
            </tr>
 
-           <tr>
+           <!-- <tr>
             <td><b>Task pid</b></td>
             <td>{{ task.pid}}</td>
-           </tr>
+           </tr> -->
 
            <tr>
             <td><b>Task ip</b></td>

images/webapp/code/rosetta/base_app/views.py

@@ -20,7 +20,7 @@ from django.contrib.auth.models import User
 from django.contrib.auth import update_session_auth_hash
 
 # Project imports
-from .models import Profile, LoginToken, Task, TaskStatuses
+from .models import Profile, LoginToken, Task, TaskStatuses, Container
 from .utils import send_email, format_exception, random_username, log_user_activity, timezonize, os_shell
 
 # Setup logging
@@ -31,7 +31,8 @@ logger = logging.getLogger(__name__)
 from .exceptions import ErrorMessage, ConsistencyException
 
 # Conf
-SUPPORTED_TASK_TYPES = ['metadesktop', 'astrocook', 'gadgetviewer']
+SUPPORTED_CONTAINER_TYPES = ['docker', 'singularity']
+SUPPORTED_REGISTRIES = ['docker_local', 'docker_hub', 'signularity_hub'] 
 TASK_DATA_DIR = "/data"
 
 #=========================
@@ -566,27 +567,32 @@ def create_task(request):
     data['user']    = request.user
     data['profile'] = Profile.objects.get(user=request.user)
     data['title']   = 'New Task'
-    data['name']    = request.POST.get('name',None)
     
-    if data['name']:
-        
-        # Type
-        data['container'] = request.POST.get('container', None)
-        if not data['container']:
-            data['error'] = 'No container given'
-            return render(request, 'error.html', {'data': data})
+    # Get containers configured on the platform, both private to this user and public
+    data['platform_containers'] = Container.objects.filter(user=request.user)
+    data['user_containers'] = Container.objects.filter(user=None)
 
-        if not data['container'] in SUPPORTED_TASK_TYPES:
-            data['error'] = 'No valid task container'
-            return render(request, 'error.html', {'data': data})
-        
-        compute = request.POST.get('compute', None)
+    # Task name if any
+    task_name = request.POST.get('task_name', None)
 
-        logger.debug(compute)
+    if task_name:
+        
+        # Task container
+        task_container_id = request.POST.get('task_container_id', None)
 
-        if compute not in ['local', 'demoremote']:
-            data['error'] = 'Unknown compute resource "{}'.format(compute)
-            return render(request, 'error.html', {'data': data})
+        # Get the container object, first try as public and then as private
+        try:
+            task_container = Container.objects.get(id=task_container_id, user=None)
+        except Container.DoesNotExist:
+            try:
+                task_container =  Container.objects.get(id=task_container_id, user=request.user)
+            except Container.DoesNotExist:
+                raise Exception('Consistency error, container with id "{}" does not exists or user "{}" does not have access rights'.format(task_container_id, request.user.email))
+
+        # Compute
+        task_compute = request.POST.get('task_compute', None)
+        if task_compute not in ['local', 'demoremote']:
+            raise ErrorMessage('Unknown compute resource "{}')
         
         # Generate the task uuid
         str_uuid = str(uuid.uuid4())
@@ -595,14 +601,15 @@ def create_task(request):
         # Create the task object
         task = Task.objects.create(uuid      = str_uuid,
                                    user      = request.user,
-                                   name      = data['name'],
+                                   name      = task_name,
                                    status    = TaskStatuses.created,
-                                   container = data['container'],
-                                   compute   = compute)
+                                   container = task_container,
+                                   compute   = task_compute)
         
+                
         # Actually start tasks
         try:
-            if compute == 'local':            
+            if task_compute == 'local':            
 
                 # Get our ip address             
                 #import netifaces
@@ -615,13 +622,15 @@ def create_task(request):
                 # Data volume
                 run_command += ' -v {}/task-{}:/data'.format(TASK_DATA_DIR, str_shortuuid)
     
-                # Host name, image entry command
-                task_container = 'task-{}'.format(data['container'])
-                run_command += ' -h task-{} -d -t localhost:5000/rosetta/metadesktop'.format(str_shortuuid, task_container)
-    
-                # Create the model
-                task = Task.objects.create(user=request.user, name=data['name'], status=TaskStatuses.created, container=data['container'])
+                # Set registry string
+                if task.container.registry == 'local':
+                    registry_string = 'localhost:5000/'
+                else:
+                    registry_string  = ''
                     
+                # Host name, image entry command
+                run_command += ' -h task-{} -d -t {}{}'.format(str_shortuuid, registry_string, task.container.image)
+      
                 # Run the task Debug
                 logger.debug('Running new task with command="{}"'.format(run_command))
                 out = os_shell(run_command, capture=True)
@@ -647,7 +656,7 @@ def create_task(request):
                     # Save
                     task.save()
 
-            elif compute == 'demoremote':
+            elif task_compute == 'demoremote':
                 logger.debug('Using Demo Remote as compute resource')
 
 
@@ -680,19 +689,125 @@ def create_task(request):
                 
 
             else:
-                raise Exception('Consistency exception: invalid compute resource "{}'.format(compute))
+                raise Exception('Consistency exception: invalid compute resource "{}'.format(task_compute))
 
         except Exception as e:
             data['error'] = 'Error in creating new Task.'
             logger.error(e)
             return render(request, 'error.html', {'data': data})
     
+        # Set created switch
+        data['created'] = True
 
     return render(request, 'create_task.html', {'data': data})
 
 
 
 
+#=========================
+#  Containers
+#=========================
+
+@private_view
+def containers(request):
+
+    # Init data
+    data={}
+    data['user']    = request.user
+    data['profile'] = Profile.objects.get(user=request.user)
+    data['title']   = 'Add compute'
+    data['name']    = request.POST.get('name',None)
+    
+    # Get containers configured on the platform, both private to this user and public
+    data['platform_containers'] = Container.objects.filter(user=request.user)
+    data['user_containers'] = Container.objects.filter(user=None)
+
+    return render(request, 'containers.html', {'data': data})
+
+
+
+#=========================
+#  Add Container view
+#=========================
+
+@private_view
+def add_container(request):
+
+    # Init data
+    data={}
+    data['user']    = request.user
+    data['profile'] = Profile.objects.get(user=request.user)
+    data['title']   = 'Add container'
+    
+    # Container image if any
+    container_image = request.POST.get('container_image',None)
+
+    if container_image:
+        
+        # Container type
+        container_type = request.POST.get('container_type', None)
+        if not container_type:
+            raise ErrorMessage('No container type given')
+        if not container_type in SUPPORTED_CONTAINER_TYPES:
+            raise ErrorMessage('No valid container type')
+
+        # Container registry
+        container_registry = request.POST.get('container_registry', None)
+        if not container_registry:
+            raise ErrorMessage('No registry type given')
+        if not container_registry in SUPPORTED_REGISTRIES:
+            raise ErrorMessage('No valid registry')
+
+        # Container service ports
+        container_service_ports = request.POST.get('container_service_ports', None)
+
+        # Log
+        logger.debug('Creating new container object with image="{}", type="{}", registry="{}", service_ports="{}"'.format(container_image, container_type, container_registry, container_service_ports))
+
+        # Create
+        Container.objects.create(user          = request.user,
+                                 image         = container_image,
+                                 type          = container_type,
+                                 registry      = container_registry,
+                                 service_ports = container_service_ports)
+        # Set added switch
+        data['added'] = True
+
+    return render(request, 'add_container.html', {'data': data})
+
+
+
+#=========================
+#  Computes view
+#=========================
+
+@private_view
+def computes(request):
+
+    # Init data
+    data={}
+    data['user']    = request.user
+    data['profile'] = Profile.objects.get(user=request.user)
+    data['title']   = 'Add compute'
+    data['name']    = request.POST.get('name',None)
+    
+
+    return render(request,  'computes.html', {'data': data})
+
+#=========================
+#  Add Compute view
+#=========================
+
+@private_view
+def add_compute(request):
 
+    # Init data
+    data={}
+    data['user']    = request.user
+    data['profile'] = Profile.objects.get(user=request.user)
+    data['title']   = 'Add compute'
+    data['name']    = request.POST.get('name',None)
+    
 
+    return render(request, 'add_compute.html', {'data': data})
 

images/webapp/code/rosetta/urls.py

@@ -42,6 +42,10 @@ urlpatterns = [
     url(r'^account/$', base_app_views.account),
     url(r'^tasks/$', base_app_views.tasks),
     url(r'^create_task/$', base_app_views.create_task),
+    url(r'^computes/$', base_app_views.computes),
+    url(r'^add_compute/$', base_app_views.add_compute),
+    url(r'^containers/$', base_app_views.containers),
+    url(r'^add_container/$', base_app_views.add_container),
 
     # Modules
     path('admin/', admin.site.urls),