Parametrized remote tasks host and access key. Added container name. Global...

Parametrized remote tasks host and access key. Added container name. Global refactoring and cleanup.

services/webapp/code/rosetta/base_app/admin.py

 from django.contrib import admin
 
-from .models import Profile, LoginToken, Task, Container
+from .models import Profile, LoginToken, Task, Container, Computing, ComputingSysConf, ComputingUserConf
 
 admin.site.register(Profile)
 admin.site.register(LoginToken)
 admin.site.register(Task)
 admin.site.register(Container)
+admin.site.register(Computing)
+admin.site.register(ComputingSysConf)
+admin.site.register(ComputingUserConf)
\ No newline at end of file

services/webapp/code/rosetta/base_app/api.py

 import logging
- 
-# Django imports
 from django.http import HttpResponse
 from django.utils import timezone
 from django.contrib.auth import authenticate, login, logout
 from django.contrib.auth.models import User, Group
-
 from rest_framework.response import Response
 from rest_framework import status, serializers, viewsets
 from rest_framework.views import APIView
-
- 
-# Project imports
-from rosetta.common import format_exception
-from rosetta.base_app.models import Profile
+from .utils import format_exception
+from .models import Profile
  
 # Setup logging
 logger = logging.getLogger(__name__)

services/webapp/code/rosetta/base_app/decorators.py

+# Imports
+import inspect
+from django.conf import settings
+from django.shortcuts import render
+from django.http import HttpResponse
+from .utils import format_exception, log_user_activity
+from .exceptions import ErrorMessage, ConsistencyException
+
+# Setup logging
+import logging
+logger = logging.getLogger(__name__)
+
+# Conf
+SUPPORTED_CONTAINER_TYPES = ['docker', 'singularity']
+SUPPORTED_REGISTRIES = ['docker_local', 'docker_hub', 'singularity_hub']
+UNSUPPORTED_TYPES_VS_REGISTRIES = ['docker:singularity_hub']
+
+
+# Public view
+def public_view(wrapped_view):
+    def public_view_wrapper(request, *argv, **kwargs):
+        # -------------- START Public/private common code --------------
+        try:
+            log_user_activity("DEBUG", "Called", request, wrapped_view.__name__)
+
+            # Try to get the templates from view kwargs
+            # Todo: Python3 compatibility: https://stackoverflow.com/questions/2677185/how-can-i-read-a-functions-signature-including-default-argument-values
+
+            argSpec=inspect.getargspec(wrapped_view)
+
+            if 'template' in argSpec.args:
+                template = argSpec.defaults[0]
+            else:
+                template = None
+
+            # Call wrapped view
+            data = wrapped_view(request, *argv, **kwargs)
+
+            if not isinstance(data, HttpResponse):
+                if template:
+                    #logger.debug('using template + data ("{}","{}")'.format(template,data))
+                    return render(request, template, {'data': data})
+                else:
+                    raise ConsistencyException('Got plain "data" output but no template defined in view')
+            else:
+                #logger.debug('using returned httpresponse')
+                return data
+
+        except Exception as e:
+            if isinstance(e, ErrorMessage):
+                error_text = str(e)
+            else:
+
+                # Raise te exception if we are in debug mode
+                if settings.DEBUG:
+                    raise
+
+                # Otherwise,
+                else:
+
+                    # first log the exception
+                    logger.error(format_exception(e))
+
+                    # and then mask it.
+                    error_text = 'something went wrong'
+
+            data = {'user': request.user,
+                    'title': 'Error',
+                    'error' : 'Error: "{}"'.format(error_text)}
+
+            if template:
+                return render(request, template, {'data': data})
+            else:
+                return render(request, 'error.html', {'data': data})
+        # --------------  END Public/private common code --------------
+    return public_view_wrapper
+
+# Private view
+def private_view(wrapped_view):
+    def private_view_wrapper(request, *argv, **kwargs):
+        if request.user.is_authenticated:
+            # -------------- START Public/private common code --------------
+            log_user_activity("DEBUG", "Called", request, wrapped_view.__name__)
+            try:
+
+                # Try to get the templates from view kwargs
+                # Todo: Python3 compatibility: https://stackoverflow.com/questions/2677185/how-can-i-read-a-functions-signature-including-default-argument-values
+
+                argSpec=inspect.getargspec(wrapped_view)
+
+                if 'template' in argSpec.args:
+                    template = argSpec.defaults[0]
+                else:
+                    template = None
+
+                # Call wrapped view
+                data = wrapped_view(request, *argv, **kwargs)
+
+                if not isinstance(data, HttpResponse):
+                    if template:
+                        #logger.debug('using template + data ("{}","{}")'.format(template,data))
+                        return render(request, template, {'data': data})
+                    else:
+                        raise ConsistencyException('Got plain "data" output but no template defined in view')
+                else:
+                    #logger.debug('using returned httpresponse')
+                    return data
+
+            except Exception as e:
+                if isinstance(e, ErrorMessage):
+                    error_text = str(e)
+                else:
+
+                    # Raise te exception if we are in debug mode
+                    if settings.DEBUG:
+                        raise
+
+                    # Otherwise,
+                    else:
+
+                        # first log the exception
+                        logger.error(format_exception(e))
+
+                        # and then mask it.
+                        error_text = 'something went wrong'
+
+                data = {'user': request.user,
+                        'title': 'Error',
+                        'error' : 'Error: "{}"'.format(error_text)}
+
+                if template:
+                    return render(request, template, {'data': data})
+                else:
+                    return render(request, 'error.html', {'data': data})
+            # --------------  END  Public/private common code --------------
+
+        else:
+            log_user_activity("DEBUG", "Redirecting to login since not authenticated", request)
+            return HttpResponseRedirect('/login')
+    return private_view_wrapper

services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py

@@ -42,13 +42,23 @@ class Command(BaseCommand):
             
             # MetaDesktop Docker
             Container.objects.create(user          = None,
+                                     name          = 'MetaDesktop latest',
                                      image         = 'rosetta/metadesktop',
                                      type          = 'docker',
                                      registry      = 'docker_local',
                                      service_ports = '8590')
 
+            # MetaDesktop Singularity
+            Container.objects.create(user          = None,
+                                     name          = 'MetaDesktop latest',
+                                     image         = 'rosetta/metadesktop',
+                                     type          = 'singularity',
+                                     registry      = 'docker_local',
+                                     service_ports = '8590')
+
             # Astrocook
             Container.objects.create(user          = None,
+                                     name          = 'Astrocook b2b819e',
                                      image         = 'sarusso/astrocook:b2b819e',
                                      type          = 'docker',
                                      registry      = 'docker_local',
@@ -64,6 +74,7 @@ class Command(BaseCommand):
             
             # JuPyter
             Container.objects.create(user          = testuser,
+                                     name          = 'Jupyter Notebook latest',
                                      image         = 'jupyter/base-notebook',
                                      type          = 'docker',
                                      registry      = 'docker_hub',
@@ -90,9 +101,15 @@ class Command(BaseCommand):
     
             # Create demo remote sys computing conf
             ComputingSysConf.objects.create(computing = demo_remote_computing,
-                                            data      = {'host': 'slurmclusterworker-one',
-                                                         'user': 'rosetta',
-                                                         'identity': 'privkey?'})
+                                            data      = {'host': 'slurmclusterworker-one'})
+
+            # Create demo remote user computing conf
+            ComputingUserConf.objects.create(user      = testuser,
+                                             computing = demo_remote_computing,
+                                             data      = {'user': 'testuser',
+                                                          'id_rsa': '/rosetta/.ssh/id_rsa',
+                                                          'id_rsa.pub': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2n4wiLiRmE1sla5+w0IW3wwPW/mqhhkm7IyCBS+rGTgnts7xsWcxobvamNdD6KSLNnjFZbBb7Yaf/BvWrwQgdqIFVU3gRWHYzoU6js+lKtBjd0e2DAVGivWCKEkSGLx7zhx7uH/Jt8kyZ4NaZq0p5+SFHBzePdR/1rURd8G8+G3OaCPKqP+JQT4RMUQHC5SNRJLcK1piYdmhDiYEyuQG4FlStKCWLCXeUY2EVirNMeQIfOgbUHJsVjH07zm1y8y7lTWDMWVZOnkG6Ap5kB+n4l1eWbslOKgDv29JTFOMU+bvGvYZh70lmLK7Hg4CMpXVgvw5VF9v97YiiigLwvC7wasBHaASwH7wUqakXYhdGFxJ23xVMSLnvJn4S++4L8t8bifRIVqhT6tZCPOU4fdOvJKCRjKrf7gcW/E33ovZFgoOCJ2vBLIh9N9ME0v7tG15JpRtgIBsCXwLcl3tVyCZJ/eyYMbc3QJGsbcPGb2CYRjDbevPCQlNavcMdlyrNIke7VimM5aW8OBJKVh5wCNRpd9XylrKo1cZHYxu/c5Lr6VUZjLpxDlSz+IuTn4VE7vmgHNPnXdlxRKjLHG/FZrZTSCWFEBcRoSa/hysLSFwwDjKd9nelOZRNBvJ+NY48vA8ixVnk4WAMlR/5qhjTRam66BVysHeRcbjJ2IGjwTJC5Q== rosetta@rosetta.platform'})
+
 
 
             # Demo slurm computing resource

services/webapp/code/rosetta/base_app/models.py

 import uuid
-import enum
-
 from django.conf import settings
 from django.db import models
 from django.contrib.auth.models import User
 from django.utils import timezone
-
 from .utils import os_shell
 
 if 'sqlite' in settings.DATABASES['default']['ENGINE']:
@@ -72,6 +69,7 @@ class Container(models.Model):
     user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True)  
     # If a container has no user, it will be available to anyone. Can be created, edited and deleted only by admins.
 
+    name          = models.CharField('Container Name', max_length=255, blank=False, null=False)    
     image         = models.CharField('Container image', max_length=255, blank=False, null=False)
     type          = models.CharField('Container type', max_length=36, blank=False, null=False)
     registry      = models.CharField('Container registry', max_length=255, blank=False, null=False)
@@ -79,7 +77,7 @@ class Container(models.Model):
     #private       = models.BooleanField('Container is private and needs auth to be pulled from the registry')
 
     def __str__(self):
-        return str('Container of type "{}" with image "{}" from registry "{}" of user "{}"'.format(self.type, self.image, self.registry, self.user))
+        return str('Container of type "{}" with image "{}" with service ports "{}" from registry "{}" of user "{}"'.format(self.type, self.image, self.service_ports, self.registry, self.user))
 
     @property
     def id(self):
@@ -159,15 +157,39 @@ class Computing(models.Model):
     def sys_conf_data(self):          
         return ComputingSysConf.objects.get(computing=self).data
     
-    #@property    
-    #def user_conf_data(self):
-    #    return {'testuser':'ciao'}
+    @property    
+    def user_conf_data(self):
+        try:
+            return self._user_conf_data
+        except AttributeError:
+            raise AttributeError('User conf data is not yet attached, please attach it before accessing.')
     
     def attach_user_conf_data(self, user):
+        if self.user and self.user != user:
+            raise Exception('Cannot attach a conf data for another user (my user="{}", another user="{}"'.format(self.user, user)) 
         try:
-            self.user_conf_data = ComputingUserConf.objects.get(computing=self).data
+            self._user_conf_data = ComputingUserConf.objects.get(computing=self, user=user).data
         except ComputingUserConf.DoesNotExist:
-            self.user_conf_data = None
+            self._user_conf_data = None
+
+    # Get id_rsa file
+    #@property
+    #def id_rsa_file(self):
+    #    try:
+    #        id_rsa_file = self.user_conf_data['id_rsa']
+    #    except (TypeError, KeyError, AttributeError):
+    #        try:
+    #            id_rsa_file = self.sys_conf_data['id_rsa']
+    #        except:
+    #            id_rsa_file = None
+    #    return id_rsa_file
+
+    def get_conf_param(self, param):
+        try:
+            param_value = self.sys_conf_data[param]
+        except (TypeError, KeyError):
+            param_value = self.user_conf_data[param]
+        return param_value
 
 
 class ComputingSysConf(models.Model):
@@ -210,6 +232,11 @@ class Task(models.Model):
     computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
     container = models.ForeignKey('Container', on_delete=models.CASCADE, related_name='+')
 
+    # Auth
+    auth_user     = models.CharField('Task auth user', max_length=36, blank=True, null=True)
+    auth_password = models.CharField('Task auth password', max_length=36, blank=True, null=True)
+    access_method = models.CharField('Task access method', max_length=36, blank=True, null=True)
+
     def save(self, *args, **kwargs):
         
         try:

services/webapp/code/rosetta/base_app/tasks.py

+from .models import TaskStatuses
+from .utils import os_shell
+from .exceptions import ErrorMessage, ConsistencyException
+
+# Setup logging
+import logging
+logger = logging.getLogger(__name__)
+
+# Conf
+TASK_DATA_DIR = "/data"
+
+
+def start_task(task):
+
+    # Handle proper config
+    if task.computing.type == 'local':
+
+        # Get our ip address
+        #import netifaces
+        #netifaces.ifaddresses('eth0')
+        #backend_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr']
+
+        # Init run command #--cap-add=NET_ADMIN --cap-add=NET_RAW
+        run_command  = 'sudo docker run  --network=rosetta_default --name rosetta-task-{}'.format( task.id)
+
+        # Data volume
+        run_command += ' -v {}/task-{}:/data'.format(TASK_DATA_DIR, task.id)
+
+        # Set registry string
+        if task.container.registry == 'local':
+            registry_string = 'localhost:5000/'
+        else:
+            registry_string  = ''
+
+        # Host name, image entry command
+        run_command += ' -h task-{} -d -t {}{}'.format(task.id, registry_string, task.container.image)
+
+        # Run the task Debug
+        logger.debug('Running new task with command="{}"'.format(run_command))
+        out = os_shell(run_command, capture=True)
+        if out.exit_code != 0:
+            raise Exception(out.stderr)
+        else:
+            task_tid = out.stdout
+            logger.debug('Created task with id: "{}"'.format(task_tid))
+
+
+            # Get task IP address
+            out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' ' + task_tid + ' | tail -n1', capture=True)
+            if out.exit_code != 0:
+                raise Exception('Error: ' + out.stderr)
+            task_ip = out.stdout
+
+            # Set fields
+            task.tid    = task_tid
+            task.status = TaskStatuses.running
+            task.ip     = task_ip
+            task.port   = int(task.container.service_ports.split(',')[0])
+
+            # Save
+            task.save()
+
+    elif task.computing.type == 'remote':
+        logger.debug('Starting a remote task "{}"'.format(task.computing))
+
+        # Get computing host
+        host = task.computing.get_conf_param('host')
+
+        # Get id_rsa
+        id_rsa_file = task.computing.get_conf_param('id_rsa')
+        if not id_rsa_file: 
+            raise Exception('This computing requires an id_rsa file but cannot find any')
+
+        # 1) Run the container on the host (non blocking)
+ 
+        if task.container.type == 'singularity':
+            
+            run_command  = 'ssh -i {} -4 -o StrictHostKeyChecking=no {} '.format(id_rsa_file, host)
+            run_command += '"export SINGULARITY_NOHTTPS=true && '
+            run_command += 'exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv '
+            
+            # Set registry
+            if task.container.registry == 'docker_local':
+                registry = 'docker://dregistry:5000/'
+            elif task.container.registry == 'docker_hub':
+                registry = 'docker://'
+            else:
+                raise NotImplementedError('Registry {} not supported'.format(task.container.registry))
+    
+            run_command+='{}{} &> /tmp/{}.log & echo \$!"'.format(registry, task.container.image, task.uuid)
+            
+        else:
+            raise NotImplementedError('Container {} not supported'.format(task.container.type))
+
+        out = os_shell(run_command, capture=True)
+        if out.exit_code != 0:
+            raise Exception(out.stderr)
+ 
+        # Save pid echoed by the command above
+        task_pid = out.stdout
+
+        # 2) Simulate the agent (i.e. report container IP and port port)
+ 
+        # Get task IP address
+        out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' '+host+' | tail -n1', capture=True)
+        if out.exit_code != 0:
+            raise Exception('Error: ' + out.stderr)
+        task_ip = out.stdout
+ 
+        # Set fields
+        task.tid    = task.uuid
+        task.status = TaskStatuses.running
+        task.ip     = task_ip
+        task.pid    = task_pid
+        task.port   = int(task.container.service_ports.split(',')[0])
+ 
+        # Save
+        task.save()
+
+
+    else:
+        raise Exception('Consistency exception: invalid computing resource "{}'.format(task.computing))
+
+
+def stop_task(task):
+
+    if task.computing.type == 'local':
+    
+        # Delete the Docker container
+        standby_supported = False
+        if standby_supported:
+            stop_command = 'sudo docker stop {}'.format(task.tid)
+        else:
+            stop_command = 'sudo docker stop {} && sudo docker rm {}'.format(task.tid,task.tid)
+    
+        out = os_shell(stop_command, capture=True)
+        if out.exit_code != 0:
+            raise Exception(out.stderr)
+    
+    elif task.computing.type == 'remote':
+
+        # Get computing host
+        host = task.computing.get_conf_param('host')
+
+        # Get id_rsa
+        id_rsa_file = task.computing.get_conf_param('id_rsa')
+
+        # Stop the task remotely
+        stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {} "kill -9 {}"'.format(id_rsa_file, host, task.pid)
+        logger.debug(stop_command)
+        out = os_shell(stop_command, capture=True)
+        if out.exit_code != 0:
+            if not 'No such process' in out.stderr:
+                raise Exception(out.stderr)
+    else:
+        raise Exception('Don\'t know how to stop tasks on "{}" computing resource.'.format(task.computing))
+    
+    # Ok, save status as deleted
+    task.status = 'stopped'
+    task.save()
+    
+    # Check if the tunnel is active and if so kill it
+    logger.debug('Checking if task "{}" has a running tunnel'.format(task.tid))
+    check_command = 'ps -ef | grep ":'+str(task.tunnel_port)+':'+str(task.ip)+':'+str(task.port)+'" | grep -v grep | awk \'{print $2}\''
+    logger.debug(check_command)
+    out = os_shell(check_command, capture=True)
+    logger.debug(out)
+    if out.exit_code == 0:
+        logger.debug('Task "{}" has a running tunnel, killing it'.format(task.tid))
+        tunnel_pid = out.stdout
+        # Kill Tunnel command
+        kill_tunnel_command= 'kill -9 {}'.format(tunnel_pid)
+    
+        # Log
+        logger.debug('Killing tunnel with command: {}'.format(kill_tunnel_command))
+    
+        # Execute
+        os_shell(kill_tunnel_command, capture=True)
+        if out.exit_code != 0:
+            raise Exception(out.stderr)

services/webapp/code/rosetta/base_app/templates/add_container.html

@@ -22,6 +22,13 @@
 
           <table class="dashboard" style="max-width:430px">
 
+           <tr>
+            <td><b>Container name</b></td>
+            <td>
+             <input type="text" name="container_name" value="" placeholder="" size="23" required />
+            </td>
+           </tr>
+    
            <tr>
             <td><b>Type</b></td><td>
               <select name="container_type" >

services/webapp/code/rosetta/base_app/templates/components/computing.html

@@ -17,7 +17,7 @@
 
        <tr>
         <td><b>Owner</b></td>
-        <td>{% if computing.user %}{{ computing.user }}{% else %}Platform{% endif %}</td>
+        <td>{% if computing.user %}{{ computing.user }}{% else %}platform{% endif %}</td>
        </tr>
 
        <tr>

services/webapp/code/rosetta/base_app/templates/components/container.html

@@ -5,6 +5,11 @@
         <td><a href="?uuid={{ container.uuid }}">{{ container.id }}</a></td>
        </tr>
 
+       <tr>
+        <td><b>Name</b></td>
+        <td>{{ container.name }}</td>
+       </tr>
+
        <tr>
         <td><b>Image</b></td>
         <td>{{ container.image }}</td>
@@ -17,7 +22,7 @@
 
        <tr>
         <td><b>Owner</b></td>
-        <td>{% if container.user %}{{ container.user }}{% else %}Platform{% endif %}</td>
+        <td>{% if container.user %}{{ container.user }}{% else %}platform{% endif %}</td>
        </tr>
 
        <tr>

services/webapp/code/rosetta/base_app/templates/containers.html

@@ -20,14 +20,7 @@
       {% if data.container %}
       {% include "components/container.html" with container=data.container %}
       {% else %} 
-      
-      {% for container in data.platform_containers %}
-      {% include "components/container.html" with container=container %}
-      <br />
-      {% endfor %}
-
-
-      {% for container in data.user_containers %}
+      {% for container in data.containers %}
       {% include "components/container.html" with container=container %}
       <br />
       {% endfor %}

services/webapp/code/rosetta/base_app/templates/create_task.html

@@ -37,11 +37,8 @@
               <!-- <option value="metadesktop" selected>Meta Desktop</option>
               <option value="astroccok">Astrocook</option>
               <option value="gadgetviewer">Gadget Viewer</option> -->
-              {% for container in data.platform_containers %}
-              <option value="{{container.uuid}}">{{container.image}} ({{container.type}})</option> -->
-              {% endfor %}
-              {% for container in data.user_containers %}
-              <option value="{{container.uuid}}">{{container.image}} ({{container.type}})</option> -->
+              {% for container in data.containers %}
+              <option value="{{container.uuid}}">{{container.name}} ({{container.type}})</option>
               {% endfor %}              
               
               </select>
@@ -52,9 +49,9 @@
            <tr>
             <td><b>Computing resource</b></td><td>
               <select name="task_computing" >
-              <option value="local" selected>Local</option>
-              <option value="demoremote">Demo remote</option>
-              <option value="demoslurm">Demo Slurm cluster</option>
+              {% for computing in data.computings %}}
+              <option value="{{ computing.uuid }}">{{ computing.name}} ({% if computing.user %}{{ computing.user }}{% else %}platform{% endif %})</option>
+              {% endfor %}
               </select>
               &nbsp; | <a href="/add_computing">Add new...</a>
             </td>
@@ -70,71 +67,36 @@
 
       {% elif data.step == 'two' %}
 
-          <h3>Choose a name and a type for your new Task.</h3> 
+          <h3>Step 2: authentication and computing details</h3> 
           
           <br/>
           
           <form action="/create_task/" method="POST">
           {% csrf_token %}
           <input type="hidden" name="step" value="two" />
+          <input type="hidden" name="task_uuid" value="{{ data.task_uuid }}" />
 
           <table class="dashboard" style="max-width:700px">
  
-           <tr>
-            <td><b>Task name </b></td>
-            <td>
-             <input type="text" name="task_name" value="" placeholder="" size="23" required />
-            </td>
-           </tr>
-
-          <tr>
-            <td><b>Task container</b></td><td>
-              <select name="task_container_uuid" >
-              <!-- <option value="metadesktop" selected>Meta Desktop</option>
-              <option value="astroccok">Astrocook</option>
-              <option value="gadgetviewer">Gadget Viewer</option> -->
-              {% for container in data.platform_containers %}
-              <option value="{{container.uuid}}">{{container.image}} ({{container.type}})</option> -->
-              {% endfor %}
-              {% for container in data.user_containers %}
-              <option value="{{container.uuid}}">{{container.image}} ({{container.type}})</option> -->
-              {% endfor %}              
-              
-              </select>
-              &nbsp; | <a href="/add_container">Add new...</a>          
-            </td>
-           </tr>
-           
-           <tr>
-            <td><b>Computing resource</b></td><td>
-              <select name="task_computing" >
-              <option value="local" selected>Local</option>
-              <option value="demoremote">Demo remote</option>
-              <option value="demoslurm">Demo Slurm cluster</option>
-              </select>
-              &nbsp; | <a href="/add_computing">Add new...</a>
-            </td>
-           </tr>
-
            <tr>
             <td><b>Task user</b></td>
             <td>
-             <input type="text" name="name" value="" placeholder="metauser" size="23" disabled />
+             <input type="text" name="auth_user" value="" placeholder="" size="23" />
             </td>
            </tr>
            
            <tr>
             <td><b>Task password</b></td>
             <td>
-             <input type="password" name="password" value="" placeholder="" size="23" disabled />
+             <input type="password" name="auth_password" value="" placeholder="" size="23" />
             </td>
            </tr>
 
            <tr>
             <td><b>Access method</b></td><td>
               <select name="access_method" >
-              <option value="http_proxy" selected>HTTP proxy</option>
-              <option value="direct_tunnel">Direct tunnel</option>
+              <option value="http_proxy" disabled>HTTP proxy</option>
+              <option value="direct_tunnel" selected>Direct tunnel</option>
               <option value="None">None</option>
               </select>
             </td>

services/webapp/code/rosetta/base_app/utils.py

@@ -59,10 +59,7 @@ def send_email(to, subject, text):
         content = Content('text/plain', text)
         mail = Mail(from_email, subject, to_email, content)
         response = sg.client.mail.send.post(request_body=mail.get())
-
-        logger.critical(response.status_code)
-        logger.critical(response.body)
-        logger.critical(response.headers)
+        logger.debug(response)
     
 
 def format_exception(e, debug=False):
@@ -427,3 +424,8 @@ class dt_range(object):
     def next(self):
         return self.__next__()
 
+
+def debug_param(**kwargs):
+    for item in kwargs:
+        logger.critical('Param "{}": "{}"'.format(item, kwargs[item]))
+

services/webapp/code/rosetta/base_app/views.py

-
-# Python imports
-import time
 import uuid
-import inspect
-import json
-import socket
-import os
 import subprocess
-
-# Django imports
 from django.conf import settings
 from django.shortcuts import render
-from django.http import HttpResponseRedirect
 from django.contrib.auth import authenticate, login, logout
-from django.shortcuts import render
 from django.http import HttpResponse, HttpResponseRedirect
-from django.contrib.auth import authenticate, login, logout
 from django.contrib.auth.models import User
-from django.contrib.auth import update_session_auth_hash
-
-# Project imports
+from django.shortcuts import redirect
 from .models import Profile, LoginToken, Task, TaskStatuses, Container, Computing
-from .utils import send_email, format_exception, random_username, log_user_activity, timezonize, os_shell, booleanize
+from .utils import send_email, format_exception, timezonize, os_shell, booleanize, debug_param
+from .decorators import public_view, private_view
+from .tasks import start_task, stop_task
+from .exceptions import ErrorMessage
 
 # Setup logging
 import logging
 logger = logging.getLogger(__name__)
 
-# Custom exceptions
-from .exceptions import ErrorMessage, ConsistencyException
-
 # Conf
 SUPPORTED_CONTAINER_TYPES = ['docker', 'singularity']
 SUPPORTED_REGISTRIES = ['docker_local', 'docker_hub', 'singularity_hub']
 UNSUPPORTED_TYPES_VS_REGISTRIES = ['docker:singularity_hub']
 
-TASK_DATA_DIR = "/data"
-
 # Task cache
 _task_cache = {}
 
-#=========================
-#  Decorators
-#=========================
-
-# Public view
-def public_view(wrapped_view):
-    def public_view_wrapper(request, *argv, **kwargs):
-        # -------------- START Public/private common code --------------
-        try:
-            log_user_activity("DEBUG", "Called", request, wrapped_view.__name__)
-
-            # Try to get the templates from view kwargs
-            # Todo: Python3 compatibility: https://stackoverflow.com/questions/2677185/how-can-i-read-a-functions-signature-including-default-argument-values
-
-            argSpec=inspect.getargspec(wrapped_view)
-
-            if 'template' in argSpec.args:
-                template = argSpec.defaults[0]
-            else:
-                template = None
-
-            # Call wrapped view
-            data = wrapped_view(request, *argv, **kwargs)
-
-            if not isinstance(data, HttpResponse):
-                if template:
-                    #logger.debug('using template + data ("{}","{}")'.format(template,data))
-                    return render(request, template, {'data': data})
-                else:
-                    raise ConsistencyException('Got plain "data" output but no template defined in view')
-            else:
-                #logger.debug('using returned httpresponse')
-                return data
-
-        except Exception as e:
-            if isinstance(e, ErrorMessage):
-                error_text = str(e)
-            else:
-
-                # Raise te exception if we are in debug mode
-                if settings.DEBUG:
-                    raise
-
-                # Otherwise,
-                else:
-
-                    # first log the exception
-                    logger.error(format_exception(e))
-
-                    # and then mask it.
-                    error_text = 'something went wrong'
-
-            data = {'user': request.user,
-                    'title': 'Error',
-                    'error' : 'Error: "{}"'.format(error_text)}
-
-            if template:
-                return render(request, template, {'data': data})
-            else:
-                return render(request, 'error.html', {'data': data})
-        # --------------  END Public/private common code --------------
-    return public_view_wrapper
-
-# Private view
-def private_view(wrapped_view):
-    def private_view_wrapper(request, *argv, **kwargs):
-        if request.user.is_authenticated:
-            # -------------- START Public/private common code --------------
-            log_user_activity("DEBUG", "Called", request, wrapped_view.__name__)
-            try:
-
-                # Try to get the templates from view kwargs
-                # Todo: Python3 compatibility: https://stackoverflow.com/questions/2677185/how-can-i-read-a-functions-signature-including-default-argument-values
-
-                argSpec=inspect.getargspec(wrapped_view)
-
-                if 'template' in argSpec.args:
-                    template = argSpec.defaults[0]
-                else:
-                    template = None
-
-                # Call wrapped view
-                data = wrapped_view(request, *argv, **kwargs)
-
-                if not isinstance(data, HttpResponse):
-                    if template:
-                        #logger.debug('using template + data ("{}","{}")'.format(template,data))
-                        return render(request, template, {'data': data})
-                    else:
-                        raise ConsistencyException('Got plain "data" output but no template defined in view')
-                else:
-                    #logger.debug('using returned httpresponse')
-                    return data
-
-            except Exception as e:
-                if isinstance(e, ErrorMessage):
-                    error_text = str(e)
-                else:
-
-                    # Raise te exception if we are in debug mode
-                    if settings.DEBUG:
-                        raise
-
-                    # Otherwise,
-                    else:
-
-                        # first log the exception
-                        logger.error(format_exception(e))
-
-                        # and then mask it.
-                        error_text = 'something went wrong'
-
-                data = {'user': request.user,
-                        'title': 'Error',
-                        'error' : 'Error: "{}"'.format(error_text)}
-
-                if template:
-                    return render(request, template, {'data': data})
-                else:
-                    return render(request, 'error.html', {'data': data})
-            # --------------  END  Public/private common code --------------
-
-        else:
-            log_user_activity("DEBUG", "Redirecting to login since not authenticated", request)
-            return HttpResponseRedirect('/login')
-    return private_view_wrapper
-
-
-
-#------------------------------------------------------
-#   Helper functions
-#------------------------------------------------------
-
-def start_task(task):
-
-    if task.computing == 'local':
-
-        # Get our ip address
-        #import netifaces
-        #netifaces.ifaddresses('eth0')
-        #backend_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr']
-
-        # Init run command #--cap-add=NET_ADMIN --cap-add=NET_RAW
-        run_command  = 'sudo docker run  --network=rosetta_default --name rosetta-task-{}'.format( task.id)
-
-        # Data volume
-        run_command += ' -v {}/task-{}:/data'.format(TASK_DATA_DIR, task.id)
-
-        # Set registry string
-        if task.container.registry == 'local':
-            registry_string = 'localhost:5000/'
-        else:
-            registry_string  = ''
-
-        # Host name, image entry command
-        run_command += ' -h task-{} -d -t {}{}'.format(task.id, registry_string, task.container.image)
-
-        # Run the task Debug
-        logger.debug('Running new task with command="{}"'.format(run_command))
-        out = os_shell(run_command, capture=True)
-        if out.exit_code != 0:
-            raise Exception(out.stderr)
-        else:
-            task_tid = out.stdout
-            logger.debug('Created task with id: "{}"'.format(task_tid))
-
-
-            # Get task IP address
-            out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' ' + task_tid + ' | tail -n1', capture=True)
-            if out.exit_code != 0:
-                raise Exception('Error: ' + out.stderr)
-            task_ip = out.stdout
-
-            # Set fields
-            task.tid    = task_tid
-            task.status = TaskStatuses.running
-            task.ip     = task_ip
-            task.port   = int(task.container.service_ports.split(',')[0])
-
-            # Save
-            task.save()
-
-    elif task.computing == 'demoremote':
-        logger.debug('Using Demo Remote as computing resource')
-
-
-        # 1) Run the singularity container on slurmclusterworker-one (non blocking)
-        run_command = 'ssh -4 -o StrictHostKeyChecking=no slurmclusterworker-one  "export SINGULARITY_NOHTTPS=true && exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv docker://dregistry:5000/rosetta/metadesktop &> /tmp/{}.log & echo \$!"'.format(task.uuid)
-        out = os_shell(run_command, capture=True)
-        if out.exit_code != 0:
-            raise Exception(out.stderr)
-
-        # Save pid echoed by the command above
-        task_pid = out.stdout
-
-        # 2) Simulate the agent (i.e. report container IP and port port)
-
-        # Get task IP address
-        out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' slurmclusterworker-one | tail -n1', capture=True)
-        if out.exit_code != 0:
-            raise Exception('Error: ' + out.stderr)
-        task_ip = out.stdout
-
-        # Set fields
-        task.tid    = task.uuid
-        task.status = TaskStatuses.running
-        task.ip     = task_ip
-        task.pid    = task_pid
-        task.port   = int(task.container.service_ports.split(',')[0])
-
-        # Save
-        task.save()
-
-
-    else:
-        raise Exception('Consistency exception: invalid computing resource "{}'.format(task.computing))
-
-
-def stop_task(task):
-
-    if task.computing == 'local':
-    
-        # Delete the Docker container
-        standby_supported = False
-        if standby_supported:
-            stop_command = 'sudo docker stop {}'.format(task.tid)
-        else:
-            stop_command = 'sudo docker stop {} && sudo docker rm {}'.format(task.tid,task.tid)
-    
-        out = os_shell(stop_command, capture=True)
-        if out.exit_code != 0:
-            raise Exception(out.stderr)
-    
-    elif task.computing == 'demoremote':
-    
-        # Stop the task remotely
-        stop_command = 'ssh -4 -o StrictHostKeyChecking=no slurmclusterworker-one  "kill -9 {}"'.format(task.pid)
-        logger.debug(stop_command)
-        out = os_shell(stop_command, capture=True)
-        if out.exit_code != 0:
-            if not 'No such process' in out.stderr:
-                raise Exception(out.stderr)
-    
-    raise Exception('Don\'t know how to stop tasks on "{}" computing resource.'.format(task.computing))
-    
-    # Ok, save status as deleted
-    task.status = 'stopped'
-    task.save()
-    
-    # Check if the tunnel is active and if so kill it
-    logger.debug('Checking if task "{}" has a running tunnel'.format(task.tid))
-    check_command = 'ps -ef | grep ":'+str(task.tunnel_port)+':'+str(task.ip)+':'+str(task.port)+'" | grep -v grep | awk \'{print $2}\''
-    logger.debug(check_command)
-    out = os_shell(check_command, capture=True)
-    logger.debug(out)
-    if out.exit_code == 0:
-        logger.debug('Task "{}" has a running tunnel, killing it'.format(task.tid))
-        tunnel_pid = out.stdout
-        # Kill Tunnel command
-        kill_tunnel_command= 'kill -9 {}'.format(tunnel_pid)
-    
-        # Log
-        logger.debug('Killing tunnel with command: {}'.format(kill_tunnel_command))
-    
-        # Execute
-        os_shell(kill_tunnel_command, capture=True)
-        if out.exit_code != 0:
-            raise Exception(out.stderr)
 
 @public_view
 def login_view(request):
@@ -544,6 +260,9 @@ def tasks(request):
                 raise ErrorMessage('Task does not exists or no access rights')
             data['task'] = task
     
+            # Attach user config to computing
+            task.computing.attach_user_conf_data(task.user)
+    
             #----------------
             #  Task actions
             #----------------
@@ -573,7 +292,7 @@ def tasks(request):
             elif action=='connect':
     
                 # Create task tunnel
-                if task.computing in ['local', 'demoremote']:
+                if task.computing.type in ['local', 'remote']:
     
                     # If there is no tunnel port allocated yet, find one
                     if not task.tunnel_port:
@@ -620,7 +339,6 @@ def tasks(request):
                     raise ErrorMessage('Connecting to tasks on computing "{}" is not supported yet'.format(task.computing))
     
                 # Ok, now redirect to the task through the tunnel
-                from django.shortcuts import redirect
                 return redirect('http://localhost:{}'.format(task.tunnel_port))
 
         except Exception as e:
@@ -668,12 +386,9 @@ def create_task(request):
     data['profile'] = Profile.objects.get(user=request.user)
     data['title']   = 'New Task'
 
-    # Get containers configured on the platform, both private to this user and public
-    data['user_containers'] = Container.objects.filter(user=request.user)
-    data['platform_containers'] = Container.objects.filter(user=None)
-
-    data['computing'] = Computing.objects.filter(user=None)
-
+    # Get containers and computings 
+    data['containers'] = list(Container.objects.filter(user=None)) + list(Container.objects.filter(user=request.user))
+    data['computings'] = list(Computing.objects.filter(user=None)) + list(Computing.objects.filter(user=request.user))
 
     # Step if any
     step = request.POST.get('step', None)
@@ -685,8 +400,6 @@ def create_task(request):
 
         # Task container
         task_container_uuid = request.POST.get('task_container_uuid', None)
-
-        # Get the container object, first try as public and then as private
         try:
             task_container = Container.objects.get(uuid=task_container_uuid, user=None)
         except Container.DoesNotExist:
@@ -695,10 +408,16 @@ def create_task(request):
             except Container.DoesNotExist:
                 raise Exception('Consistency error, container with uuid "{}" does not exists or user "{}" does not have access rights'.format(task_container_uuid, request.user.email))
 
-        # Compute
-        task_computing = request.POST.get('task_computing', None)
-        if task_computing not in ['local', 'demoremote']:
-            raise ErrorMessage('Unknown computing resource "{}')
+        # task computing
+        task_computing_uuid = request.POST.get('task_computing', None)
+        try:
+            task_computing = Computing.objects.get(uuid=task_computing_uuid, user=None)
+        except Computing.DoesNotExist:
+            try:
+                task_computing =  Computing.objects.get(uuid=task_computing_uuid, user=request.user)
+            except Computing.DoesNotExist:
+                raise Exception('Consistency error, computing with uuid "{}" does not exists or user "{}" does not have access rights'.format(task_computing_uuid, request.user.email))
+
 
         # Generate the task uuid
         task_uuid = str(uuid.uuid4())
@@ -714,8 +433,9 @@ def create_task(request):
         # Save the task in the cache
         _task_cache[task_uuid] = task
 
-        # Set step
+        # Set step and task uuid
         data['step'] = 'two'
+        data['task_uuid'] = task.uuid
         
     elif step == 'two':
         
@@ -723,13 +443,23 @@ def create_task(request):
         task_uuid = request.POST.get('task_uuid', None)
         task = _task_cache[task_uuid]
 
+        # Add auth
+        task.task_auth_user     = request.POST.get('auth_user', None)
+        task.task_auth_password = request.POST.get('auth_password', None)
+        task.task_access_method = request.POST.get('access_method', None)
+        
         
         # Add auth and/or computing parameters to the task if any
+        # TODO... (i..e num cores)
         
         # Save the task in the DB
+        task.save()
+
+        # Attach user config to computing
+        task.computing.attach_user_conf_data(task.user)
 
         # Start the task
-        #start_task(task)
+        start_task(task)
 
         # Set step        
         data['step'] = 'created'
@@ -771,10 +501,13 @@ def task_log(request):
     data['task']    = task 
     data['refresh'] = refresh
 
+    # Attach user conf in any
+    task.computing.attach_user_conf_data(request.user) 
+
     # Get the log
     try:
 
-        if task.computing == 'local':
+        if task.computing.type == 'local':
 
             # View the Docker container log (attach)
             view_log_command = 'sudo docker logs {}'.format(task.tid,)
@@ -785,10 +518,16 @@ def task_log(request):
             else:
                 data['log'] = out.stdout
 
-        elif task.computing == 'demoremote':
+        elif task.computing.type == 'remote':
+
+            # Get computing host
+            host = task.computing.get_conf_param('host')
+    
+            # Get id_rsa
+            id_rsa_file = task.computing.get_conf_param('id_rsa')
 
             # View the Singularity container log
-            view_log_command = 'ssh -4 -o StrictHostKeyChecking=no slurmclusterworker-one  "cat /tmp/{}.log"'.format(task.uuid)
+            view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}  "cat /tmp/{}.log"'.format(id_rsa_file, host, task.uuid)
             logger.debug(view_log_command)
             out = os_shell(view_log_command, capture=True)
             if out.exit_code != 0:
@@ -803,7 +542,7 @@ def task_log(request):
     except Exception as e:
         data['error'] = 'Error in viewing task log'
         logger.error('Error in viewing task log with uuid="{}": "{}"'.format(uuid, e))
-        return render(request, 'error.html', {'data': data})
+        raise
 
     return render(request, 'task_log.html', {'data': data})
 
@@ -859,9 +598,8 @@ def containers(request):
     # Container list
     #----------------
 
-    # Get containers configured on the platform, both private to this user and public
-    data['user_containers'] = Container.objects.filter(user=request.user)
-    data['platform_containers'] = Container.objects.filter(user=None)
+    # Get containers
+    data['containers'] = list(Container.objects.filter(user=None)) + list(Container.objects.filter(user=request.user))
 
     return render(request, 'containers.html', {'data': data})
 
@@ -903,6 +641,9 @@ def add_container(request):
         if container_type+':'+container_registry in UNSUPPORTED_TYPES_VS_REGISTRIES:
             raise ErrorMessage('Sorry, container type "{}" is not compatible with registry type "{}"'.format(container_type, container_registry))
 
+        # Container name
+        container_name = request.POST.get('container_name', None)
+
         # Container service ports. TODO: support multiple ports? 
         container_service_ports = request.POST.get('container_service_ports', None)
         
@@ -919,6 +660,7 @@ def add_container(request):
         # Create
         Container.objects.create(user          = request.user,
                                  image         = container_image,
+                                 name          = container_name,
                                  type          = container_type,
                                  registry      = container_registry,
                                  service_ports = container_service_ports)
@@ -943,11 +685,11 @@ def computings(request):
     data['title']   = 'Add computing'
     data['name']    = request.POST.get('name',None)
     
-    data['platform_computings'] = Computing.objects.filter(user=None)
+    data['computings'] = list(Computing.objects.filter(user=None)) + Computing.objects.filter(user=request.user)
     
     # Attach user conf in any
-    for platform_computing in data['platform_computings']:
-        platform_computing.attach_user_conf_data(request.user)
+    for computing in data['computings']:
+        computing.attach_user_conf_data(request.user) 
 
     return render(request, 'computings.html', {'data': data})
 

services/webapp/code/rosetta/common.py

-import pytz
-import time
-import calendar
-import logging
-from datetime import datetime
-import traceback
-from rest_framework import serializers
-
-try:
-    from dateutil.tz import tzoffset
-except ImportError:
-    tzoffset = None
-    
-class ConsistencyException(Exception):
-    pass
-
-class AlreadyExistentException(Exception):
-    pass
-
-class DoNotCommitTransactionException(Exception):
-    pass
-
-def format_exception(e):
-    return 'Exception: ' + str(e) + '; Traceback: ' + traceback.format_exc().replace('\n','|')
-
-class HyperlinkedModelSerializerWithId(serializers.HyperlinkedModelSerializer):
-    """Extend the HyperlinkedModelSerializer to add IDs as well for the best of
-    both worlds.
-    """
-    id = serializers.ReadOnlyField()
-
-
-# def setup_logger(logger, loglevel):
-#     handler = logging.StreamHandler()
-#     formatter = logging.Formatter('%(name)s - %(levelname)s: %(message)s')
-#     handler.setFormatter(formatter)
-#     logger.addHandler(handler)
-#     logger.setLevel(loglevel)
-#     return logger
-
-
-#===================================
-#  Time management
-#===================================
-
-# Note: most of the following routines are extrapolated from the
-# time package of the Luna project (https://github.com/sarusso/Luna)
-# by courtesy of Stefano Alberto Russo. If you find and fix any bug,
-# please open a pull request with the fix for Luna as well. Thank you!
-
-def timezonize(timezone):
-    if not 'pytz' in str(type(timezone)):
-        timezone = pytz.timezone(timezone)
-    return timezone
-
-def t_now():
-    return time.time()  
-
-
-def dt(*args, **kwargs):
-    '''Initialize a datetime object in the proper way. Using the standard datetime leads to a lot of
-     problems with the tz package. Also, it forces UTC timezone if no timezone is specified'''
-    
-    if 'tz' in kwargs:
-        tzinfo = kwargs.pop('tz')
-    else:
-        tzinfo  = kwargs.pop('tzinfo', None)
-        
-    offset_s  = kwargs.pop('offset_s', None)   
-    trustme   = kwargs.pop('trustme', None)
-    
-    if kwargs:
-        raise Exception('Unhandled arg: "{}".'.format(kwargs))
-        
-    if (tzinfo is None):
-        # Force UTC if None
-        timezone = timezonize('UTC')
-        
-    else:
-        timezone = timezonize(tzinfo)
-    
-    if offset_s:
-        # Special case for the offset
-        if not tzoffset:
-            raise Exception('For ISO date with offset please install dateutil')
-        time_dt = datetime(*args, tzinfo=tzoffset(None, offset_s))
-    else:
-        # Standard  timezone
-        time_dt = timezone.localize(datetime(*args))
-
-    # Check consistency    
-    if not trustme and timezone != pytz.UTC:
-        if not check_dt_consistency(time_dt):
-            raise Exception('Sorry, time {} does not exists on timezone {}'.format(time_dt, timezone))
-
-    return  time_dt
-
-def dt_from_s(timestamp_s, tz=None):
-    if not tz:
-        tz = "UTC"
-    try:
-        timestamp_dt = datetime.utcfromtimestamp(float(timestamp_s))
-    except TypeError:
-        raise Exception('timestamp_s argument must be string or number, got {}'.format(type(timestamp_s)))
-
-    pytz_tz = timezonize(tz)
-    timestamp_dt = timestamp_dt.replace(tzinfo=pytz.utc).astimezone(pytz_tz)
-    
-    return timestamp_dt
-
-def s_from_dt(dt):
-    if not (isinstance(dt, datetime)):
-        raise Exception('s_from_dt function called without datetime argument, got type "{}" instead.'.format(dt.__class__.__name__))
-    microseconds_part = (dt.microsecond/1000000.0) if dt.microsecond else 0
-    return  ( calendar.timegm(dt.utctimetuple()) + microseconds_part)
-
-def check_dt_consistency(date_dt):
-    if date_dt.tzinfo is None:
-        return True
-    else: 
-        if date_dt.utcoffset() != dt_from_s(s_from_dt(date_dt), tz=date_dt.tzinfo).utcoffset():
-            return False
-        else:
-            return True
-
-def dt_from_str(string, timezone=None):
-
-    # Supported formats on UTC
-    # 1) YYYY-MM-DDThh:mm:ssZ
-    # 2) YYYY-MM-DDThh:mm:ss.{u}Z
-
-    # Supported formats with offset    
-    # 3) YYYY-MM-DDThh:mm:ss+ZZ:ZZ
-    # 4) YYYY-MM-DDThh:mm:ss.{u}+ZZ:ZZ
-    
-    # Also:
-    # 5) YYYY-MM-DDThh:mm:ss (without the trailing Z, and assume it on UTC)
-
-    # Split and parse standard part
-    date, time = string.split('T')
-    
-    if time.endswith('Z'):
-        # UTC
-        offset_s = 0
-        time = time[:-1]
-        
-    elif ('+') in time:
-        # Positive offset
-        time, offset = time.split('+')
-        # Set time and extract positive offset
-        if ':' in offset:
-            offset_s = (int(offset[0:2])*60 + int(offset[3:5]))* 60
-        else:
-            offset_s = (int(offset[0:2])*60 + int(offset[2:4]))* 60
-               
-        
-    elif ('-') in time:
-        # Negative offset
-        time, offset = time.split('-')
-        # Set time and extract negative offset
-        if ':' in offset:
-            offset_s = -1 * (int(offset[0:2])*60 + int(offset[3:5]))* 60
-        else:
-            offset_s = -1 * (int(offset[0:2])*60 + int(offset[2:4]))* 60
-    
-    
-    else:
-        # Assume UTC
-        offset_s = 0
-        #raise InputException('Format error')
-    
-    # Handle time
-    hour, minute, second = time.split(':')
-    
-    # Now parse date (easy)
-    year, month, day = date.split('-') 
-
-    # Convert everything to int
-    year    = int(year)
-    month   = int(month)
-    day     = int(day)
-    hour    = int(hour)
-    minute  = int(minute)
-    if '.' in second:
-        usecond = int(second.split('.')[1])
-        second  = int(second.split('.')[0])
-    else:
-        second  = int(second)
-        usecond = 0
-    
-    return dt(year, month, day, hour, minute, second, usecond, offset_s=offset_s)
-
-
-def dt_to_str(dt):
-    '''Return the ISO representation of the datetime as argument'''
-    return dt.isoformat()