From 3a54c605275bca6c7f136fa2f3021488e6ba03f9 Mon Sep 17 00:00:00 2001
From: Stefano Alberto Russo <stefano.russo@gmail.com>
Date: Sat, 7 Mar 2020 13:02:31 +0100
Subject: [PATCH] Parametrized remote tasks host and access key. Added
container name. Global refactoring and cleanup.
---
.../webapp/code/rosetta/base_app/admin.py | 5 +-
services/webapp/code/rosetta/base_app/api.py | 10 +-
.../code/rosetta/base_app/decorators.py | 140 +++++++
.../management/commands/base_app_populate.py | 23 +-
.../webapp/code/rosetta/base_app/models.py | 47 ++-
.../webapp/code/rosetta/base_app/tasks.py | 180 +++++++++
.../base_app/templates/add_container.html | 7 +
.../templates/components/computing.html | 2 +-
.../templates/components/container.html | 7 +-
.../base_app/templates/containers.html | 11 +-
.../base_app/templates/create_task.html | 62 +--
.../webapp/code/rosetta/base_app/utils.py | 10 +-
.../webapp/code/rosetta/base_app/views.py | 376 +++---------------
services/webapp/code/rosetta/common.py | 196 ---------
14 files changed, 476 insertions(+), 600 deletions(-)
create mode 100644 services/webapp/code/rosetta/base_app/decorators.py
create mode 100644 services/webapp/code/rosetta/base_app/tasks.py
delete mode 100644 services/webapp/code/rosetta/common.py
diff --git a/services/webapp/code/rosetta/base_app/admin.py b/services/webapp/code/rosetta/base_app/admin.py
index 2588ae2..3b91d61 100644
--- a/services/webapp/code/rosetta/base_app/admin.py
+++ b/services/webapp/code/rosetta/base_app/admin.py
@@ -1,8 +1,11 @@
from django.contrib import admin
-from .models import Profile, LoginToken, Task, Container
+from .models import Profile, LoginToken, Task, Container, Computing, ComputingSysConf, ComputingUserConf
admin.site.register(Profile)
admin.site.register(LoginToken)
admin.site.register(Task)
admin.site.register(Container)
+admin.site.register(Computing)
+admin.site.register(ComputingSysConf)
+admin.site.register(ComputingUserConf)
\ No newline at end of file
diff --git a/services/webapp/code/rosetta/base_app/api.py b/services/webapp/code/rosetta/base_app/api.py
index c2b7858..26590af 100644
--- a/services/webapp/code/rosetta/base_app/api.py
+++ b/services/webapp/code/rosetta/base_app/api.py
@@ -1,19 +1,13 @@
import logging
-
-# Django imports
from django.http import HttpResponse
from django.utils import timezone
from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.models import User, Group
-
from rest_framework.response import Response
from rest_framework import status, serializers, viewsets
from rest_framework.views import APIView
-
-
-# Project imports
-from rosetta.common import format_exception
-from rosetta.base_app.models import Profile
+from .utils import format_exception
+from .models import Profile
# Setup logging
logger = logging.getLogger(__name__)
diff --git a/services/webapp/code/rosetta/base_app/decorators.py b/services/webapp/code/rosetta/base_app/decorators.py
new file mode 100644
index 0000000..b8ef2d0
--- /dev/null
+++ b/services/webapp/code/rosetta/base_app/decorators.py
@@ -0,0 +1,140 @@
+# Imports
+import inspect
+from django.conf import settings
+from django.shortcuts import render
+from django.http import HttpResponse
+from .utils import format_exception, log_user_activity
+from .exceptions import ErrorMessage, ConsistencyException
+
+# Setup logging
+import logging
+logger = logging.getLogger(__name__)
+
+# Conf
+SUPPORTED_CONTAINER_TYPES = ['docker', 'singularity']
+SUPPORTED_REGISTRIES = ['docker_local', 'docker_hub', 'singularity_hub']
+UNSUPPORTED_TYPES_VS_REGISTRIES = ['docker:singularity_hub']
+
+
+# Public view
+def public_view(wrapped_view):
+ def public_view_wrapper(request, *argv, **kwargs):
+ # -------------- START Public/private common code --------------
+ try:
+ log_user_activity("DEBUG", "Called", request, wrapped_view.__name__)
+
+ # Try to get the templates from view kwargs
+ # Todo: Python3 compatibility: https://stackoverflow.com/questions/2677185/how-can-i-read-a-functions-signature-including-default-argument-values
+
+ argSpec=inspect.getargspec(wrapped_view)
+
+ if 'template' in argSpec.args:
+ template = argSpec.defaults[0]
+ else:
+ template = None
+
+ # Call wrapped view
+ data = wrapped_view(request, *argv, **kwargs)
+
+ if not isinstance(data, HttpResponse):
+ if template:
+ #logger.debug('using template + data ("{}","{}")'.format(template,data))
+ return render(request, template, {'data': data})
+ else:
+ raise ConsistencyException('Got plain "data" output but no template defined in view')
+ else:
+ #logger.debug('using returned httpresponse')
+ return data
+
+ except Exception as e:
+ if isinstance(e, ErrorMessage):
+ error_text = str(e)
+ else:
+
+ # Raise te exception if we are in debug mode
+ if settings.DEBUG:
+ raise
+
+ # Otherwise,
+ else:
+
+ # first log the exception
+ logger.error(format_exception(e))
+
+ # and then mask it.
+ error_text = 'something went wrong'
+
+ data = {'user': request.user,
+ 'title': 'Error',
+ 'error' : 'Error: "{}"'.format(error_text)}
+
+ if template:
+ return render(request, template, {'data': data})
+ else:
+ return render(request, 'error.html', {'data': data})
+ # -------------- END Public/private common code --------------
+ return public_view_wrapper
+
+# Private view
+def private_view(wrapped_view):
+ def private_view_wrapper(request, *argv, **kwargs):
+ if request.user.is_authenticated:
+ # -------------- START Public/private common code --------------
+ log_user_activity("DEBUG", "Called", request, wrapped_view.__name__)
+ try:
+
+ # Try to get the templates from view kwargs
+ # Todo: Python3 compatibility: https://stackoverflow.com/questions/2677185/how-can-i-read-a-functions-signature-including-default-argument-values
+
+ argSpec=inspect.getargspec(wrapped_view)
+
+ if 'template' in argSpec.args:
+ template = argSpec.defaults[0]
+ else:
+ template = None
+
+ # Call wrapped view
+ data = wrapped_view(request, *argv, **kwargs)
+
+ if not isinstance(data, HttpResponse):
+ if template:
+ #logger.debug('using template + data ("{}","{}")'.format(template,data))
+ return render(request, template, {'data': data})
+ else:
+ raise ConsistencyException('Got plain "data" output but no template defined in view')
+ else:
+ #logger.debug('using returned httpresponse')
+ return data
+
+ except Exception as e:
+ if isinstance(e, ErrorMessage):
+ error_text = str(e)
+ else:
+
+ # Raise te exception if we are in debug mode
+ if settings.DEBUG:
+ raise
+
+ # Otherwise,
+ else:
+
+ # first log the exception
+ logger.error(format_exception(e))
+
+ # and then mask it.
+ error_text = 'something went wrong'
+
+ data = {'user': request.user,
+ 'title': 'Error',
+ 'error' : 'Error: "{}"'.format(error_text)}
+
+ if template:
+ return render(request, template, {'data': data})
+ else:
+ return render(request, 'error.html', {'data': data})
+ # -------------- END Public/private common code --------------
+
+ else:
+ log_user_activity("DEBUG", "Redirecting to login since not authenticated", request)
+ return HttpResponseRedirect('/login')
+ return private_view_wrapper
diff --git a/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py b/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py
index 2aba472..1b91182 100644
--- a/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py
+++ b/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py
@@ -42,13 +42,23 @@ class Command(BaseCommand):
# MetaDesktop Docker
Container.objects.create(user = None,
+ name = 'MetaDesktop latest',
image = 'rosetta/metadesktop',
type = 'docker',
registry = 'docker_local',
service_ports = '8590')
+ # MetaDesktop Singularity
+ Container.objects.create(user = None,
+ name = 'MetaDesktop latest',
+ image = 'rosetta/metadesktop',
+ type = 'singularity',
+ registry = 'docker_local',
+ service_ports = '8590')
+
# Astrocook
Container.objects.create(user = None,
+ name = 'Astrocook b2b819e',
image = 'sarusso/astrocook:b2b819e',
type = 'docker',
registry = 'docker_local',
@@ -64,6 +74,7 @@ class Command(BaseCommand):
# JuPyter
Container.objects.create(user = testuser,
+ name = 'Jupyter Notebook latest',
image = 'jupyter/base-notebook',
type = 'docker',
registry = 'docker_hub',
@@ -90,9 +101,15 @@ class Command(BaseCommand):
# Create demo remote sys computing conf
ComputingSysConf.objects.create(computing = demo_remote_computing,
- data = {'host': 'slurmclusterworker-one',
- 'user': 'rosetta',
- 'identity': 'privkey?'})
+ data = {'host': 'slurmclusterworker-one'})
+
+ # Create demo remote user computing conf
+ ComputingUserConf.objects.create(user = testuser,
+ computing = demo_remote_computing,
+ data = {'user': 'testuser',
+ 'id_rsa': '/rosetta/.ssh/id_rsa',
+ 'id_rsa.pub': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2n4wiLiRmE1sla5+w0IW3wwPW/mqhhkm7IyCBS+rGTgnts7xsWcxobvamNdD6KSLNnjFZbBb7Yaf/BvWrwQgdqIFVU3gRWHYzoU6js+lKtBjd0e2DAVGivWCKEkSGLx7zhx7uH/Jt8kyZ4NaZq0p5+SFHBzePdR/1rURd8G8+G3OaCPKqP+JQT4RMUQHC5SNRJLcK1piYdmhDiYEyuQG4FlStKCWLCXeUY2EVirNMeQIfOgbUHJsVjH07zm1y8y7lTWDMWVZOnkG6Ap5kB+n4l1eWbslOKgDv29JTFOMU+bvGvYZh70lmLK7Hg4CMpXVgvw5VF9v97YiiigLwvC7wasBHaASwH7wUqakXYhdGFxJ23xVMSLnvJn4S++4L8t8bifRIVqhT6tZCPOU4fdOvJKCRjKrf7gcW/E33ovZFgoOCJ2vBLIh9N9ME0v7tG15JpRtgIBsCXwLcl3tVyCZJ/eyYMbc3QJGsbcPGb2CYRjDbevPCQlNavcMdlyrNIke7VimM5aW8OBJKVh5wCNRpd9XylrKo1cZHYxu/c5Lr6VUZjLpxDlSz+IuTn4VE7vmgHNPnXdlxRKjLHG/FZrZTSCWFEBcRoSa/hysLSFwwDjKd9nelOZRNBvJ+NY48vA8ixVnk4WAMlR/5qhjTRam66BVysHeRcbjJ2IGjwTJC5Q== rosetta@rosetta.platform'})
+
# Demo slurm computing resource
diff --git a/services/webapp/code/rosetta/base_app/models.py b/services/webapp/code/rosetta/base_app/models.py
index 2310a1a..f738899 100644
--- a/services/webapp/code/rosetta/base_app/models.py
+++ b/services/webapp/code/rosetta/base_app/models.py
@@ -1,11 +1,8 @@
import uuid
-import enum
-
from django.conf import settings
from django.db import models
from django.contrib.auth.models import User
from django.utils import timezone
-
from .utils import os_shell
if 'sqlite' in settings.DATABASES['default']['ENGINE']:
@@ -71,7 +68,8 @@ class Container(models.Model):
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True)
# If a container has no user, it will be available to anyone. Can be created, edited and deleted only by admins.
-
+
+ name = models.CharField('Container Name', max_length=255, blank=False, null=False)
image = models.CharField('Container image', max_length=255, blank=False, null=False)
type = models.CharField('Container type', max_length=36, blank=False, null=False)
registry = models.CharField('Container registry', max_length=255, blank=False, null=False)
@@ -79,7 +77,7 @@ class Container(models.Model):
#private = models.BooleanField('Container is private and needs auth to be pulled from the registry')
def __str__(self):
- return str('Container of type "{}" with image "{}" from registry "{}" of user "{}"'.format(self.type, self.image, self.registry, self.user))
+ return str('Container of type "{}" with image "{}" with service ports "{}" from registry "{}" of user "{}"'.format(self.type, self.image, self.service_ports, self.registry, self.user))
@property
def id(self):
@@ -159,15 +157,39 @@ class Computing(models.Model):
def sys_conf_data(self):
return ComputingSysConf.objects.get(computing=self).data
- #@property
- #def user_conf_data(self):
- # return {'testuser':'ciao'}
+ @property
+ def user_conf_data(self):
+ try:
+ return self._user_conf_data
+ except AttributeError:
+ raise AttributeError('User conf data is not yet attached, please attach it before accessing.')
def attach_user_conf_data(self, user):
+ if self.user and self.user != user:
+ raise Exception('Cannot attach a conf data for another user (my user="{}", another user="{}"'.format(self.user, user))
try:
- self.user_conf_data = ComputingUserConf.objects.get(computing=self).data
+ self._user_conf_data = ComputingUserConf.objects.get(computing=self, user=user).data
except ComputingUserConf.DoesNotExist:
- self.user_conf_data = None
+ self._user_conf_data = None
+
+ # Get id_rsa file
+ #@property
+ #def id_rsa_file(self):
+ # try:
+ # id_rsa_file = self.user_conf_data['id_rsa']
+ # except (TypeError, KeyError, AttributeError):
+ # try:
+ # id_rsa_file = self.sys_conf_data['id_rsa']
+ # except:
+ # id_rsa_file = None
+ # return id_rsa_file
+
+ def get_conf_param(self, param):
+ try:
+ param_value = self.sys_conf_data[param]
+ except (TypeError, KeyError):
+ param_value = self.user_conf_data[param]
+ return param_value
class ComputingSysConf(models.Model):
@@ -210,6 +232,11 @@ class Task(models.Model):
computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
container = models.ForeignKey('Container', on_delete=models.CASCADE, related_name='+')
+ # Auth
+ auth_user = models.CharField('Task auth user', max_length=36, blank=True, null=True)
+ auth_password = models.CharField('Task auth password', max_length=36, blank=True, null=True)
+ access_method = models.CharField('Task access method', max_length=36, blank=True, null=True)
+
def save(self, *args, **kwargs):
try:
diff --git a/services/webapp/code/rosetta/base_app/tasks.py b/services/webapp/code/rosetta/base_app/tasks.py
new file mode 100644
index 0000000..b2102b9
--- /dev/null
+++ b/services/webapp/code/rosetta/base_app/tasks.py
@@ -0,0 +1,180 @@
+from .models import TaskStatuses
+from .utils import os_shell
+from .exceptions import ErrorMessage, ConsistencyException
+
+# Setup logging
+import logging
+logger = logging.getLogger(__name__)
+
+# Conf
+TASK_DATA_DIR = "/data"
+
+
+def start_task(task):
+
+ # Handle proper config
+ if task.computing.type == 'local':
+
+ # Get our ip address
+ #import netifaces
+ #netifaces.ifaddresses('eth0')
+ #backend_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr']
+
+ # Init run command #--cap-add=NET_ADMIN --cap-add=NET_RAW
+ run_command = 'sudo docker run --network=rosetta_default --name rosetta-task-{}'.format( task.id)
+
+ # Data volume
+ run_command += ' -v {}/task-{}:/data'.format(TASK_DATA_DIR, task.id)
+
+ # Set registry string
+ if task.container.registry == 'local':
+ registry_string = 'localhost:5000/'
+ else:
+ registry_string = ''
+
+ # Host name, image entry command
+ run_command += ' -h task-{} -d -t {}{}'.format(task.id, registry_string, task.container.image)
+
+ # Run the task Debug
+ logger.debug('Running new task with command="{}"'.format(run_command))
+ out = os_shell(run_command, capture=True)
+ if out.exit_code != 0:
+ raise Exception(out.stderr)
+ else:
+ task_tid = out.stdout
+ logger.debug('Created task with id: "{}"'.format(task_tid))
+
+
+ # Get task IP address
+ out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' ' + task_tid + ' | tail -n1', capture=True)
+ if out.exit_code != 0:
+ raise Exception('Error: ' + out.stderr)
+ task_ip = out.stdout
+
+ # Set fields
+ task.tid = task_tid
+ task.status = TaskStatuses.running
+ task.ip = task_ip
+ task.port = int(task.container.service_ports.split(',')[0])
+
+ # Save
+ task.save()
+
+ elif task.computing.type == 'remote':
+ logger.debug('Starting a remote task "{}"'.format(task.computing))
+
+ # Get computing host
+ host = task.computing.get_conf_param('host')
+
+ # Get id_rsa
+ id_rsa_file = task.computing.get_conf_param('id_rsa')
+ if not id_rsa_file:
+ raise Exception('This computing requires an id_rsa file but cannot find any')
+
+ # 1) Run the container on the host (non blocking)
+
+ if task.container.type == 'singularity':
+
+ run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {} '.format(id_rsa_file, host)
+ run_command += '"export SINGULARITY_NOHTTPS=true && '
+ run_command += 'exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv '
+
+ # Set registry
+ if task.container.registry == 'docker_local':
+ registry = 'docker://dregistry:5000/'
+ elif task.container.registry == 'docker_hub':
+ registry = 'docker://'
+ else:
+ raise NotImplementedError('Registry {} not supported'.format(task.container.registry))
+
+ run_command+='{}{} &> /tmp/{}.log & echo \$!"'.format(registry, task.container.image, task.uuid)
+
+ else:
+ raise NotImplementedError('Container {} not supported'.format(task.container.type))
+
+ out = os_shell(run_command, capture=True)
+ if out.exit_code != 0:
+ raise Exception(out.stderr)
+
+ # Save pid echoed by the command above
+ task_pid = out.stdout
+
+ # 2) Simulate the agent (i.e. report container IP and port port)
+
+ # Get task IP address
+ out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' '+host+' | tail -n1', capture=True)
+ if out.exit_code != 0:
+ raise Exception('Error: ' + out.stderr)
+ task_ip = out.stdout
+
+ # Set fields
+ task.tid = task.uuid
+ task.status = TaskStatuses.running
+ task.ip = task_ip
+ task.pid = task_pid
+ task.port = int(task.container.service_ports.split(',')[0])
+
+ # Save
+ task.save()
+
+
+ else:
+ raise Exception('Consistency exception: invalid computing resource "{}'.format(task.computing))
+
+
+def stop_task(task):
+
+ if task.computing.type == 'local':
+
+ # Delete the Docker container
+ standby_supported = False
+ if standby_supported:
+ stop_command = 'sudo docker stop {}'.format(task.tid)
+ else:
+ stop_command = 'sudo docker stop {} && sudo docker rm {}'.format(task.tid,task.tid)
+
+ out = os_shell(stop_command, capture=True)
+ if out.exit_code != 0:
+ raise Exception(out.stderr)
+
+ elif task.computing.type == 'remote':
+
+ # Get computing host
+ host = task.computing.get_conf_param('host')
+
+ # Get id_rsa
+ id_rsa_file = task.computing.get_conf_param('id_rsa')
+
+ # Stop the task remotely
+ stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {} "kill -9 {}"'.format(id_rsa_file, host, task.pid)
+ logger.debug(stop_command)
+ out = os_shell(stop_command, capture=True)
+ if out.exit_code != 0:
+ if not 'No such process' in out.stderr:
+ raise Exception(out.stderr)
+ else:
+ raise Exception('Don\'t know how to stop tasks on "{}" computing resource.'.format(task.computing))
+
+ # Ok, save status as deleted
+ task.status = 'stopped'
+ task.save()
+
+ # Check if the tunnel is active and if so kill it
+ logger.debug('Checking if task "{}" has a running tunnel'.format(task.tid))
+ check_command = 'ps -ef | grep ":'+str(task.tunnel_port)+':'+str(task.ip)+':'+str(task.port)+'" | grep -v grep | awk \'{print $2}\''
+ logger.debug(check_command)
+ out = os_shell(check_command, capture=True)
+ logger.debug(out)
+ if out.exit_code == 0:
+ logger.debug('Task "{}" has a running tunnel, killing it'.format(task.tid))
+ tunnel_pid = out.stdout
+ # Kill Tunnel command
+ kill_tunnel_command= 'kill -9 {}'.format(tunnel_pid)
+
+ # Log
+ logger.debug('Killing tunnel with command: {}'.format(kill_tunnel_command))
+
+ # Execute
+ os_shell(kill_tunnel_command, capture=True)
+ if out.exit_code != 0:
+ raise Exception(out.stderr)
diff --git a/services/webapp/code/rosetta/base_app/templates/add_container.html b/services/webapp/code/rosetta/base_app/templates/add_container.html
index e231d99..bd5811e 100644
--- a/services/webapp/code/rosetta/base_app/templates/add_container.html
+++ b/services/webapp/code/rosetta/base_app/templates/add_container.html
@@ -21,6 +21,13 @@
{% csrf_token %}
<table class="dashboard" style="max-width:430px">
+
+ <tr>
+ <td><b>Container name</b></td>
+ <td>
+ <input type="text" name="container_name" value="" placeholder="" size="23" required />
+ </td>
+ </tr>
<tr>
<td><b>Type</b></td><td>
diff --git a/services/webapp/code/rosetta/base_app/templates/components/computing.html b/services/webapp/code/rosetta/base_app/templates/components/computing.html
index ad51279..e918725 100644
--- a/services/webapp/code/rosetta/base_app/templates/components/computing.html
+++ b/services/webapp/code/rosetta/base_app/templates/components/computing.html
@@ -17,7 +17,7 @@
<tr>
<td><b>Owner</b></td>
- <td>{% if computing.user %}{{ computing.user }}{% else %}Platform{% endif %}</td>
+ <td>{% if computing.user %}{{ computing.user }}{% else %}platform{% endif %}</td>
</tr>
<tr>
diff --git a/services/webapp/code/rosetta/base_app/templates/components/container.html b/services/webapp/code/rosetta/base_app/templates/components/container.html
index 10d2681..e4c1edb 100644
--- a/services/webapp/code/rosetta/base_app/templates/components/container.html
+++ b/services/webapp/code/rosetta/base_app/templates/components/container.html
@@ -5,6 +5,11 @@
<td><a href="?uuid={{ container.uuid }}">{{ container.id }}</a></td>
</tr>
+ <tr>
+ <td><b>Name</b></td>
+ <td>{{ container.name }}</td>
+ </tr>
+
<tr>
<td><b>Image</b></td>
<td>{{ container.image }}</td>
@@ -17,7 +22,7 @@
<tr>
<td><b>Owner</b></td>
- <td>{% if container.user %}{{ container.user }}{% else %}Platform{% endif %}</td>
+ <td>{% if container.user %}{{ container.user }}{% else %}platform{% endif %}</td>
</tr>
<tr>
diff --git a/services/webapp/code/rosetta/base_app/templates/containers.html b/services/webapp/code/rosetta/base_app/templates/containers.html
index 750f3f4..2c461e8 100644
--- a/services/webapp/code/rosetta/base_app/templates/containers.html
+++ b/services/webapp/code/rosetta/base_app/templates/containers.html
@@ -19,19 +19,12 @@
{% if data.container %}
{% include "components/container.html" with container=data.container %}
- {% else %}
-
- {% for container in data.platform_containers %}
+ {% else %}
+ {% for container in data.containers %}
{% include "components/container.html" with container=container %}
<br />
{% endfor %}
-
- {% for container in data.user_containers %}
- {% include "components/container.html" with container=container %}
- <br />
- {% endfor %}
-
<br />
<a href="/add_container">Add new...</a>
diff --git a/services/webapp/code/rosetta/base_app/templates/create_task.html b/services/webapp/code/rosetta/base_app/templates/create_task.html
index 87d4a8d..d3119ff 100644
--- a/services/webapp/code/rosetta/base_app/templates/create_task.html
+++ b/services/webapp/code/rosetta/base_app/templates/create_task.html
@@ -37,11 +37,8 @@
<!-- <option value="metadesktop" selected>Meta Desktop</option>
<option value="astroccok">Astrocook</option>
<option value="gadgetviewer">Gadget Viewer</option> -->
- {% for container in data.platform_containers %}
- <option value="{{container.uuid}}">{{container.image}} ({{container.type}})</option> -->
- {% endfor %}
- {% for container in data.user_containers %}
- <option value="{{container.uuid}}">{{container.image}} ({{container.type}})</option> -->
+ {% for container in data.containers %}
+ <option value="{{container.uuid}}">{{container.name}} ({{container.type}})</option>
{% endfor %}
</select>
@@ -52,9 +49,9 @@
<tr>
<td><b>Computing resource</b></td><td>
<select name="task_computing" >
- <option value="local" selected>Local</option>
- <option value="demoremote">Demo remote</option>
- <option value="demoslurm">Demo Slurm cluster</option>
+ {% for computing in data.computings %}}
+ <option value="{{ computing.uuid }}">{{ computing.name}} ({% if computing.user %}{{ computing.user }}{% else %}platform{% endif %})</option>
+ {% endfor %}
</select>
| <a href="/add_computing">Add new...</a>
</td>
@@ -70,71 +67,36 @@
{% elif data.step == 'two' %}
- <h3>Choose a name and a type for your new Task.</h3>
+ <h3>Step 2: authentication and computing details</h3>
<br/>
<form action="/create_task/" method="POST">
{% csrf_token %}
<input type="hidden" name="step" value="two" />
+ <input type="hidden" name="task_uuid" value="{{ data.task_uuid }}" />
<table class="dashboard" style="max-width:700px">
-
- <tr>
- <td><b>Task name </b></td>
- <td>
- <input type="text" name="task_name" value="" placeholder="" size="23" required />
- </td>
- </tr>
-
- <tr>
- <td><b>Task container</b></td><td>
- <select name="task_container_uuid" >
- <!-- <option value="metadesktop" selected>Meta Desktop</option>
- <option value="astroccok">Astrocook</option>
- <option value="gadgetviewer">Gadget Viewer</option> -->
- {% for container in data.platform_containers %}
- <option value="{{container.uuid}}">{{container.image}} ({{container.type}})</option> -->
- {% endfor %}
- {% for container in data.user_containers %}
- <option value="{{container.uuid}}">{{container.image}} ({{container.type}})</option> -->
- {% endfor %}
-
- </select>
- | <a href="/add_container">Add new...</a>
- </td>
- </tr>
-
- <tr>
- <td><b>Computing resource</b></td><td>
- <select name="task_computing" >
- <option value="local" selected>Local</option>
- <option value="demoremote">Demo remote</option>
- <option value="demoslurm">Demo Slurm cluster</option>
- </select>
- | <a href="/add_computing">Add new...</a>
- </td>
- </tr>
-
+
<tr>
<td><b>Task user</b></td>
<td>
- <input type="text" name="name" value="" placeholder="metauser" size="23" disabled />
+ <input type="text" name="auth_user" value="" placeholder="" size="23" />
</td>
</tr>
<tr>
<td><b>Task password</b></td>
<td>
- <input type="password" name="password" value="" placeholder="" size="23" disabled />
+ <input type="password" name="auth_password" value="" placeholder="" size="23" />
</td>
</tr>
<tr>
<td><b>Access method</b></td><td>
<select name="access_method" >
- <option value="http_proxy" selected>HTTP proxy</option>
- <option value="direct_tunnel">Direct tunnel</option>
+ <option value="http_proxy" disabled>HTTP proxy</option>
+ <option value="direct_tunnel" selected>Direct tunnel</option>
<option value="None">None</option>
</select>
</td>
diff --git a/services/webapp/code/rosetta/base_app/utils.py b/services/webapp/code/rosetta/base_app/utils.py
index c3726e9..0ec1348 100644
--- a/services/webapp/code/rosetta/base_app/utils.py
+++ b/services/webapp/code/rosetta/base_app/utils.py
@@ -59,10 +59,7 @@ def send_email(to, subject, text):
content = Content('text/plain', text)
mail = Mail(from_email, subject, to_email, content)
response = sg.client.mail.send.post(request_body=mail.get())
-
- logger.critical(response.status_code)
- logger.critical(response.body)
- logger.critical(response.headers)
+ logger.debug(response)
def format_exception(e, debug=False):
@@ -427,3 +424,8 @@ class dt_range(object):
def next(self):
return self.__next__()
+
+def debug_param(**kwargs):
+ for item in kwargs:
+ logger.critical('Param "{}": "{}"'.format(item, kwargs[item]))
+
diff --git a/services/webapp/code/rosetta/base_app/views.py b/services/webapp/code/rosetta/base_app/views.py
index 1766838..16b78ee 100644
--- a/services/webapp/code/rosetta/base_app/views.py
+++ b/services/webapp/code/rosetta/base_app/views.py
@@ -1,313 +1,29 @@
-
-# Python imports
-import time
import uuid
-import inspect
-import json
-import socket
-import os
import subprocess
-
-# Django imports
from django.conf import settings
from django.shortcuts import render
-from django.http import HttpResponseRedirect
from django.contrib.auth import authenticate, login, logout
-from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect
-from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.models import User
-from django.contrib.auth import update_session_auth_hash
-
-# Project imports
+from django.shortcuts import redirect
from .models import Profile, LoginToken, Task, TaskStatuses, Container, Computing
-from .utils import send_email, format_exception, random_username, log_user_activity, timezonize, os_shell, booleanize
+from .utils import send_email, format_exception, timezonize, os_shell, booleanize, debug_param
+from .decorators import public_view, private_view
+from .tasks import start_task, stop_task
+from .exceptions import ErrorMessage
# Setup logging
import logging
logger = logging.getLogger(__name__)
-# Custom exceptions
-from .exceptions import ErrorMessage, ConsistencyException
-
# Conf
SUPPORTED_CONTAINER_TYPES = ['docker', 'singularity']
SUPPORTED_REGISTRIES = ['docker_local', 'docker_hub', 'singularity_hub']
UNSUPPORTED_TYPES_VS_REGISTRIES = ['docker:singularity_hub']
-TASK_DATA_DIR = "/data"
-
# Task cache
_task_cache = {}
-#=========================
-# Decorators
-#=========================
-
-# Public view
-def public_view(wrapped_view):
- def public_view_wrapper(request, *argv, **kwargs):
- # -------------- START Public/private common code --------------
- try:
- log_user_activity("DEBUG", "Called", request, wrapped_view.__name__)
-
- # Try to get the templates from view kwargs
- # Todo: Python3 compatibility: https://stackoverflow.com/questions/2677185/how-can-i-read-a-functions-signature-including-default-argument-values
-
- argSpec=inspect.getargspec(wrapped_view)
-
- if 'template' in argSpec.args:
- template = argSpec.defaults[0]
- else:
- template = None
-
- # Call wrapped view
- data = wrapped_view(request, *argv, **kwargs)
-
- if not isinstance(data, HttpResponse):
- if template:
- #logger.debug('using template + data ("{}","{}")'.format(template,data))
- return render(request, template, {'data': data})
- else:
- raise ConsistencyException('Got plain "data" output but no template defined in view')
- else:
- #logger.debug('using returned httpresponse')
- return data
-
- except Exception as e:
- if isinstance(e, ErrorMessage):
- error_text = str(e)
- else:
-
- # Raise te exception if we are in debug mode
- if settings.DEBUG:
- raise
-
- # Otherwise,
- else:
-
- # first log the exception
- logger.error(format_exception(e))
-
- # and then mask it.
- error_text = 'something went wrong'
-
- data = {'user': request.user,
- 'title': 'Error',
- 'error' : 'Error: "{}"'.format(error_text)}
-
- if template:
- return render(request, template, {'data': data})
- else:
- return render(request, 'error.html', {'data': data})
- # -------------- END Public/private common code --------------
- return public_view_wrapper
-
-# Private view
-def private_view(wrapped_view):
- def private_view_wrapper(request, *argv, **kwargs):
- if request.user.is_authenticated:
- # -------------- START Public/private common code --------------
- log_user_activity("DEBUG", "Called", request, wrapped_view.__name__)
- try:
-
- # Try to get the templates from view kwargs
- # Todo: Python3 compatibility: https://stackoverflow.com/questions/2677185/how-can-i-read-a-functions-signature-including-default-argument-values
-
- argSpec=inspect.getargspec(wrapped_view)
-
- if 'template' in argSpec.args:
- template = argSpec.defaults[0]
- else:
- template = None
-
- # Call wrapped view
- data = wrapped_view(request, *argv, **kwargs)
-
- if not isinstance(data, HttpResponse):
- if template:
- #logger.debug('using template + data ("{}","{}")'.format(template,data))
- return render(request, template, {'data': data})
- else:
- raise ConsistencyException('Got plain "data" output but no template defined in view')
- else:
- #logger.debug('using returned httpresponse')
- return data
-
- except Exception as e:
- if isinstance(e, ErrorMessage):
- error_text = str(e)
- else:
-
- # Raise te exception if we are in debug mode
- if settings.DEBUG:
- raise
-
- # Otherwise,
- else:
-
- # first log the exception
- logger.error(format_exception(e))
-
- # and then mask it.
- error_text = 'something went wrong'
-
- data = {'user': request.user,
- 'title': 'Error',
- 'error' : 'Error: "{}"'.format(error_text)}
-
- if template:
- return render(request, template, {'data': data})
- else:
- return render(request, 'error.html', {'data': data})
- # -------------- END Public/private common code --------------
-
- else:
- log_user_activity("DEBUG", "Redirecting to login since not authenticated", request)
- return HttpResponseRedirect('/login')
- return private_view_wrapper
-
-
-
-#------------------------------------------------------
-# Helper functions
-#------------------------------------------------------
-
-def start_task(task):
-
- if task.computing == 'local':
-
- # Get our ip address
- #import netifaces
- #netifaces.ifaddresses('eth0')
- #backend_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr']
-
- # Init run command #--cap-add=NET_ADMIN --cap-add=NET_RAW
- run_command = 'sudo docker run --network=rosetta_default --name rosetta-task-{}'.format( task.id)
-
- # Data volume
- run_command += ' -v {}/task-{}:/data'.format(TASK_DATA_DIR, task.id)
-
- # Set registry string
- if task.container.registry == 'local':
- registry_string = 'localhost:5000/'
- else:
- registry_string = ''
-
- # Host name, image entry command
- run_command += ' -h task-{} -d -t {}{}'.format(task.id, registry_string, task.container.image)
-
- # Run the task Debug
- logger.debug('Running new task with command="{}"'.format(run_command))
- out = os_shell(run_command, capture=True)
- if out.exit_code != 0:
- raise Exception(out.stderr)
- else:
- task_tid = out.stdout
- logger.debug('Created task with id: "{}"'.format(task_tid))
-
-
- # Get task IP address
- out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' ' + task_tid + ' | tail -n1', capture=True)
- if out.exit_code != 0:
- raise Exception('Error: ' + out.stderr)
- task_ip = out.stdout
-
- # Set fields
- task.tid = task_tid
- task.status = TaskStatuses.running
- task.ip = task_ip
- task.port = int(task.container.service_ports.split(',')[0])
-
- # Save
- task.save()
-
- elif task.computing == 'demoremote':
- logger.debug('Using Demo Remote as computing resource')
-
-
- # 1) Run the singularity container on slurmclusterworker-one (non blocking)
- run_command = 'ssh -4 -o StrictHostKeyChecking=no slurmclusterworker-one "export SINGULARITY_NOHTTPS=true && exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv docker://dregistry:5000/rosetta/metadesktop &> /tmp/{}.log & echo \$!"'.format(task.uuid)
- out = os_shell(run_command, capture=True)
- if out.exit_code != 0:
- raise Exception(out.stderr)
-
- # Save pid echoed by the command above
- task_pid = out.stdout
-
- # 2) Simulate the agent (i.e. report container IP and port port)
-
- # Get task IP address
- out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' slurmclusterworker-one | tail -n1', capture=True)
- if out.exit_code != 0:
- raise Exception('Error: ' + out.stderr)
- task_ip = out.stdout
-
- # Set fields
- task.tid = task.uuid
- task.status = TaskStatuses.running
- task.ip = task_ip
- task.pid = task_pid
- task.port = int(task.container.service_ports.split(',')[0])
-
- # Save
- task.save()
-
-
- else:
- raise Exception('Consistency exception: invalid computing resource "{}'.format(task.computing))
-
-
-def stop_task(task):
-
- if task.computing == 'local':
-
- # Delete the Docker container
- standby_supported = False
- if standby_supported:
- stop_command = 'sudo docker stop {}'.format(task.tid)
- else:
- stop_command = 'sudo docker stop {} && sudo docker rm {}'.format(task.tid,task.tid)
-
- out = os_shell(stop_command, capture=True)
- if out.exit_code != 0:
- raise Exception(out.stderr)
-
- elif task.computing == 'demoremote':
-
- # Stop the task remotely
- stop_command = 'ssh -4 -o StrictHostKeyChecking=no slurmclusterworker-one "kill -9 {}"'.format(task.pid)
- logger.debug(stop_command)
- out = os_shell(stop_command, capture=True)
- if out.exit_code != 0:
- if not 'No such process' in out.stderr:
- raise Exception(out.stderr)
-
- raise Exception('Don\'t know how to stop tasks on "{}" computing resource.'.format(task.computing))
-
- # Ok, save status as deleted
- task.status = 'stopped'
- task.save()
-
- # Check if the tunnel is active and if so kill it
- logger.debug('Checking if task "{}" has a running tunnel'.format(task.tid))
- check_command = 'ps -ef | grep ":'+str(task.tunnel_port)+':'+str(task.ip)+':'+str(task.port)+'" | grep -v grep | awk \'{print $2}\''
- logger.debug(check_command)
- out = os_shell(check_command, capture=True)
- logger.debug(out)
- if out.exit_code == 0:
- logger.debug('Task "{}" has a running tunnel, killing it'.format(task.tid))
- tunnel_pid = out.stdout
- # Kill Tunnel command
- kill_tunnel_command= 'kill -9 {}'.format(tunnel_pid)
-
- # Log
- logger.debug('Killing tunnel with command: {}'.format(kill_tunnel_command))
-
- # Execute
- os_shell(kill_tunnel_command, capture=True)
- if out.exit_code != 0:
- raise Exception(out.stderr)
@public_view
def login_view(request):
@@ -544,6 +260,9 @@ def tasks(request):
raise ErrorMessage('Task does not exists or no access rights')
data['task'] = task
+ # Attach user config to computing
+ task.computing.attach_user_conf_data(task.user)
+
#----------------
# Task actions
#----------------
@@ -573,7 +292,7 @@ def tasks(request):
elif action=='connect':
# Create task tunnel
- if task.computing in ['local', 'demoremote']:
+ if task.computing.type in ['local', 'remote']:
# If there is no tunnel port allocated yet, find one
if not task.tunnel_port:
@@ -620,7 +339,6 @@ def tasks(request):
raise ErrorMessage('Connecting to tasks on computing "{}" is not supported yet'.format(task.computing))
# Ok, now redirect to the task through the tunnel
- from django.shortcuts import redirect
return redirect('http://localhost:{}'.format(task.tunnel_port))
except Exception as e:
@@ -668,12 +386,9 @@ def create_task(request):
data['profile'] = Profile.objects.get(user=request.user)
data['title'] = 'New Task'
- # Get containers configured on the platform, both private to this user and public
- data['user_containers'] = Container.objects.filter(user=request.user)
- data['platform_containers'] = Container.objects.filter(user=None)
-
- data['computing'] = Computing.objects.filter(user=None)
-
+ # Get containers and computings
+ data['containers'] = list(Container.objects.filter(user=None)) + list(Container.objects.filter(user=request.user))
+ data['computings'] = list(Computing.objects.filter(user=None)) + list(Computing.objects.filter(user=request.user))
# Step if any
step = request.POST.get('step', None)
@@ -685,8 +400,6 @@ def create_task(request):
# Task container
task_container_uuid = request.POST.get('task_container_uuid', None)
-
- # Get the container object, first try as public and then as private
try:
task_container = Container.objects.get(uuid=task_container_uuid, user=None)
except Container.DoesNotExist:
@@ -695,10 +408,16 @@ def create_task(request):
except Container.DoesNotExist:
raise Exception('Consistency error, container with uuid "{}" does not exists or user "{}" does not have access rights'.format(task_container_uuid, request.user.email))
- # Compute
- task_computing = request.POST.get('task_computing', None)
- if task_computing not in ['local', 'demoremote']:
- raise ErrorMessage('Unknown computing resource "{}')
+ # task computing
+ task_computing_uuid = request.POST.get('task_computing', None)
+ try:
+ task_computing = Computing.objects.get(uuid=task_computing_uuid, user=None)
+ except Computing.DoesNotExist:
+ try:
+ task_computing = Computing.objects.get(uuid=task_computing_uuid, user=request.user)
+ except Computing.DoesNotExist:
+ raise Exception('Consistency error, computing with uuid "{}" does not exists or user "{}" does not have access rights'.format(task_computing_uuid, request.user.email))
+
# Generate the task uuid
task_uuid = str(uuid.uuid4())
@@ -714,8 +433,9 @@ def create_task(request):
# Save the task in the cache
_task_cache[task_uuid] = task
- # Set step
+ # Set step and task uuid
data['step'] = 'two'
+ data['task_uuid'] = task.uuid
elif step == 'two':
@@ -723,13 +443,23 @@ def create_task(request):
task_uuid = request.POST.get('task_uuid', None)
task = _task_cache[task_uuid]
+ # Add auth
+ task.task_auth_user = request.POST.get('auth_user', None)
+ task.task_auth_password = request.POST.get('auth_password', None)
+ task.task_access_method = request.POST.get('access_method', None)
+
# Add auth and/or computing parameters to the task if any
-
+ # TODO... (i..e num cores)
+
# Save the task in the DB
+ task.save()
+
+ # Attach user config to computing
+ task.computing.attach_user_conf_data(task.user)
# Start the task
- #start_task(task)
+ start_task(task)
# Set step
data['step'] = 'created'
@@ -771,10 +501,13 @@ def task_log(request):
data['task'] = task
data['refresh'] = refresh
+ # Attach user conf in any
+ task.computing.attach_user_conf_data(request.user)
+
# Get the log
try:
- if task.computing == 'local':
+ if task.computing.type == 'local':
# View the Docker container log (attach)
view_log_command = 'sudo docker logs {}'.format(task.tid,)
@@ -785,10 +518,16 @@ def task_log(request):
else:
data['log'] = out.stdout
- elif task.computing == 'demoremote':
+ elif task.computing.type == 'remote':
+
+ # Get computing host
+ host = task.computing.get_conf_param('host')
+
+ # Get id_rsa
+ id_rsa_file = task.computing.get_conf_param('id_rsa')
# View the Singularity container log
- view_log_command = 'ssh -4 -o StrictHostKeyChecking=no slurmclusterworker-one "cat /tmp/{}.log"'.format(task.uuid)
+ view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {} "cat /tmp/{}.log"'.format(id_rsa_file, host, task.uuid)
logger.debug(view_log_command)
out = os_shell(view_log_command, capture=True)
if out.exit_code != 0:
@@ -803,7 +542,7 @@ def task_log(request):
except Exception as e:
data['error'] = 'Error in viewing task log'
logger.error('Error in viewing task log with uuid="{}": "{}"'.format(uuid, e))
- return render(request, 'error.html', {'data': data})
+ raise
return render(request, 'task_log.html', {'data': data})
@@ -859,9 +598,8 @@ def containers(request):
# Container list
#----------------
- # Get containers configured on the platform, both private to this user and public
- data['user_containers'] = Container.objects.filter(user=request.user)
- data['platform_containers'] = Container.objects.filter(user=None)
+ # Get containers
+ data['containers'] = list(Container.objects.filter(user=None)) + list(Container.objects.filter(user=request.user))
return render(request, 'containers.html', {'data': data})
@@ -903,9 +641,12 @@ def add_container(request):
if container_type+':'+container_registry in UNSUPPORTED_TYPES_VS_REGISTRIES:
raise ErrorMessage('Sorry, container type "{}" is not compatible with registry type "{}"'.format(container_type, container_registry))
+ # Container name
+ container_name = request.POST.get('container_name', None)
+
# Container service ports. TODO: support multiple ports?
container_service_ports = request.POST.get('container_service_ports', None)
-
+
if container_service_ports:
try:
for container_service_port in container_service_ports.split(','):
@@ -919,6 +660,7 @@ def add_container(request):
# Create
Container.objects.create(user = request.user,
image = container_image,
+ name = container_name,
type = container_type,
registry = container_registry,
service_ports = container_service_ports)
@@ -943,11 +685,11 @@ def computings(request):
data['title'] = 'Add computing'
data['name'] = request.POST.get('name',None)
- data['platform_computings'] = Computing.objects.filter(user=None)
+ data['computings'] = list(Computing.objects.filter(user=None)) + Computing.objects.filter(user=request.user)
# Attach user conf in any
- for platform_computing in data['platform_computings']:
- platform_computing.attach_user_conf_data(request.user)
+ for computing in data['computings']:
+ computing.attach_user_conf_data(request.user)
return render(request, 'computings.html', {'data': data})
diff --git a/services/webapp/code/rosetta/common.py b/services/webapp/code/rosetta/common.py
deleted file mode 100644
index 418fd24..0000000
--- a/services/webapp/code/rosetta/common.py
+++ /dev/null
@@ -1,196 +0,0 @@
-import pytz
-import time
-import calendar
-import logging
-from datetime import datetime
-import traceback
-from rest_framework import serializers
-
-try:
- from dateutil.tz import tzoffset
-except ImportError:
- tzoffset = None
-
-class ConsistencyException(Exception):
- pass
-
-class AlreadyExistentException(Exception):
- pass
-
-class DoNotCommitTransactionException(Exception):
- pass
-
-def format_exception(e):
- return 'Exception: ' + str(e) + '; Traceback: ' + traceback.format_exc().replace('\n','|')
-
-class HyperlinkedModelSerializerWithId(serializers.HyperlinkedModelSerializer):
- """Extend the HyperlinkedModelSerializer to add IDs as well for the best of
- both worlds.
- """
- id = serializers.ReadOnlyField()
-
-
-# def setup_logger(logger, loglevel):
-# handler = logging.StreamHandler()
-# formatter = logging.Formatter('%(name)s - %(levelname)s: %(message)s')
-# handler.setFormatter(formatter)
-# logger.addHandler(handler)
-# logger.setLevel(loglevel)
-# return logger
-
-
-#===================================
-# Time management
-#===================================
-
-# Note: most of the following routines are extrapolated from the
-# time package of the Luna project (https://github.com/sarusso/Luna)
-# by courtesy of Stefano Alberto Russo. If you find and fix any bug,
-# please open a pull request with the fix for Luna as well. Thank you!
-
-def timezonize(timezone):
- if not 'pytz' in str(type(timezone)):
- timezone = pytz.timezone(timezone)
- return timezone
-
-def t_now():
- return time.time()
-
-
-def dt(*args, **kwargs):
- '''Initialize a datetime object in the proper way. Using the standard datetime leads to a lot of
- problems with the tz package. Also, it forces UTC timezone if no timezone is specified'''
-
- if 'tz' in kwargs:
- tzinfo = kwargs.pop('tz')
- else:
- tzinfo = kwargs.pop('tzinfo', None)
-
- offset_s = kwargs.pop('offset_s', None)
- trustme = kwargs.pop('trustme', None)
-
- if kwargs:
- raise Exception('Unhandled arg: "{}".'.format(kwargs))
-
- if (tzinfo is None):
- # Force UTC if None
- timezone = timezonize('UTC')
-
- else:
- timezone = timezonize(tzinfo)
-
- if offset_s:
- # Special case for the offset
- if not tzoffset:
- raise Exception('For ISO date with offset please install dateutil')
- time_dt = datetime(*args, tzinfo=tzoffset(None, offset_s))
- else:
- # Standard timezone
- time_dt = timezone.localize(datetime(*args))
-
- # Check consistency
- if not trustme and timezone != pytz.UTC:
- if not check_dt_consistency(time_dt):
- raise Exception('Sorry, time {} does not exists on timezone {}'.format(time_dt, timezone))
-
- return time_dt
-
-def dt_from_s(timestamp_s, tz=None):
- if not tz:
- tz = "UTC"
- try:
- timestamp_dt = datetime.utcfromtimestamp(float(timestamp_s))
- except TypeError:
- raise Exception('timestamp_s argument must be string or number, got {}'.format(type(timestamp_s)))
-
- pytz_tz = timezonize(tz)
- timestamp_dt = timestamp_dt.replace(tzinfo=pytz.utc).astimezone(pytz_tz)
-
- return timestamp_dt
-
-def s_from_dt(dt):
- if not (isinstance(dt, datetime)):
- raise Exception('s_from_dt function called without datetime argument, got type "{}" instead.'.format(dt.__class__.__name__))
- microseconds_part = (dt.microsecond/1000000.0) if dt.microsecond else 0
- return ( calendar.timegm(dt.utctimetuple()) + microseconds_part)
-
-def check_dt_consistency(date_dt):
- if date_dt.tzinfo is None:
- return True
- else:
- if date_dt.utcoffset() != dt_from_s(s_from_dt(date_dt), tz=date_dt.tzinfo).utcoffset():
- return False
- else:
- return True
-
-def dt_from_str(string, timezone=None):
-
- # Supported formats on UTC
- # 1) YYYY-MM-DDThh:mm:ssZ
- # 2) YYYY-MM-DDThh:mm:ss.{u}Z
-
- # Supported formats with offset
- # 3) YYYY-MM-DDThh:mm:ss+ZZ:ZZ
- # 4) YYYY-MM-DDThh:mm:ss.{u}+ZZ:ZZ
-
- # Also:
- # 5) YYYY-MM-DDThh:mm:ss (without the trailing Z, and assume it on UTC)
-
- # Split and parse standard part
- date, time = string.split('T')
-
- if time.endswith('Z'):
- # UTC
- offset_s = 0
- time = time[:-1]
-
- elif ('+') in time:
- # Positive offset
- time, offset = time.split('+')
- # Set time and extract positive offset
- if ':' in offset:
- offset_s = (int(offset[0:2])*60 + int(offset[3:5]))* 60
- else:
- offset_s = (int(offset[0:2])*60 + int(offset[2:4]))* 60
-
-
- elif ('-') in time:
- # Negative offset
- time, offset = time.split('-')
- # Set time and extract negative offset
- if ':' in offset:
- offset_s = -1 * (int(offset[0:2])*60 + int(offset[3:5]))* 60
- else:
- offset_s = -1 * (int(offset[0:2])*60 + int(offset[2:4]))* 60
-
-
- else:
- # Assume UTC
- offset_s = 0
- #raise InputException('Format error')
-
- # Handle time
- hour, minute, second = time.split(':')
-
- # Now parse date (easy)
- year, month, day = date.split('-')
-
- # Convert everything to int
- year = int(year)
- month = int(month)
- day = int(day)
- hour = int(hour)
- minute = int(minute)
- if '.' in second:
- usecond = int(second.split('.')[1])
- second = int(second.split('.')[0])
- else:
- second = int(second)
- usecond = 0
-
- return dt(year, month, day, hour, minute, second, usecond, offset_s=offset_s)
-
-
-def dt_to_str(dt):
- '''Return the ISO representation of the datetime as argument'''
- return dt.isoformat()
--
GitLab