diff --git a/run_webapp_unit_tests.sh b/run_webapp_unit_tests.sh
index b6f78879e7c33d26897f41981332fa90ed17b4ed..051d9f4e5217c5d08b2d5759c572b8bd213fe5bf 100755
--- a/run_webapp_unit_tests.sh
+++ b/run_webapp_unit_tests.sh
@@ -6,4 +6,8 @@
DJANGO_LOG_LEVEL="CRITICAL"
ROSETTA_LOG_LEVEL="CRITICAL"
-rosetta/shell webapp "cd /opt/webapp_code && DJANGO_LOG_LEVEL=$DJANGO_LOG_LEVEL ROSETTA_LOG_LEVEL=$ROSETTA_LOG_LEVEL python3 manage.py test $@"
+# Set DB to SQLIte in-memory
+DJANGO_DB_ENGINE="django.db.backends.sqlite3"
+DJANGO_DB_NAME=":memory:"
+
+rosetta/shell webapp "export DJANGO_DB_ENGINE=$DJANGO_DB_ENGINE && export DJANGO_DB_NAME=$DJANGO_DB_NAME && cd /opt/webapp_code && python3 manage.py makemigrations && DJANGO_LOG_LEVEL=$DJANGO_LOG_LEVEL ROSETTA_LOG_LEVEL=$ROSETTA_LOG_LEVEL python3 manage.py test $@"
diff --git a/services/base/keys/id_rsa.pub b/services/base/keys/id_rsa.pub
index e4e1df61a89a45e50620f4bc74ba5547481ecd2d..9a0504b546f020c935181e7649fef5b6c32de8d0 100644
--- a/services/base/keys/id_rsa.pub
+++ b/services/base/keys/id_rsa.pub
@@ -1 +1 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2n4wiLiRmE1sla5+w0IW3wwPW/mqhhkm7IyCBS+rGTgnts7xsWcxobvamNdD6KSLNnjFZbBb7Yaf/BvWrwQgdqIFVU3gRWHYzoU6js+lKtBjd0e2DAVGivWCKEkSGLx7zhx7uH/Jt8kyZ4NaZq0p5+SFHBzePdR/1rURd8G8+G3OaCPKqP+JQT4RMUQHC5SNRJLcK1piYdmhDiYEyuQG4FlStKCWLCXeUY2EVirNMeQIfOgbUHJsVjH07zm1y8y7lTWDMWVZOnkG6Ap5kB+n4l1eWbslOKgDv29JTFOMU+bvGvYZh70lmLK7Hg4CMpXVgvw5VF9v97YiiigLwvC7wasBHaASwH7wUqakXYhdGFxJ23xVMSLnvJn4S++4L8t8bifRIVqhT6tZCPOU4fdOvJKCRjKrf7gcW/E33ovZFgoOCJ2vBLIh9N9ME0v7tG15JpRtgIBsCXwLcl3tVyCZJ/eyYMbc3QJGsbcPGb2CYRjDbevPCQlNavcMdlyrNIke7VimM5aW8OBJKVh5wCNRpd9XylrKo1cZHYxu/c5Lr6VUZjLpxDlSz+IuTn4VE7vmgHNPnXdlxRKjLHG/FZrZTSCWFEBcRoSa/hysLSFwwDjKd9nelOZRNBvJ+NY48vA8ixVnk4WAMlR/5qhjTRam66BVysHeRcbjJ2IGjwTJC5Q== docker@dev.ops
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2n4wiLiRmE1sla5+w0IW3wwPW/mqhhkm7IyCBS+rGTgnts7xsWcxobvamNdD6KSLNnjFZbBb7Yaf/BvWrwQgdqIFVU3gRWHYzoU6js+lKtBjd0e2DAVGivWCKEkSGLx7zhx7uH/Jt8kyZ4NaZq0p5+SFHBzePdR/1rURd8G8+G3OaCPKqP+JQT4RMUQHC5SNRJLcK1piYdmhDiYEyuQG4FlStKCWLCXeUY2EVirNMeQIfOgbUHJsVjH07zm1y8y7lTWDMWVZOnkG6Ap5kB+n4l1eWbslOKgDv29JTFOMU+bvGvYZh70lmLK7Hg4CMpXVgvw5VF9v97YiiigLwvC7wasBHaASwH7wUqakXYhdGFxJ23xVMSLnvJn4S++4L8t8bifRIVqhT6tZCPOU4fdOvJKCRjKrf7gcW/E33ovZFgoOCJ2vBLIh9N9ME0v7tG15JpRtgIBsCXwLcl3tVyCZJ/eyYMbc3QJGsbcPGb2CYRjDbevPCQlNavcMdlyrNIke7VimM5aW8OBJKVh5wCNRpd9XylrKo1cZHYxu/c5Lr6VUZjLpxDlSz+IuTn4VE7vmgHNPnXdlxRKjLHG/FZrZTSCWFEBcRoSa/hysLSFwwDjKd9nelOZRNBvJ+NY48vA8ixVnk4WAMlR/5qhjTRam66BVysHeRcbjJ2IGjwTJC5Q== rosetta@rosetta.platform
diff --git a/services/webapp/code/rosetta/base_app/fields.py b/services/webapp/code/rosetta/base_app/fields.py
new file mode 100644
index 0000000000000000000000000000000000000000..bc59f97038c09247ba208fcbc4a65e1c38ab12e2
--- /dev/null
+++ b/services/webapp/code/rosetta/base_app/fields.py
@@ -0,0 +1,29 @@
+import json
+from django.db.models import Field
+
+class JSONField(Field):
+ def db_type(self, connection):
+ return 'text'
+
+ def from_db_value(self, value, expression, connection):
+ if value is not None:
+ return self.to_python(value)
+ return value
+
+ def to_python(self, value):
+ if value is not None:
+ try:
+ return json.loads(value)
+ except (TypeError, ValueError):
+ return value
+ return value
+
+ def get_prep_value(self, value):
+ if value is not None:
+ return str(json.dumps(value))
+ return value
+
+ def value_to_string(self, obj):
+ return self.value_from_object(obj)
+
+# Credits: https://medium.com/@philamersune/using-postgresql-jsonfield-in-sqlite-95ad4ad2e5f1
\ No newline at end of file
diff --git a/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py b/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py
index af7f6664adbe8e6b66cb4b5e375537ca29aaaaa2..2aba472e8f252cb1bbd161543c31fae0f5436dbc 100644
--- a/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py
+++ b/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py
@@ -1,6 +1,6 @@
from django.core.management.base import BaseCommand
from django.contrib.auth.models import User
-from ...models import Profile, Container, Computing
+from ...models import Profile, Container, Computing, ComputingSysConf, ComputingUserConf
class Command(BaseCommand):
help = 'Adds the admin superuser with \'a\' password.'
@@ -69,22 +69,50 @@ class Command(BaseCommand):
registry = 'docker_hub',
service_ports = '8888')
+ # Computing resources
+ computing_resources = Computing.objects.all()
+ if computing_resources:
+ print('Not creating demo computing resources as they already exist')
+ else:
+ print('Creating demo computing resources containers...')
+ # Local computing resource
+ Computing.objects.create(user = None,
+ name = 'Local',
+ type = 'local')
+
+ # Demo remote computing resource
+ demo_remote_computing = Computing.objects.create(user = None,
+ name = 'Demo remote',
+ type = 'remote',
+ requires_sys_conf = True,
+ requires_user_conf = False)
+
+ # Create demo remote sys computing conf
+ ComputingSysConf.objects.create(computing = demo_remote_computing,
+ data = {'host': 'slurmclusterworker-one',
+ 'user': 'rosetta',
+ 'identity': 'privkey?'})
- # Computing resources
- #Computing.objects.create(user = None,
- # name = 'L',
- # type = '')
- # Computing resources
- #Computing.objects.create(user = None,
- # name = 'L',
- # type = '')
+ # Demo slurm computing resource
+ demo_slurm_computing = Computing.objects.create(user = None,
+ name = 'Demo Slurm',
+ type = 'slurm',
+ requires_sys_conf = True,
+ requires_user_conf = True)
+
+ # Create demo slurm sys computing conf
+ ComputingSysConf.objects.create(computing = demo_slurm_computing,
+ data = {'master': 'slurmclusterworker-master'})
+
+ # Create demo slurm user computing conf
+ ComputingUserConf.objects.create(user = testuser,
+ computing = demo_slurm_computing,
+ data = {'user': 'testuser',
+ 'id_rsa': '/rosetta/.ssh/id_rsa',
+ 'id_rsa.pub': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2n4wiLiRmE1sla5+w0IW3wwPW/mqhhkm7IyCBS+rGTgnts7xsWcxobvamNdD6KSLNnjFZbBb7Yaf/BvWrwQgdqIFVU3gRWHYzoU6js+lKtBjd0e2DAVGivWCKEkSGLx7zhx7uH/Jt8kyZ4NaZq0p5+SFHBzePdR/1rURd8G8+G3OaCPKqP+JQT4RMUQHC5SNRJLcK1piYdmhDiYEyuQG4FlStKCWLCXeUY2EVirNMeQIfOgbUHJsVjH07zm1y8y7lTWDMWVZOnkG6Ap5kB+n4l1eWbslOKgDv29JTFOMU+bvGvYZh70lmLK7Hg4CMpXVgvw5VF9v97YiiigLwvC7wasBHaASwH7wUqakXYhdGFxJ23xVMSLnvJn4S++4L8t8bifRIVqhT6tZCPOU4fdOvJKCRjKrf7gcW/E33ovZFgoOCJ2vBLIh9N9ME0v7tG15JpRtgIBsCXwLcl3tVyCZJ/eyYMbc3QJGsbcPGb2CYRjDbevPCQlNavcMdlyrNIke7VimM5aW8OBJKVh5wCNRpd9XylrKo1cZHYxu/c5Lr6VUZjLpxDlSz+IuTn4VE7vmgHNPnXdlxRKjLHG/FZrZTSCWFEBcRoSa/hysLSFwwDjKd9nelOZRNBvJ+NY48vA8ixVnk4WAMlR/5qhjTRam66BVysHeRcbjJ2IGjwTJC5Q== rosetta@rosetta.platform'})
- # Computing resources
- #Computing.objects.create(user = None,
- # name = 'L',
- # type = '')
diff --git a/services/webapp/code/rosetta/base_app/models.py b/services/webapp/code/rosetta/base_app/models.py
index 17a3fbc5e0f1102de38417a0da9c17edabdeeb1e..2310a1abe3bf8738f05aabda8d2f9187173b9e7b 100644
--- a/services/webapp/code/rosetta/base_app/models.py
+++ b/services/webapp/code/rosetta/base_app/models.py
@@ -1,12 +1,25 @@
import uuid
import enum
+from django.conf import settings
from django.db import models
from django.contrib.auth.models import User
from django.utils import timezone
from .utils import os_shell
+if 'sqlite' in settings.DATABASES['default']['ENGINE']:
+ from .fields import JSONField
+else:
+ from django.contrib.postgres.fields import JSONField
+
+class ConfigurationError(Exception):
+ pass
+
+class ConsistencyError(Exception):
+ pass
+
+
# Setup logging
import logging
logger = logging.getLogger(__name__)
@@ -89,7 +102,11 @@ class Computing(models.Model):
user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True)
# If a compute resource has no user, it will be available to anyone. Can be created, edited and deleted only by admins.
- name = models.CharField('Computing Name', max_length=255, blank=False, null=False)
+ name = models.CharField('Computing Name', max_length=255, blank=False, null=False)
+ type = models.CharField('Computing Type', max_length=255, blank=False, null=False)
+
+ requires_sys_conf = models.BooleanField(default=False)
+ requires_user_conf = models.BooleanField(default=False)
def __str__(self):
return str('Computing Resource "{}" of user "{}"'.format(self.name, self.user))
@@ -98,6 +115,81 @@ class Computing(models.Model):
def id(self):
return str(self.uuid).split('-')[0]
+ # Validate conf
+ def validate_conf_data(self, sys_conf_data=None, user_conf_data=None):
+
+ if self.type == 'local':
+ pass
+
+ elif self.type == 'remote':
+ # Check that we have all the data for a remote computing resource
+
+ # Look for host:
+ host_found = False
+ if sys_conf_data and 'host' in sys_conf_data and sys_conf_data['host']: host_found=True
+ if user_conf_data and 'host' in user_conf_data and user_conf_data['host']: host_found=True
+ if not host_found:
+ raise ConfigurationError('Missing host in conf')
+
+
+ # Look for user:
+ user_found = False
+ if sys_conf_data and 'user' in sys_conf_data and sys_conf_data['user']: user_found=True
+ if user_conf_data and 'user' in user_conf_data and user_conf_data['user']: user_found=True
+ if not user_found:
+ raise ConfigurationError('Missing user in conf')
+
+ # Look for password/identity:
+ password_found = False
+ identity_found = False
+ if sys_conf_data and 'password' in sys_conf_data and sys_conf_data['password']: password_found=True
+ if user_conf_data and 'password' in user_conf_data and user_conf_data['password']: password_found=True
+ if sys_conf_data and 'identity' in sys_conf_data and sys_conf_data['identity']: identity_found=True
+ if user_conf_data and 'identity' in user_conf_data and user_conf_data['identity']: identity_found=True
+ if not password_found and not identity_found:
+ raise ConfigurationError('Missing password or identity in conf')
+
+ elif self.type == 'slurm':
+ raise NotImplementedError('Not yet implemented for Slurm')
+
+ else:
+ raise ConsistencyError('Unknown computing type "{}"'.format(self.type))
+
+ @property
+ def sys_conf_data(self):
+ return ComputingSysConf.objects.get(computing=self).data
+
+ #@property
+ #def user_conf_data(self):
+ # return {'testuser':'ciao'}
+
+ def attach_user_conf_data(self, user):
+ try:
+ self.user_conf_data = ComputingUserConf.objects.get(computing=self).data
+ except ComputingUserConf.DoesNotExist:
+ self.user_conf_data = None
+
+
+class ComputingSysConf(models.Model):
+ uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
+ data = JSONField(blank=True, null=True)
+
+ @property
+ def id(self):
+ return str(self.uuid).split('-')[0]
+
+
+class ComputingUserConf(models.Model):
+ uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+ user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True)
+ computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
+ data = JSONField(blank=True, null=True)
+
+ @property
+ def id(self):
+ return str(self.uuid).split('-')[0]
+
#=========================
# Tasks
@@ -109,14 +201,14 @@ class Task(models.Model):
name = models.CharField('Task name', max_length=36, blank=False, null=False)
status = models.CharField('Task status', max_length=36, blank=True, null=True)
created = models.DateTimeField('Created on', default=timezone.now)
- computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
pid = models.IntegerField('Task pid', blank=True, null=True)
port = models.IntegerField('Task port', blank=True, null=True)
ip = models.CharField('Task ip address', max_length=36, blank=True, null=True)
tunnel_port = models.IntegerField('Task tunnel port', blank=True, null=True)
# Links
- container = models.ForeignKey('Container', on_delete=models.CASCADE, related_name='+')
+ computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
+ container = models.ForeignKey('Container', on_delete=models.CASCADE, related_name='+')
def save(self, *args, **kwargs):
diff --git a/services/webapp/code/rosetta/base_app/templates/add_container.html b/services/webapp/code/rosetta/base_app/templates/add_container.html
index 8f9af73bfcbffe4b94c35b588b8d3208b0217a81..e231d99c5c408188732332e00f282a3edb93e6fe 100644
--- a/services/webapp/code/rosetta/base_app/templates/add_container.html
+++ b/services/webapp/code/rosetta/base_app/templates/add_container.html
@@ -49,9 +49,9 @@
</tr>
<tr>
- <td><b>Service port(s)</b></td>
+ <td><b>Service port</b></td>
<td>
- <input type="text" name="container_service_ports" value="" placeholder="" size="5" required />
+ <input type="text" name="container_service_port" value="" placeholder="" size="5" />
</td>
</tr>
diff --git a/services/webapp/code/rosetta/base_app/templates/components/computing.html b/services/webapp/code/rosetta/base_app/templates/components/computing.html
new file mode 100644
index 0000000000000000000000000000000000000000..ad512797282b1ef3004bfff875aa02953f895209
--- /dev/null
+++ b/services/webapp/code/rosetta/base_app/templates/components/computing.html
@@ -0,0 +1,41 @@
+ <table class="dashboard">
+
+ <tr>
+ <td><b>ID</b></td>
+ <td><a href="?uuid={{ computing.uuid }}">{{ computing.id }}</a></td>
+ </tr>
+
+ <tr>
+ <td><b>Name</b></td>
+ <td>{{ computing.name }}</td>
+ </tr>
+
+ <tr>
+ <td><b>Type</b></td>
+ <td>{{ computing.type }}</td>
+ </tr>
+
+ <tr>
+ <td><b>Owner</b></td>
+ <td>{% if computing.user %}{{ computing.user }}{% else %}Platform{% endif %}</td>
+ </tr>
+
+ <tr>
+ <td><b>Sys Conf</b></td>
+ <td>{{ computing.sys_conf_data }}</td>
+ </tr>
+
+ <tr>
+ <td><b>User Conf</b></td>
+ <td>{{ computing.user_conf_data }}</td>
+ </tr>
+
+
+ {% if computing.user %}
+ <tr>
+ <td><b>Operations</b></td>
+ <td><a href="?action=delete&uuid={{ computing.uuid }}">Delete</a></td>
+ </tr>
+ {% endif %}
+ </table>
+ <br/>
\ No newline at end of file
diff --git a/services/webapp/code/rosetta/base_app/templates/computings.html b/services/webapp/code/rosetta/base_app/templates/computings.html
index 5ae68940183e6c832939a5f672a0c4260f41004d..26982d46d0aca41620460ea4734b8f62ccb74674 100644
--- a/services/webapp/code/rosetta/base_app/templates/computings.html
+++ b/services/webapp/code/rosetta/base_app/templates/computings.html
@@ -10,15 +10,15 @@
<div class="span8 offset2">
{% if data.computing %}
- <h1><a href="/computings">Computing Resources List</a> > {{ data.computing.id }} </h1>
+ <h1><a href="/computings">Computing List</a> > {{ data.computing.id }} </h1>
{% else %}
- <h1>Computing Resources List</h1>
+ <h1>Computing List</h1>
{% endif %}
<hr/>
{% if data.computing %}
- {% include "components/computing.html" with computing=data.computing %}
+ {% include "components/computing.html" with computing=data.computin %}
{% else %}
{% for computing in data.platform_computings %}
diff --git a/services/webapp/code/rosetta/base_app/templates/create_task.html b/services/webapp/code/rosetta/base_app/templates/create_task.html
index e6c1ed63459b531eec7ff83295dbc29072233656..87d4a8d1138fff9cf78c134fda9f7a9bf72ac014 100644
--- a/services/webapp/code/rosetta/base_app/templates/create_task.html
+++ b/services/webapp/code/rosetta/base_app/templates/create_task.html
@@ -11,14 +11,16 @@
<h1>New Task</h1>
<hr>
- {% if not data.created %}
+ {% if data.step == 'one' %}
- <h3>Choose a name and a type for your new Task.</h3>
+ <h3>Step 1: name, container and computing.</h3>
<br/>
<form action="/create_task/" method="POST">
{% csrf_token %}
+ <input type="hidden" name="step" value="one" />
+
<table class="dashboard" style="max-width:700px">
@@ -29,22 +31,63 @@
</td>
</tr>
- <!--
- <tr>
- <td><b>Task user</b></td>
- <td>
- <input type="text" name="name" value="" placeholder="metauser" size="23" disabled />
+ <tr>
+ <td><b>Task container</b></td><td>
+ <select name="task_container_uuid" >
+ <!-- <option value="metadesktop" selected>Meta Desktop</option>
+ <option value="astroccok">Astrocook</option>
+ <option value="gadgetviewer">Gadget Viewer</option> -->
+ {% for container in data.platform_containers %}
+ <option value="{{container.uuid}}">{{container.image}} ({{container.type}})</option> -->
+ {% endfor %}
+ {% for container in data.user_containers %}
+ <option value="{{container.uuid}}">{{container.image}} ({{container.type}})</option> -->
+ {% endfor %}
+
+ </select>
+ | <a href="/add_container">Add new...</a>
</td>
- </tr>-->
+ </tr>
- <!--<tr>
- <td><b>Task password</b></td>
- <td>
- <input type="password" name="password" value="" placeholder="" size="23" disabled />
+ <tr>
+ <td><b>Computing resource</b></td><td>
+ <select name="task_computing" >
+ <option value="local" selected>Local</option>
+ <option value="demoremote">Demo remote</option>
+ <option value="demoslurm">Demo Slurm cluster</option>
+ </select>
+ | <a href="/add_computing">Add new...</a>
</td>
- </tr> -->
+ </tr>
+
+ <tr>
+ <td colspan=2 align=center style="padding:20px">
+ <input type="submit" value="Next">
+ </td>
+ </tr>
+ </table>
+ </form>
+
+ {% elif data.step == 'two' %}
+
+ <h3>Choose a name and a type for your new Task.</h3>
+
+ <br/>
+
+ <form action="/create_task/" method="POST">
+ {% csrf_token %}
+ <input type="hidden" name="step" value="two" />
+ <table class="dashboard" style="max-width:700px">
+
<tr>
+ <td><b>Task name </b></td>
+ <td>
+ <input type="text" name="task_name" value="" placeholder="" size="23" required />
+ </td>
+ </tr>
+
+ <tr>
<td><b>Task container</b></td><td>
<select name="task_container_uuid" >
<!-- <option value="metadesktop" selected>Meta Desktop</option>
@@ -73,6 +116,30 @@
</td>
</tr>
+ <tr>
+ <td><b>Task user</b></td>
+ <td>
+ <input type="text" name="name" value="" placeholder="metauser" size="23" disabled />
+ </td>
+ </tr>
+
+ <tr>
+ <td><b>Task password</b></td>
+ <td>
+ <input type="password" name="password" value="" placeholder="" size="23" disabled />
+ </td>
+ </tr>
+
+ <tr>
+ <td><b>Access method</b></td><td>
+ <select name="access_method" >
+ <option value="http_proxy" selected>HTTP proxy</option>
+ <option value="direct_tunnel">Direct tunnel</option>
+ <option value="None">None</option>
+ </select>
+ </td>
+ </tr>
+
<!-- <tr>
<td><b>Run using</b></td><td>
<select name="run_using" >
@@ -97,20 +164,7 @@
</table>
</form>
- <br/>
- <br/>
- <br/>
- <br/>
- <br/>
- <br/>
- <br/>
- <br/>
- <br/>
-
-
-
-
{% else %}
Ok, task created. Go back to your <a href="/tasks">task list</a>.
diff --git a/services/webapp/code/rosetta/base_app/tests/test_models.py b/services/webapp/code/rosetta/base_app/tests/test_models.py
new file mode 100644
index 0000000000000000000000000000000000000000..29d62ee9b49d286adc2bfd4f33c73a96e7030622
--- /dev/null
+++ b/services/webapp/code/rosetta/base_app/tests/test_models.py
@@ -0,0 +1,47 @@
+import json
+
+from django.contrib.auth.models import User
+
+from .common import BaseAPITestCase
+from ..models import Profile, Computing, ComputingSysConf
+
+class Modeltest(BaseAPITestCase):
+
+ def setUp(self):
+
+ # Create test users
+ self.user = User.objects.create_user('testuser', password='testpass')
+ self.anotheruser = User.objects.create_user('anotheruser', password='anotherpass')
+
+ # Create test profile
+ Profile.objects.create(user=self.user, authtoken='ync719tce917tec197t29cn712eg')
+
+
+ def test_computing(self):
+ '''Test Computing and their Conf models'''
+
+ computing = Computing.objects.create(name='MyComp', type='remote')
+
+ computingSysConf = ComputingSysConf.objects.create(computing=computing, data={'myvar':42})
+
+ self.assertEqual(ComputingSysConf.objects.all()[0].data, {'myvar':42})
+
+ # Will raise, no host or user or pass/identity
+ with self.assertRaises(Exception):
+ computing.validate_conf_data(sys_conf_data=computingSysConf.data)
+
+ # Complete conf
+ computingSysConf_1 = ComputingSysConf.objects.create(computing=computing, data={'host':'localhost', 'user':'testuser', 'password':'testpass'})
+
+ # Will not raise
+ computing.validate_conf_data(sys_conf_data=computingSysConf_1.data)
+
+
+ # Complete conf
+ #computingSysConf_1 = ComputingSysConf.objects.create(computing=computing, data={'host':'localhost', 'user':'testuser', 'password':'testpass'})
+
+ # Will not raise
+ #computing.validate_conf_data(sys_conf_data=computingSysConf_1.data)
+
+
+
diff --git a/services/webapp/code/rosetta/base_app/views.py b/services/webapp/code/rosetta/base_app/views.py
index 85f051829536728f6186ae7956bfdc863a825cf8..1766838fee93548b652636de1fd0e85dd16b393e 100644
--- a/services/webapp/code/rosetta/base_app/views.py
+++ b/services/webapp/code/rosetta/base_app/views.py
@@ -37,6 +37,9 @@ UNSUPPORTED_TYPES_VS_REGISTRIES = ['docker:singularity_hub']
TASK_DATA_DIR = "/data"
+# Task cache
+_task_cache = {}
+
#=========================
# Decorators
#=========================
@@ -166,6 +169,145 @@ def private_view(wrapped_view):
+#------------------------------------------------------
+# Helper functions
+#------------------------------------------------------
+
+def start_task(task):
+
+ if task.computing == 'local':
+
+ # Get our ip address
+ #import netifaces
+ #netifaces.ifaddresses('eth0')
+ #backend_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr']
+
+ # Init run command #--cap-add=NET_ADMIN --cap-add=NET_RAW
+ run_command = 'sudo docker run --network=rosetta_default --name rosetta-task-{}'.format( task.id)
+
+ # Data volume
+ run_command += ' -v {}/task-{}:/data'.format(TASK_DATA_DIR, task.id)
+
+ # Set registry string
+ if task.container.registry == 'local':
+ registry_string = 'localhost:5000/'
+ else:
+ registry_string = ''
+
+ # Host name, image entry command
+ run_command += ' -h task-{} -d -t {}{}'.format(task.id, registry_string, task.container.image)
+
+ # Run the task Debug
+ logger.debug('Running new task with command="{}"'.format(run_command))
+ out = os_shell(run_command, capture=True)
+ if out.exit_code != 0:
+ raise Exception(out.stderr)
+ else:
+ task_tid = out.stdout
+ logger.debug('Created task with id: "{}"'.format(task_tid))
+
+
+ # Get task IP address
+ out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' ' + task_tid + ' | tail -n1', capture=True)
+ if out.exit_code != 0:
+ raise Exception('Error: ' + out.stderr)
+ task_ip = out.stdout
+
+ # Set fields
+ task.tid = task_tid
+ task.status = TaskStatuses.running
+ task.ip = task_ip
+ task.port = int(task.container.service_ports.split(',')[0])
+
+ # Save
+ task.save()
+
+ elif task.computing == 'demoremote':
+ logger.debug('Using Demo Remote as computing resource')
+
+
+ # 1) Run the singularity container on slurmclusterworker-one (non blocking)
+ run_command = 'ssh -4 -o StrictHostKeyChecking=no slurmclusterworker-one "export SINGULARITY_NOHTTPS=true && exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv docker://dregistry:5000/rosetta/metadesktop &> /tmp/{}.log & echo \$!"'.format(task.uuid)
+ out = os_shell(run_command, capture=True)
+ if out.exit_code != 0:
+ raise Exception(out.stderr)
+
+ # Save pid echoed by the command above
+ task_pid = out.stdout
+
+ # 2) Simulate the agent (i.e. report container IP and port port)
+
+ # Get task IP address
+ out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' slurmclusterworker-one | tail -n1', capture=True)
+ if out.exit_code != 0:
+ raise Exception('Error: ' + out.stderr)
+ task_ip = out.stdout
+
+ # Set fields
+ task.tid = task.uuid
+ task.status = TaskStatuses.running
+ task.ip = task_ip
+ task.pid = task_pid
+ task.port = int(task.container.service_ports.split(',')[0])
+
+ # Save
+ task.save()
+
+
+ else:
+ raise Exception('Consistency exception: invalid computing resource "{}'.format(task.computing))
+
+
+def stop_task(task):
+
+ if task.computing == 'local':
+
+ # Delete the Docker container
+ standby_supported = False
+ if standby_supported:
+ stop_command = 'sudo docker stop {}'.format(task.tid)
+ else:
+ stop_command = 'sudo docker stop {} && sudo docker rm {}'.format(task.tid,task.tid)
+
+ out = os_shell(stop_command, capture=True)
+ if out.exit_code != 0:
+ raise Exception(out.stderr)
+
+ elif task.computing == 'demoremote':
+
+ # Stop the task remotely
+ stop_command = 'ssh -4 -o StrictHostKeyChecking=no slurmclusterworker-one "kill -9 {}"'.format(task.pid)
+ logger.debug(stop_command)
+ out = os_shell(stop_command, capture=True)
+ if out.exit_code != 0:
+ if not 'No such process' in out.stderr:
+ raise Exception(out.stderr)
+
+ raise Exception('Don\'t know how to stop tasks on "{}" computing resource.'.format(task.computing))
+
+ # Ok, save status as deleted
+ task.status = 'stopped'
+ task.save()
+
+ # Check if the tunnel is active and if so kill it
+ logger.debug('Checking if task "{}" has a running tunnel'.format(task.tid))
+ check_command = 'ps -ef | grep ":'+str(task.tunnel_port)+':'+str(task.ip)+':'+str(task.port)+'" | grep -v grep | awk \'{print $2}\''
+ logger.debug(check_command)
+ out = os_shell(check_command, capture=True)
+ logger.debug(out)
+ if out.exit_code == 0:
+ logger.debug('Task "{}" has a running tunnel, killing it'.format(task.tid))
+ tunnel_pid = out.stdout
+ # Kill Tunnel command
+ kill_tunnel_command= 'kill -9 {}'.format(tunnel_pid)
+
+ # Log
+ logger.debug('Killing tunnel with command: {}'.format(kill_tunnel_command))
+
+ # Execute
+ os_shell(kill_tunnel_command, capture=True)
+ if out.exit_code != 0:
+ raise Exception(out.stderr)
@public_view
def login_view(request):
@@ -425,63 +567,9 @@ def tasks(request):
logger.error('Error in deleting task with uuid="{}": "{}"'.format(uuid, e))
return render(request, 'error.html', {'data': data})
- elif action=='stop': # or delete,a and if delete also remove object
- try:
- if task.computing == 'local':
-
- # Delete the Docker container
- if standby_supported:
- stop_command = 'sudo docker stop {}'.format(task.tid)
- else:
- stop_command = 'sudo docker stop {} && sudo docker rm {}'.format(task.tid,task.tid)
-
- out = os_shell(stop_command, capture=True)
- if out.exit_code != 0:
- raise Exception(out.stderr)
-
- elif task.computing == 'demoremote':
-
- # Stop the task remotely
- stop_command = 'ssh -4 -o StrictHostKeyChecking=no slurmclusterworker-one "kill -9 {}"'.format(task.pid)
- logger.debug(stop_command)
- out = os_shell(stop_command, capture=True)
- if out.exit_code != 0:
- if not 'No such process' in out.stderr:
- raise Exception(out.stderr)
-
- else:
- data['error']= 'Don\'t know how to stop tasks on "{}" computing resource.'.format(task.computing)
- return render(request, 'error.html', {'data': data})
-
- # Ok, save status as deleted
- task.status = 'stopped'
- task.save()
-
- # Check if the tunnel is active and if so kill it
- logger.debug('Checking if task "{}" has a running tunnel'.format(task.tid))
- check_command = 'ps -ef | grep ":'+str(task.tunnel_port)+':'+str(task.ip)+':'+str(task.port)+'" | grep -v grep | awk \'{print $2}\''
- logger.debug(check_command)
- out = os_shell(check_command, capture=True)
- logger.debug(out)
- if out.exit_code == 0:
- logger.debug('Task "{}" has a running tunnel, killing it'.format(task.tid))
- tunnel_pid = out.stdout
- # Kill Tunnel command
- kill_tunnel_command= 'kill -9 {}'.format(tunnel_pid)
-
- # Log
- logger.debug('Killing tunnel with command: {}'.format(kill_tunnel_command))
-
- # Execute
- os_shell(kill_tunnel_command, capture=True)
- if out.exit_code != 0:
- raise Exception(out.stderr)
-
- except Exception as e:
- data['error'] = 'Error in stopping the task'
- logger.error('Error in stopping task with uuid="{}": "{}"'.format(uuid, e))
- return render(request, 'error.html', {'data': data})
-
+ elif action=='stop': # or delete,a and if delete also remove object
+ stop_task(task)
+
elif action=='connect':
# Create task tunnel
@@ -584,10 +672,16 @@ def create_task(request):
data['user_containers'] = Container.objects.filter(user=request.user)
data['platform_containers'] = Container.objects.filter(user=None)
- # Task name if any
- task_name = request.POST.get('task_name', None)
+ data['computing'] = Computing.objects.filter(user=None)
+
+
+ # Step if any
+ step = request.POST.get('step', None)
- if task_name:
+ if step == 'one':
+
+ # We have a step one submitted, get the first tab parameters
+ task_name = request.POST.get('task_name', None)
# Task container
task_container_uuid = request.POST.get('task_container_uuid', None)
@@ -607,109 +701,45 @@ def create_task(request):
raise ErrorMessage('Unknown computing resource "{}')
# Generate the task uuid
- str_uuid = str(uuid.uuid4())
- str_shortuuid = str_uuid.split('-')[0]
+ task_uuid = str(uuid.uuid4())
# Create the task object
- task = Task.objects.create(uuid = str_uuid,
- user = request.user,
- name = task_name,
- status = TaskStatuses.created,
- container = task_container,
- computing = task_computing)
-
+ task = Task(uuid = task_uuid,
+ user = request.user,
+ name = task_name,
+ status = TaskStatuses.created,
+ container = task_container,
+ computing = task_computing)
- # Actually start tasks
- try:
- if task_computing == 'local':
-
- # Get our ip address
- #import netifaces
- #netifaces.ifaddresses('eth0')
- #backend_ip = netifaces.ifaddresses('eth0')[netifaces.AF_INET][0]['addr']
+ # Save the task in the cache
+ _task_cache[task_uuid] = task
- # Init run command #--cap-add=NET_ADMIN --cap-add=NET_RAW
- run_command = 'sudo docker run --network=rosetta_default --name rosetta-task-{}'.format( str_shortuuid)
+ # Set step
+ data['step'] = 'two'
+
+ elif step == 'two':
+
+ # Get back the task
+ task_uuid = request.POST.get('task_uuid', None)
+ task = _task_cache[task_uuid]
- # Data volume
- run_command += ' -v {}/task-{}:/data'.format(TASK_DATA_DIR, str_shortuuid)
+
+ # Add auth and/or computing parameters to the task if any
- # Set registry string
- if task.container.registry == 'local':
- registry_string = 'localhost:5000/'
- else:
- registry_string = ''
+ # Save the task in the DB
- # Host name, image entry command
- run_command += ' -h task-{} -d -t {}{}'.format(str_shortuuid, registry_string, task.container.image)
-
- # Run the task Debug
- logger.debug('Running new task with command="{}"'.format(run_command))
- out = os_shell(run_command, capture=True)
- if out.exit_code != 0:
- raise Exception(out.stderr)
- else:
- task_tid = out.stdout
- logger.debug('Created task with id: "{}"'.format(task_tid))
+ # Start the task
+ #start_task(task)
+ # Set step
+ data['step'] = 'created'
- # Get task IP address
- out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' ' + task_tid + ' | tail -n1', capture=True)
- if out.exit_code != 0:
- raise Exception('Error: ' + out.stderr)
- task_ip = out.stdout
-
- # Set fields
- task.tid = task_tid
- task.status = TaskStatuses.running
- task.ip = task_ip
- task.port = int(task.container.service_ports.split(',')[0])
-
- # Save
- task.save()
-
- elif task_computing == 'demoremote':
- logger.debug('Using Demo Remote as computing resource')
-
-
- # 1) Run the singularity container on slurmclusterworker-one (non blocking)
- run_command = 'ssh -4 -o StrictHostKeyChecking=no slurmclusterworker-one "export SINGULARITY_NOHTTPS=true && exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv docker://dregistry:5000/rosetta/metadesktop &> /tmp/{}.log & echo \$!"'.format(task.uuid)
- out = os_shell(run_command, capture=True)
- if out.exit_code != 0:
- raise Exception(out.stderr)
-
- # Save pid echoed by the command above
- task_pid = out.stdout
-
- # 2) Simulate the agent (i.e. report container IP and port port)
-
- # Get task IP address
- out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' slurmclusterworker-one | tail -n1', capture=True)
- if out.exit_code != 0:
- raise Exception('Error: ' + out.stderr)
- task_ip = out.stdout
-
- # Set fields
- task.tid = task.uuid
- task.status = TaskStatuses.running
- task.ip = task_ip
- task.pid = task_pid
- task.port = int(task.container.service_ports.split(',')[0])
-
- # Save
- task.save()
-
-
- else:
- raise Exception('Consistency exception: invalid computing resource "{}'.format(task_computing))
-
- except Exception as e:
- data['error'] = 'Error in creating new Task.'
- logger.error(e)
- return render(request, 'error.html', {'data': data})
+ else:
+
+ # Set step
+ data['step'] = 'one'
+
- # Set created switch
- data['created'] = True
return render(request, 'create_task.html', {'data': data})
@@ -873,14 +903,15 @@ def add_container(request):
if container_type+':'+container_registry in UNSUPPORTED_TYPES_VS_REGISTRIES:
raise ErrorMessage('Sorry, container type "{}" is not compatible with registry type "{}"'.format(container_type, container_registry))
- # Container service ports
+ # Container service ports. TODO: support multiple ports?
container_service_ports = request.POST.get('container_service_ports', None)
- try:
- for container_service_port in container_service_ports:
- int(container_service_port)
- except:
- raise ErrorMessage('Invalid service port "{}"'.format(container_service_port))
+ if container_service_ports:
+ try:
+ for container_service_port in container_service_ports.split(','):
+ int(container_service_port)
+ except:
+ raise ErrorMessage('Invalid service port(s) in "{}"'.format(container_service_ports))
# Log
logger.debug('Creating new container object with image="{}", type="{}", registry="{}", service_ports="{}"'.format(container_image, container_type, container_registry, container_service_ports))
@@ -912,7 +943,11 @@ def computings(request):
data['title'] = 'Add computing'
data['name'] = request.POST.get('name',None)
- data['computings'] = Computing.objects.all()
+ data['platform_computings'] = Computing.objects.filter(user=None)
+
+ # Attach user conf in any
+ for platform_computing in data['platform_computings']:
+ platform_computing.attach_user_conf_data(request.user)
return render(request, 'computings.html', {'data': data})
diff --git a/services/webapp/prestartup_webapp.sh b/services/webapp/prestartup_webapp.sh
index befdd9ac85231f6e79973421843ea6c12774b324..7730e0dc9517a229bb431395d9cb5bebeacd085c 100644
--- a/services/webapp/prestartup_webapp.sh
+++ b/services/webapp/prestartup_webapp.sh
@@ -8,36 +8,40 @@ chown rosetta:rosetta /var/log/webapp
mkdir -p /data/resources
chown rosetta:rosetta /data/resources
+
#-----------------------------
# Set migrations data folder
#-----------------------------
-if [[ "x$(mount | grep /devmigrations)" == "x" ]] ; then
- # If the migrations folder is not mounted (not a Docker volume), use the /data directory via links to use data persistency
+if [[ "xDJANGO_DB_NAME" == "x:memory:" ]] ; then
+ # Use the /tmp directory via links to use ephemeral data
+ mkdir -p /tmp/migrations
+ $MIGRATIONS_DATA_FOLDER=/tmp/migrations
+ echo "Using temporary migrations in $MIGRATIONS_DATA_FOLDER"
+else
+ # Use the /data directory via links to use data persistency
MIGRATIONS_DATA_FOLDER="/data/migrations"
# Also if the migrations folder in /data does not exist, create it now
mkdir -p /data/migrations
-else
- # If the migrations folder is mounted (a Docker volume), use it as we are in dev mode
- MIGRATIONS_DATA_FOLDER="/devmigrations"
+ echo "Persisting migrations in $MIGRATIONS_DATA_FOLDER"
fi
-echo "Persisting migrations in $MIGRATIONS_DATA_FOLDER"
#-----------------------------
# Handle Base App migrations
#-----------------------------
-
+
# Remove potential leftovers
rm -f /opt/webapp_code/rosetta/base_app/migrations
+
+# If migrations were not already initialized, do it now
if [ ! -d "$MIGRATIONS_DATA_FOLDER/base_app" ] ; then
- # If migrations were not already initialized, do it now
echo "Initializing migrations for base_app"...
mkdir $MIGRATIONS_DATA_FOLDER/base_app && chown rosetta:rosetta $MIGRATIONS_DATA_FOLDER/base_app
touch $MIGRATIONS_DATA_FOLDER/base_app/__init__.py && chown rosetta:rosetta $MIGRATIONS_DATA_FOLDER/base_app/__init__.py
fi
-# Use the persisted migrations
+# Use the right migrations folder
ln -s $MIGRATIONS_DATA_FOLDER/base_app /opt/webapp_code/rosetta/base_app/migrations