From ee9076f7339f473f04e8f0b8686cbbb28fae0337 Mon Sep 17 00:00:00 2001
From: Stefano Alberto Russo <stefano.russo@gmail.com>
Date: Fri, 1 May 2020 16:11:56 +0200
Subject: [PATCH] Added support for computing parameters and for stopping Slurm
Jobs, improved agent and output file handling. Minor fixes
---
services/webapp/code/rosetta/base_app/api.py | 1 -
.../rosetta/base_app/computing_managers.py | 104 ++++++++++++++----
.../management/commands/base_app_populate.py | 80 +++++++-------
.../webapp/code/rosetta/base_app/models.py | 35 +++---
.../base_app/templates/add_container.html | 30 ++++-
.../templates/components/container.html | 36 +++++-
.../base_app/templates/create_task.html | 40 +++++--
.../webapp/code/rosetta/base_app/views.py | 79 ++++++++++---
8 files changed, 302 insertions(+), 103 deletions(-)
diff --git a/services/webapp/code/rosetta/base_app/api.py b/services/webapp/code/rosetta/base_app/api.py
index 6bbcb2c..ff89265 100644
--- a/services/webapp/code/rosetta/base_app/api.py
+++ b/services/webapp/code/rosetta/base_app/api.py
@@ -308,7 +308,6 @@ print(port)
logger.info('Setting task "{}" to ip "{}" and port "{}"'.format(task.uuid, task_ip, task_port))
task.status = TaskStatuses.running
task.ip = task_ip
- #task.pid = task_pid
task.port = int(task_port)
task.save()
return HttpResponse('OK')
diff --git a/services/webapp/code/rosetta/base_app/computing_managers.py b/services/webapp/code/rosetta/base_app/computing_managers.py
index 4258d2a..2a49a13 100644
--- a/services/webapp/code/rosetta/base_app/computing_managers.py
+++ b/services/webapp/code/rosetta/base_app/computing_managers.py
@@ -72,6 +72,7 @@ class ComputingManager(object):
return self._get_task_log(task, **kwargs)
+
class LocalComputingManager(ComputingManager):
def _start_task(self, task):
@@ -134,6 +135,11 @@ class LocalComputingManager(ComputingManager):
out = os_shell(stop_command, capture=True)
if out.exit_code != 0:
raise Exception(out.stderr)
+
+ # Set task as stopped
+ task.status = TaskStatuses.stopped
+ task.save()
+
def _get_task_log(self, task, **kwargs):
@@ -147,6 +153,7 @@ class LocalComputingManager(ComputingManager):
return out.stdout
+
class RemoteComputingManager(ComputingManager):
def _start_task(self, task, **kwargs):
@@ -157,7 +164,7 @@ class RemoteComputingManager(ComputingManager):
user = task.computing.get_conf_param('user')
# Get user keys
- if task.computing.require_user_keys:
+ if task.computing.require_user_auth_keys:
user_keys = Keys.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
@@ -219,9 +226,8 @@ class RemoteComputingManager(ComputingManager):
task_pid = out.stdout
# Set fields
-
- #task.status = TaskStatuses.sumbitted
- task.pid = task_pid
+ #task.status = TaskStatuses.running
+ task.pid = task_pid
# Save
task.save()
@@ -230,7 +236,7 @@ class RemoteComputingManager(ComputingManager):
def _stop_task(self, task, **kwargs):
# Get user keys
- if task.computing.require_user_keys:
+ if task.computing.require_user_auth_keys:
user_keys = Keys.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
@@ -247,13 +253,17 @@ class RemoteComputingManager(ComputingManager):
if not 'No such process' in out.stderr:
raise Exception(out.stderr)
+ # Set task as stopped
+ task.status = TaskStatuses.stopped
+ task.save()
+
def _get_task_log(self, task, **kwargs):
# Get computing host
host = task.computing.get_conf_param('host')
# Get id_rsa
- if task.computing.require_user_keys:
+ if task.computing.require_user_auth_keys:
user_keys = Keys.objects.get(user=task.user, default=True)
id_rsa_file = user_keys.private_key_file
else:
@@ -280,16 +290,34 @@ class SlurmComputingManager(ComputingManager):
user = task.computing.get_conf_param('user')
# Get user keys
- if task.computing.require_user_keys:
+ if task.computing.require_user_auth_keys:
user_keys = Keys.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
+ # Get task computing parameters and set sbatch args
+ sbatch_args = ''
+ if task.computing_options:
+ task_partition = task.computing_options.get('partition', None)
+ task_cpus = task.computing_options.get('cpus', None)
+ task_memory = task.computing_options.get('memory', None)
+
+ # Set sbatch args
+ sbatch_args = ''
+ if task_partition:
+ sbatch_args += '-p {} '.format(task_partition)
+ #if task_cpus:
+ # sbatch_args += '-c {} '.format()
+ #if task_memory:
+ # sbatch_args += '-m {} '.format()
+
+ # Set output and error files
+ sbatch_args += ' --output=\$HOME/{}.log --error=\$HOME/{}.log '.format(task.uuid, task.uuid)
+
# 1) Run the container on the host (non blocking)
-
if task.container.type == 'singularity':
- if not task.container.dynamic_ports:
+ if not task.container.supports_dynamic_ports:
raise Exception('This task does not support dynamic port allocation and is therefore not supported using singularity on Slurm')
# Set pass if any
@@ -304,18 +332,12 @@ class SlurmComputingManager(ComputingManager):
run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
- run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}:8080/api/v1/base/agent/?task_uuid={} -O /tmp/agent_{}.py &> /dev/null && export BASE_PORT=\\\\\\$(python /tmp/agent_{}.py 2> /tmp/{}.log) && '.format(webapp_ip, task.uuid, task.uuid, task.uuid, task.uuid)
+ run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}:8080/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> /dev/null && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_ip, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT && {} '.format(authstring)
run_command += 'exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv '
-
-
+
# Double to escape for python six for shell (double times three as \\\ escapes a single slash in shell)
- # ssh -i /rosetta/.ssh/id_rsa -4 -o StrictHostKeyChecking=no slurmclustermaster-main "echo \"wget 172.18.0.5:8080/api/v1/base/agent/?task_uuid=558c65c3-8b72-4d6b-8119-e1dcf6f81177 -O /tmp/agent_558c65c3-8b72-4d6b-8119-e1dcf6f81177.py &> /dev/null
- # && export BASE_PORT=\\\$(python /tmp/agent_558c65c3-8b72-4d6b-8119-e1dcf6f81177.py 2> /tmp/558c65c3-8b72-4d6b-8119-e1dcf6f81177.log) && export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\$BASE_PORT && export SINGULARITYENV_AUTH_PASS=testpass
- # && exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv docker://dregistry:5000/rosetta/metadesktop &> /tmp/558c65c3-8b72-4d6b-8119-e1dcf6f81177.log\" > /tmp/558c65c3-8b72-4d6b-8119-e1dcf6f81177.sh"
-
-
# Set registry
if task.container.registry == 'docker_local':
registry = 'docker://dregistry:5000/'
@@ -324,7 +346,7 @@ class SlurmComputingManager(ComputingManager):
else:
raise NotImplementedError('Registry {} not supported'.format(task.container.registry))
- run_command+='{}{} &> /tmp/{}.log\\" > /tmp/{}.sh && sbatch -p partition1 /tmp/{}.sh"\''.format(registry, task.container.image, task.uuid, task.uuid, task.uuid)
+ run_command+='{}{} &> \$HOME/{}.log\\" > \$HOME/{}.sh && sbatch {} \$HOME/{}.sh"\''.format(registry, task.container.image, task.uuid, task.uuid, sbatch_args, task.uuid)
else:
@@ -334,9 +356,53 @@ class SlurmComputingManager(ComputingManager):
if out.exit_code != 0:
raise Exception(out.stderr)
+ # Log
+ logger.debug('Shell exec output: "{}"'.format(out))
+
+ # Parse sbatch output. Example: Output(stdout='Submitted batch job 3', stderr='', exit_code=0)
+ job_id = out.stdout.split(' ')[-1]
+ try:
+ int(job_id)
+ except:
+ raise Exception('Cannot find int job id from output string "{}"'.format(out.stdout))
+
+ # Load back the task to avoid concurrency problems in the agent call
+ task_uuid = task.uuid
+ task = Task.objects.get(uuid=task_uuid)
+
+ # Save job id as task pid
+ task.pid = job_id
+
+ # Set status (only fi we get here before the agent which sets the status as running via the API)
+ if task.status != TaskStatuses.running:
+ task.status = TaskStatuses.sumbitted
+
+ # Save
+ task.save()
+
def _stop_task(self, task, **kwargs):
- raise NotImplementedError('Not implemented')
+
+ # Get user keys
+ if task.computing.require_user_auth_keys:
+ user_keys = Keys.objects.get(user=task.user, default=True)
+ else:
+ raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
+
+ # Get computing host
+ host = task.computing.get_conf_param('master')
+ user = task.computing.get_conf_param('user')
+
+ # Stop the task remotely
+ stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid)
+ logger.debug(stop_command)
+ out = os_shell(stop_command, capture=True)
+ if out.exit_code != 0:
+ raise Exception(out.stderr)
+
+ # Set task as topped
+ task.status = TaskStatuses.stopped
+ task.save()
def _get_task_log(self, task, **kwargs):
diff --git a/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py b/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py
index 28ce4ac..bb9a77b 100644
--- a/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py
+++ b/services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py
@@ -48,33 +48,37 @@ class Command(BaseCommand):
print('Creating public containers...')
# MetaDesktop Docker
- Container.objects.create(user = None,
- name = 'MetaDesktop latest',
- image = 'rosetta/metadesktop',
- type = 'docker',
- registry = 'docker_local',
- default_ports = '8590',
- dynamic_ports = True,
- require_pass = True)
+ Container.objects.create(user = None,
+ name = 'MetaDesktop latest',
+ image = 'rosetta/metadesktop',
+ type = 'docker',
+ registry = 'docker_local',
+ ports = '8590',
+ supports_dynamic_ports = True,
+ supports_user_auth = False,
+ supports_pass_auth = True)
# MetaDesktop Singularity
- Container.objects.create(user = None,
- name = 'MetaDesktop latest',
- image = 'rosetta/metadesktop',
- type = 'singularity',
- registry = 'docker_local',
- default_ports = '8590',
- dynamic_ports = True,
- require_pass = True)
+ Container.objects.create(user = None,
+ name = 'MetaDesktop latest',
+ image = 'rosetta/metadesktop',
+ type = 'singularity',
+ registry = 'docker_local',
+ ports = '8590',
+ supports_dynamic_ports = True,
+ supports_user_auth = False,
+ supports_pass_auth = True)
# Astrocook
- Container.objects.create(user = None,
- name = 'Astrocook b2b819e',
- image = 'sarusso/astrocook:b2b819e',
- type = 'docker',
- registry = 'docker_local',
- dynamic_ports = False,
- default_ports = '8590')
+ Container.objects.create(user = None,
+ name = 'Astrocook b2b819e',
+ image = 'sarusso/astrocook:b2b819e',
+ type = 'docker',
+ registry = 'docker_local',
+ ports = '8590',
+ supports_dynamic_ports = False,
+ supports_user_auth = False,
+ supports_pass_auth = False)
# Private containers
@@ -85,13 +89,15 @@ class Command(BaseCommand):
print('Creating testuser private containers...')
# JuPyter
- Container.objects.create(user = testuser,
- name = 'Jupyter Notebook latest',
- image = 'jupyter/base-notebook',
- type = 'docker',
- registry = 'docker_hub',
- dynamic_ports = False,
- default_ports = '8888')
+ Container.objects.create(user = testuser,
+ name = 'Jupyter Notebook latest',
+ image = 'jupyter/base-notebook',
+ type = 'docker',
+ registry = 'docker_hub',
+ ports = '8888',
+ supports_dynamic_ports = False,
+ supports_user_auth = False,
+ supports_pass_auth = False)
# Computing resources
computing_resources = Computing.objects.all()
@@ -107,8 +113,8 @@ class Command(BaseCommand):
name = 'Local',
type = 'local',
require_sys_conf = False,
- require_user_conf = False,
- require_user_keys = False)
+ require_user_auth_conf = False,
+ require_user_auth_keys = False)
#==============================
@@ -118,8 +124,8 @@ class Command(BaseCommand):
name = 'Demo remote',
type = 'remote',
require_sys_conf = True,
- require_user_conf = True,
- require_user_keys = True)
+ require_user_auth_conf = True,
+ require_user_auth_keys = True)
ComputingSysConf.objects.create(computing = demo_remote_auth_computing,
data = {'host': 'slurmclusterworker-one'})
@@ -136,12 +142,12 @@ class Command(BaseCommand):
name = 'Demo Slurm',
type = 'slurm',
require_sys_conf = True,
- require_user_conf = True,
- require_user_keys = True)
+ require_user_auth_conf = True,
+ require_user_auth_keys = True)
# Create demo slurm sys computing conf
ComputingSysConf.objects.create(computing = demo_slurm_computing,
- data = {'master': 'slurmclusterworker-master'})
+ data = {'master': 'slurmclustermaster-main'})
# Create demo slurm user computing conf
ComputingUserConf.objects.create(user = testuser,
diff --git a/services/webapp/code/rosetta/base_app/models.py b/services/webapp/code/rosetta/base_app/models.py
index 2110a1a..28aef52 100644
--- a/services/webapp/code/rosetta/base_app/models.py
+++ b/services/webapp/code/rosetta/base_app/models.py
@@ -76,18 +76,20 @@ class Container(models.Model):
user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True)
# If a container has no user, it will be available to anyone. Can be created, edited and deleted only by admins.
- name = models.CharField('Container Name', max_length=255, blank=False, null=False)
- image = models.CharField('Container image', max_length=255, blank=False, null=False)
- type = models.CharField('Container type', max_length=36, blank=False, null=False)
- registry = models.CharField('Container registry', max_length=255, blank=False, null=False)
- default_ports = models.CharField('Container service ports', max_length=36, blank=True, null=True)
- dynamic_ports = models.BooleanField(default=False)
- require_user = models.BooleanField(default=False)
- require_pass = models.BooleanField(default=False)
+ name = models.CharField('Container Name', max_length=255, blank=False, null=False)
+ image = models.CharField('Container image', max_length=255, blank=False, null=False)
+ type = models.CharField('Container type', max_length=36, blank=False, null=False)
+ registry = models.CharField('Container registry', max_length=255, blank=False, null=False)
+ ports = models.CharField('Container service ports', max_length=36, blank=True, null=True)
+
+ # Capabilities
+ supports_dynamic_ports = models.BooleanField(default=False)
+ supports_user_auth = models.BooleanField(default=False)
+ supports_pass_auth = models.BooleanField(default=False)
def __str__(self):
- return str('Container of type "{}" with image "{}" with service ports "{}" from registry "{}" of user "{}"'.format(self.type, self.image, self.default_ports, self.registry, self.user))
+ return str('Container of type "{}" with image "{}" and ports "{}" from registry "{}" of user "{}"'.format(self.type, self.image, self.ports, self.registry, self.user))
@property
@@ -110,8 +112,8 @@ class Computing(models.Model):
type = models.CharField('Computing Type', max_length=255, blank=False, null=False)
require_sys_conf = models.BooleanField(default=False)
- require_user_conf = models.BooleanField(default=False)
- require_user_keys = models.BooleanField(default=False)
+ require_user_auth_conf = models.BooleanField(default=False)
+ require_user_auth_keys = models.BooleanField(default=False)
def __str__(self):
@@ -223,6 +225,8 @@ class Task(models.Model):
auth_pass = models.CharField('Task auth pass', max_length=36, blank=True, null=True)
access_method = models.CharField('Task access method', max_length=36, blank=True, null=True)
+ # Computing options
+ computing_options = JSONField(blank=True, null=True)
def save(self, *args, **kwargs):
@@ -234,11 +238,6 @@ class Task(models.Model):
# Call parent save
super(Task, self).save(*args, **kwargs)
-
- def __str__(self):
- return str('Task "{}" of user "{}" in status "{}" (TID "{}")'.format(self.name, self.user.email, self.status, self.tid))
-
-
def update_status(self):
if self.computing == 'local':
@@ -266,6 +265,10 @@ class Task(models.Model):
return str(self.uuid).split('-')[0]
+ def __str__(self):
+ return str('Task "{}" of user "{}" running on "{}" in status "{}" created at "{}"'.format(self.name, self.user, self.computing, self.status, self.created))
+
+
#=========================
# Keys
diff --git a/services/webapp/code/rosetta/base_app/templates/add_container.html b/services/webapp/code/rosetta/base_app/templates/add_container.html
index bd5811e..4ff7f72 100644
--- a/services/webapp/code/rosetta/base_app/templates/add_container.html
+++ b/services/webapp/code/rosetta/base_app/templates/add_container.html
@@ -23,7 +23,7 @@
<table class="dashboard" style="max-width:430px">
<tr>
- <td><b>Container name</b></td>
+ <td><b>Name</b></td>
<td>
<input type="text" name="container_name" value="" placeholder="" size="23" required />
</td>
@@ -49,19 +49,41 @@
</tr>
<tr>
- <td><b>Container image</b></td>
+ <td><b>Image</b></td>
<td>
<input type="text" name="container_image" value="" placeholder="" size="23" required />
</td>
</tr>
<tr>
- <td><b>Service port</b></td>
+ <td><b>Ports</b></td>
<td>
- <input type="text" name="container_service_port" value="" placeholder="" size="5" />
+ <input type="text" name="container_ports" value="" placeholder="" size="5" />
</td>
</tr>
+ <tr>
+ <td colspan=2>
+ <b>Supports dynamic ports</b>
+ <input type="checkbox" name="container_supports_dynamic_ports" value="True" />
+ </td>
+ </tr>
+
+ <tr>
+ <td colspan=2>
+ <b>Supports user auth</b>
+ <input type="checkbox" name="container_supports_user_auth" value="True" />
+ </td>
+ </tr>
+
+ <tr>
+ <td colspan=2>
+ <b>Supports pass auth</b>
+ <input type="checkbox" name="container_supports_pass_auth" value="True" />
+ </td>
+ </tr>
+
+
<tr>
<td colspan=2 align=center style="padding:20px">
<input type="submit" value="Add">
diff --git a/services/webapp/code/rosetta/base_app/templates/components/container.html b/services/webapp/code/rosetta/base_app/templates/components/container.html
index 41486ee..072d1a4 100644
--- a/services/webapp/code/rosetta/base_app/templates/components/container.html
+++ b/services/webapp/code/rosetta/base_app/templates/components/container.html
@@ -31,9 +31,41 @@
</tr>
<tr>
- <td><b>Service port(s)</b></td>
- <td>{{ container.default_ports}}</td>
+ <td><b>Ports</b></td>
+ <td>{{ container.ports }}</td>
</tr>
+
+ <tr>
+ <td colspan=2>
+ <b>Supports dynamic ports</b>
+ {% if container.supports_dynamic_ports %}
+ <input type="checkbox" name="container_supports_user_auth" checked disabled/>
+ {% else %}
+ <input type="checkbox" name="container_supports_user_auth" disabled/>
+ {% endif %}
+ </td>
+ </tr>
+
+ <tr>
+ <td colspan=2>
+ <b>Supports user auth</b>
+ {% if container.supports_user_auth %}
+ <input type="checkbox" name="supports_user_auth" checked disabled/>
+ {% else %}
+ <input type="checkbox" name="supports_user_auth" disabled/>
+ {% endif %} </td>
+ </tr>
+
+ <tr>
+ <td colspan=2>
+ <b>Supports pass auth</b>
+ {% if container.supports_pass_auth %}
+ <input type="checkbox" name="supports_pass_auth" checked disabled/>
+ {% else %}
+ <input type="checkbox" name="supports_pass_auth" disabled/>
+ {% endif %} </td>
+ </tr>
+
{% if container.user %}
<tr>
diff --git a/services/webapp/code/rosetta/base_app/templates/create_task.html b/services/webapp/code/rosetta/base_app/templates/create_task.html
index ccc1e7a..3e3ca9d 100644
--- a/services/webapp/code/rosetta/base_app/templates/create_task.html
+++ b/services/webapp/code/rosetta/base_app/templates/create_task.html
@@ -23,7 +23,8 @@
<table class="dashboard" style="max-width:700px">
-
+ <tr><td></td></tr>
+
<tr>
<td><b>Task name </b></td>
<td>
@@ -68,17 +69,29 @@
{% elif data.step == 'two' %}
<h3>Step 2: authentication and computing details</h3>
-
+
<br/>
+ {% if data.task.container.type == 'singularity' and not data.task.container.supports_dynamic_ports %}
+ <div> <p style="font-size:15px; max-width:700px; margin-bottom:20px">
+ <i class="fa fa-exclamation-triangle" style="color:orange"></i> This container does not support dynamic ports and you are running it with Singularity, without network insulation. This means that if the container port is already occupied, it will not be able to start.
+ </p></div>
+ {% endif %}
+
+ {% if data.task.container.ports and not data.task.container.supports_pass_auth %}
+ <div> <p style="font-size:15px; max-width:700px; margin-bottom:20px">
+ <i class="fa fa-exclamation-triangle" style="color:orange"></i> This container does not support any authentication. This means that anyone running on the same network will be able to access it services.
+ </p></div>
+ {% endif %}
+
<form action="/create_task/" method="POST">
{% csrf_token %}
<input type="hidden" name="step" value="two" />
<input type="hidden" name="task_uuid" value="{{ data.task.uuid }}" />
<table class="dashboard" style="max-width:700px">
-
- {% if data.task.container.require_user %}
+ <tr><td></td></tr>
+ {% if data.task.container.supports_user_auth %}
<tr>
<td><b>Task user</b></td>
<td>
@@ -87,7 +100,7 @@
</tr>
{% endif %}
- {% if data.task.container.require_pass %}
+ {% if data.task.container.supports_pass_auth %}
<tr>
<td valign="top"><b>Set task password</b></td>
<td>
@@ -100,7 +113,7 @@
<tr>
<td><b>Access method</b></td><td>
<select name="access_method" >
- <option value="http_proxy" disabled>HTTP proxy</option>
+ <option value="https_proxy" disabled>HTTPS proxy</option>
<option value="direct_tunnel" selected>Direct tunnel</option>
<option value="None">None</option>
</select>
@@ -115,7 +128,20 @@
</select>
</td>
</tr> -->
-
+
+ {% if data.task.computing.type == 'slurm' %}
+ <tr>
+ <td><b>Computing options</b></td>
+ <td>
+ <table>
+ <tr><td>Partition</td><td><input type="text" name="computing_partition" value="" placeholder="" size="20" /></td></tr>
+ <tr><td>Cpus</td><td><input type="text" name="computing_cpus" value="" placeholder="" size="5" /></td></tr>
+ <tr><td>Memory</td><td><input type="text" name="computing_memory" value="" placeholder="" size="5" /></td></tr>
+ </table>
+ </td>
+ </tr>
+ {% endif %}
+
<tr><td colspan=2>
<table><tr><td style="border: 1px solid lightgray;" >
I understand that files saved or modified in this container, if not explicitly saved to a persistent storage, will be LOST when the task ends.
diff --git a/services/webapp/code/rosetta/base_app/views.py b/services/webapp/code/rosetta/base_app/views.py
index 95cb157..af2fd9d 100644
--- a/services/webapp/code/rosetta/base_app/views.py
+++ b/services/webapp/code/rosetta/base_app/views.py
@@ -310,7 +310,7 @@ def tasks(request):
tunnel_port = port
break
if not tunnel_port:
- logger.error('Cannot find a free port for the tunnel for task "{}"'.format(task.tid))
+ logger.error('Cannot find a free port for the tunnel for task "{}"'.format(task))
raise ErrorMessage('Cannot find a free port for the tunnel to the task')
task.tunnel_port = tunnel_port
@@ -318,14 +318,14 @@ def tasks(request):
# Check if the tunnel is active and if not create it
- logger.debug('Checking if task "{}" has a running tunnel'.format(task.tid))
+ logger.debug('Checking if task "{}" has a running tunnel'.format(task))
out = os_shell('ps -ef | grep ":{}:{}:{}" | grep -v grep'.format(task.tunnel_port, task.ip, task.port), capture=True)
if out.exit_code == 0:
- logger.debug('Task "{}" has a running tunnel, using it'.format(task.tid))
+ logger.debug('Task "{}" has a running tunnel, using it'.format(task))
else:
- logger.debug('Task "{}" has no running tunnel, creating it'.format(task.tid))
+ logger.debug('Task "{}" has no running tunnel, creating it'.format(task))
# Tunnel command
tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} localhost & '.format(task.tunnel_port, task.ip, task.port)
@@ -451,8 +451,29 @@ def create_task(request):
if len(task.auth_pass) < 6:
raise ErrorMessage('Task password must be at least 6 chars')
- # Add auth and/or computing parameters to the task if any
- # TODO... (i..e num cores)
+ # Computing options # TODO: This is hardcoded thinking about Slurm
+ computing_cpus = request.POST.get('computing_cpus', None)
+ computing_memory = request.POST.get('computing_memory', None)
+ computing_partition = request.POST.get('computing_partition', None)
+
+ computing_options = {}
+ if computing_cpus:
+ try:
+ int(computing_cpus)
+ except:
+ raise Exception('Cannot convert computing_cpus to int')
+ computing_options['cpus'] = int(computing_cpus)
+
+ if computing_memory:
+ computing_options['memory'] = computing_memory
+
+ if computing_partition:
+ computing_options['partition'] = computing_partition
+
+ if computing_options:
+ task.computing_options = computing_options
+
+ logger.debug('computing_options="{}"'.format(computing_options))
# Save the task in the DB
task.save()
@@ -621,25 +642,49 @@ def add_container(request):
container_name = request.POST.get('container_name', None)
# Container service ports. TODO: support multiple ports?
- container_default_ports = request.POST.get('container_default_ports', None)
+ container_ports = request.POST.get('container_ports', None)
- if container_default_ports:
+ if container_ports:
try:
- for container_service_port in container_default_ports.split(','):
+ for container_service_port in container_ports.split(','):
int(container_service_port)
except:
- raise ErrorMessage('Invalid service port(s) in "{}"'.format(container_default_ports))
+ raise ErrorMessage('Invalid container port(s) in "{}"'.format(container_ports))
+
+
+ # Capabilities
+ container_supports_dynamic_ports = request.POST.get('container_supports_dynamic_ports', None)
+ if container_supports_dynamic_ports and container_supports_dynamic_ports == 'True':
+ container_supports_dynamic_ports = True
+ else:
+ container_supports_dynamic_ports = False
+
+ container_supports_user_auth = request.POST.get('container_supports_user_auth', None)
+ if container_supports_user_auth and container_supports_user_auth == 'True':
+ container_supports_user_auth = True
+ else:
+ container_supports_user_auth = False
+
+ container_supports_pass_auth = request.POST.get('container_supports_pass_auth', None)
+ if container_supports_pass_auth and container_supports_pass_auth == 'True':
+ container_supports_pass_auth = True
+ else:
+ container_supports_pass_auth = False
# Log
- logger.debug('Creating new container object with image="{}", type="{}", registry="{}", default_ports="{}"'.format(container_image, container_type, container_registry, container_default_ports))
+ logger.debug('Creating new container object with image="{}", type="{}", registry="{}", ports="{}"'.format(container_image, container_type, container_registry, container_ports))
# Create
- Container.objects.create(user = request.user,
- image = container_image,
- name = container_name,
- type = container_type,
- registry = container_registry,
- default_ports = container_default_ports)
+ Container.objects.create(user = request.user,
+ image = container_image,
+ name = container_name,
+ type = container_type,
+ registry = container_registry,
+ ports = container_ports,
+ supports_dynamic_ports = container_supports_dynamic_ports,
+ supports_user_auth = container_supports_user_auth,
+ supports_pass_auth = container_supports_pass_auth,
+ )
# Set added switch
data['added'] = True
--
GitLab