From c450b4402cc93d6eef4fa700fe858c53f6bf7160 Mon Sep 17 00:00:00 2001
From: Stefano Alberto Russo <stefano.russo@gmail.com>
Date: Thu, 14 May 2020 20:05:30 +0200
Subject: [PATCH] Fixed bug in creating tunnels and moved from "volumes" and
"bindings" to "binds".
---
.../rosetta/core_app/computing_managers.py | 74 +++++++++----------
.../webapp/code/rosetta/core_app/models.py | 2 +-
.../core_app/templates/components/task.html | 4 +-
.../core_app/templates/create_task.html | 6 +-
.../webapp/code/rosetta/core_app/views.py | 20 +++--
5 files changed, 56 insertions(+), 50 deletions(-)
diff --git a/services/webapp/code/rosetta/core_app/computing_managers.py b/services/webapp/code/rosetta/core_app/computing_managers.py
index 0aeee44..80189bc 100644
--- a/services/webapp/code/rosetta/core_app/computing_managers.py
+++ b/services/webapp/code/rosetta/core_app/computing_managers.py
@@ -188,27 +188,27 @@ class RemoteComputingManager(ComputingManager):
else:
authstring = ''
- # Set bindings, only from sys config if the resource is not owned by the user
+ # Set binds, only from sys config if the resource is not owned by the user
if task.computing.user != task.user:
- bindings = task.computing.get_conf_param('bindings', from_sys_only=True )
+ binds = task.computing.get_conf_param('binds', from_sys_only=True )
else:
- bindings = task.computing.get_conf_param('bindings')
- if not bindings:
- bindings = ''
+ binds = task.computing.get_conf_param('binds')
+ if not binds:
+ binds = ''
else:
- bindings = '-B {}'.format(bindings)
+ binds = '-B {}'.format(binds)
- # Manage task extra volumes
- if task.extra_volumes:
- if not bindings:
- bindings = '-B {}'.format(task.extra_volumes)
+ # Manage task extra binds
+ if task.extra_binds:
+ if not binds:
+ binds = '-B {}'.format(task.extra_binds)
else:
- bindings += ',{}'.format(task.extra_volumes)
+ binds += ',{}'.format(task.extra_binds)
run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
run_command += '/bin/bash -c \'"wget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> /dev/null && export BASE_PORT=\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
- run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings)
+ run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)
# Set registry
if task.container.registry == 'docker_local':
@@ -345,27 +345,27 @@ class SlurmComputingManager(ComputingManager):
else:
authstring = ''
- # Set bindings, only from sys config if the resource is not owned by the user
+ # Set binds, only from sys config if the resource is not owned by the user
if task.computing.user != task.user:
- bindings = task.computing.get_conf_param('bindings', from_sys_only=True )
+ binds = task.computing.get_conf_param('binds', from_sys_only=True )
else:
- bindings = task.computing.get_conf_param('bindings')
- if not bindings:
- bindings = ''
+ binds = task.computing.get_conf_param('binds')
+ if not binds:
+ binds = ''
else:
- bindings = '-B {}'.format(bindings)
+ binds = '-B {}'.format(binds)
- # Manage task extra volumes
- if task.extra_volumes:
- if not bindings:
- bindings = '-B {}'.format(task.extra_volumes)
+ # Manage task extra binds
+ if task.extra_binds:
+ if not binds:
+ binds = '-B {}'.format(task.extra_binds)
else:
- bindings += ',{}'.format(task.extra_volumes)
+ binds += ',{}'.format(task.extra_binds)
run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT && {} '.format(authstring)
- run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings)
+ run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)
# Double to escape for Pythom, six for shell (double times three as \\\ escapes a single slash in shell)
@@ -498,22 +498,22 @@ class RemotehopComputingManager(ComputingManager):
else:
authstring = ''
- # Set bindings, only from sys config if the resource is not owned by the user
+ # Set binds, only from sys config if the resource is not owned by the user
if task.computing.user != task.user:
- bindings = task.computing.get_conf_param('bindings', from_sys_only=True )
+ binds = task.computing.get_conf_param('binds', from_sys_only=True )
else:
- bindings = task.computing.get_conf_param('bindings')
- if not bindings:
- bindings = ''
+ binds = task.computing.get_conf_param('binds')
+ if not binds:
+ binds = ''
else:
- bindings = '-B {}'.format(bindings)
+ binds = '-B {}'.format(binds)
- # Manage task extra volumes
- if task.extra_volumes:
- if not bindings:
- bindings = '-B {}'.format(task.extra_volumes)
+ # Manage task extra binds
+ if task.extra_binds:
+ if not binds:
+ binds = '-B {}'.format(task.extra_binds)
else:
- bindings += ',{}'.format(task.extra_volumes)
+ binds += ',{}'.format(task.extra_binds)
run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
run_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} /bin/bash -c \''.format(second_user, second_host)
@@ -523,13 +523,13 @@ class RemotehopComputingManager(ComputingManager):
if setup_command:
run_command += setup_command + ' && '
run_command += '\'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
- run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings)
+ run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)
else:
run_command += ' : && ' # Trick to prevent some issues in exporting variables
if setup_command:
run_command += setup_command + ' && '
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT={} && {} '.format(task.port, authstring)
- run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings)
+ run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)
# Set registry
if task.container.registry == 'docker_local':
diff --git a/services/webapp/code/rosetta/core_app/models.py b/services/webapp/code/rosetta/core_app/models.py
index 933ba01..ab0911a 100644
--- a/services/webapp/code/rosetta/core_app/models.py
+++ b/services/webapp/code/rosetta/core_app/models.py
@@ -272,7 +272,7 @@ class Task(models.Model):
port = models.IntegerField('Task port', blank=True, null=True)
ip = models.CharField('Task ip address', max_length=36, blank=True, null=True)
tunnel_port = models.IntegerField('Task tunnel port', blank=True, null=True)
- extra_volumes = models.CharField('Extra volumes', max_length=4096, blank=True, null=True)
+ extra_binds = models.CharField('Extra binds', max_length=4096, blank=True, null=True)
# Links
computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
diff --git a/services/webapp/code/rosetta/core_app/templates/components/task.html b/services/webapp/code/rosetta/core_app/templates/components/task.html
index bc0f165..1c48ca1 100644
--- a/services/webapp/code/rosetta/core_app/templates/components/task.html
+++ b/services/webapp/code/rosetta/core_app/templates/components/task.html
@@ -47,8 +47,8 @@
</tr>
<tr>
- <td><b>Extra volumes</b></td>
- <td>{{ task.extra_volumes }}</td>
+ <td><b>Extra binds</b></td>
+ <td>{{ task.extra_binds }}</td>
</tr>
<tr>
diff --git a/services/webapp/code/rosetta/core_app/templates/create_task.html b/services/webapp/code/rosetta/core_app/templates/create_task.html
index 4c0b303..97f281f 100644
--- a/services/webapp/code/rosetta/core_app/templates/create_task.html
+++ b/services/webapp/code/rosetta/core_app/templates/create_task.html
@@ -153,10 +153,10 @@
{% if data.task_container.type == 'singularity' %}
<tr>
- <td valign="top" style="width:180px"><b>Extra volumes</b></td>
+ <td valign="top" style="width:180px"><b>Extra binds</b></td>
<td>
<input type="text" name="extra_volumes" value="" placeholder="" size="23" /><br>
- <p style="line-height: 0.95"><font size=-1>Here you can set extra volume bindings on top of the ones already define by the administrator. Format is <i>host_directory:container_directory</i>, comma separated.</font></p>
+ <p style="line-height: 0.95"><font size=-1>Here you can set extra binds on top of the ones already define by the administrator. Format is <i>host_directory:container_directory</i>, comma separated.</font></p>
</td>
</tr>
{% endif %}
@@ -195,7 +195,7 @@
<tr><td colspan=2>
<table><tr><td style="border: 1px solid lightgray;" >
- I understand that files saved or modified in this container, if not explicitly saved to a persistent volume, will be LOST when the task ends.
+ I understand that files saved or modified in this container, if not explicitly saved to a persistent bind, will be LOST when the task ends.
</td><td style="border: 1px solid lightgray;" >
<input class="form-check-input" type="checkbox" value="" id="invalidCheck" required>
</td></table>
diff --git a/services/webapp/code/rosetta/core_app/views.py b/services/webapp/code/rosetta/core_app/views.py
index c3fb5ce..10477df 100644
--- a/services/webapp/code/rosetta/core_app/views.py
+++ b/services/webapp/code/rosetta/core_app/views.py
@@ -392,9 +392,12 @@ def tasks(request):
else:
logger.debug('Task "{}" has no running tunnel, creating it'.format(task))
+ # Get user keys
+ user_keys = KeyPair.objects.get(user=task.user, default=True)
+
# Tunnel command
if task.computing.type == 'remotehop':
-
+
# Get computing params
first_host = task.computing.get_conf_param('first_host')
first_user = task.computing.get_conf_param('first_user')
@@ -402,11 +405,14 @@ def tasks(request):
#second_user = task.computing.get_conf_param('second_user')
#setup_command = task.computing.get_conf_param('setup_command')
#base_port = task.computing.get_conf_param('base_port')
-
- tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} {}@{} & '.format(task.tunnel_port, task.ip, task.port, first_user, first_host)
+
+ tunnel_command= 'ssh -4 i {} -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} {}@{} & '.format(user_keys.private_key_file, task.tunnel_port, task.ip, task.port, first_user, first_host)
- else:
+ elif task.computing.type == 'local':
tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} localhost & '.format(task.tunnel_port, task.ip, task.port)
+
+ else:
+ tunnel_command= 'ssh -4 i {} -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} localhost & '.format(user_keys.private_key_file, task.tunnel_port, task.ip, task.port)
background_tunnel_command = 'nohup {} >/dev/null 2>&1 &'.format(tunnel_command)
@@ -561,7 +567,7 @@ def create_task(request):
computing_cpus = request.POST.get('computing_cpus', None)
computing_memory = request.POST.get('computing_memory', None)
computing_partition = request.POST.get('computing_partition', None)
- extra_volumes = request.POST.get('extra_volumes', None)
+ extra_binds = request.POST.get('extra_binds', None)
computing_options = {}
if computing_cpus:
@@ -588,8 +594,8 @@ def create_task(request):
if task.container.ports:
task.port = task.container.port
- # Set exttra volumes if any:
- task.extra_volumes = extra_volumes
+ # Set extra binds if any:
+ task.extra_binds = extra_binds
# Save the task before starting it, or the computing manager will not be able to work properly
task.save()
--
GitLab