Skip to content
Snippets Groups Projects
Commit c450b440 authored by Stefano Alberto Russo's avatar Stefano Alberto Russo
Browse files

Fixed bug in creating tunnels and moved from "volumes" and "bindings" to "binds".

parent aefaca3d
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
services/webapp/code/rosetta/core_app/computing_managers.py
+ 37
− 37
View file @ c450b440
...@@ -188,27 +188,27 @@ class RemoteComputingManager(ComputingManager): ...@@ -188,27 +188,27 @@ class RemoteComputingManager(ComputingManager):
else: else:
authstring = '' authstring = ''
# Set bindings, only from sys config if the resource is not owned by the user # Set binds, only from sys config if the resource is not owned by the user
if task.computing.user != task.user: if task.computing.user != task.user:
bindings = task.computing.get_conf_param('bindings', from_sys_only=True ) binds = task.computing.get_conf_param('binds', from_sys_only=True )
else: else:
bindings = task.computing.get_conf_param('bindings') binds = task.computing.get_conf_param('binds')
if not bindings: if not binds:
bindings = '' binds = ''
else: else:
bindings = '-B {}'.format(bindings) binds = '-B {}'.format(binds)
# Manage task extra volumes # Manage task extra binds
if task.extra_volumes: if task.extra_binds:
if not bindings: if not binds:
bindings = '-B {}'.format(task.extra_volumes) binds = '-B {}'.format(task.extra_binds)
else: else:
bindings += ',{}'.format(task.extra_volumes) binds += ',{}'.format(task.extra_binds)
run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host) run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
run_command += '/bin/bash -c \'"wget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> /dev/null && export BASE_PORT=\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid) run_command += '/bin/bash -c \'"wget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> /dev/null && export BASE_PORT=\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring) run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings) run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)
# Set registry # Set registry
if task.container.registry == 'docker_local': if task.container.registry == 'docker_local':
...@@ -345,27 +345,27 @@ class SlurmComputingManager(ComputingManager): ...@@ -345,27 +345,27 @@ class SlurmComputingManager(ComputingManager):
else: else:
authstring = '' authstring = ''
# Set bindings, only from sys config if the resource is not owned by the user # Set binds, only from sys config if the resource is not owned by the user
if task.computing.user != task.user: if task.computing.user != task.user:
bindings = task.computing.get_conf_param('bindings', from_sys_only=True ) binds = task.computing.get_conf_param('binds', from_sys_only=True )
else: else:
bindings = task.computing.get_conf_param('bindings') binds = task.computing.get_conf_param('binds')
if not bindings: if not binds:
bindings = '' binds = ''
else: else:
bindings = '-B {}'.format(bindings) binds = '-B {}'.format(binds)
# Manage task extra volumes # Manage task extra binds
if task.extra_volumes: if task.extra_binds:
if not bindings: if not binds:
bindings = '-B {}'.format(task.extra_volumes) binds = '-B {}'.format(task.extra_binds)
else: else:
bindings += ',{}'.format(task.extra_volumes) binds += ',{}'.format(task.extra_binds)
run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host) run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid) run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT && {} '.format(authstring) run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT && {} '.format(authstring)
run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings) run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)
# Double to escape for Pythom, six for shell (double times three as \\\ escapes a single slash in shell) # Double to escape for Pythom, six for shell (double times three as \\\ escapes a single slash in shell)
...@@ -498,22 +498,22 @@ class RemotehopComputingManager(ComputingManager): ...@@ -498,22 +498,22 @@ class RemotehopComputingManager(ComputingManager):
else: else:
authstring = '' authstring = ''
# Set bindings, only from sys config if the resource is not owned by the user # Set binds, only from sys config if the resource is not owned by the user
if task.computing.user != task.user: if task.computing.user != task.user:
bindings = task.computing.get_conf_param('bindings', from_sys_only=True ) binds = task.computing.get_conf_param('binds', from_sys_only=True )
else: else:
bindings = task.computing.get_conf_param('bindings') binds = task.computing.get_conf_param('binds')
if not bindings: if not binds:
bindings = '' binds = ''
else: else:
bindings = '-B {}'.format(bindings) binds = '-B {}'.format(binds)
# Manage task extra volumes # Manage task extra binds
if task.extra_volumes: if task.extra_binds:
if not bindings: if not binds:
bindings = '-B {}'.format(task.extra_volumes) binds = '-B {}'.format(task.extra_binds)
else: else:
bindings += ',{}'.format(task.extra_volumes) binds += ',{}'.format(task.extra_binds)
run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host) run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
run_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} /bin/bash -c \''.format(second_user, second_host) run_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} /bin/bash -c \''.format(second_user, second_host)
...@@ -523,13 +523,13 @@ class RemotehopComputingManager(ComputingManager): ...@@ -523,13 +523,13 @@ class RemotehopComputingManager(ComputingManager):
if setup_command: if setup_command:
run_command += setup_command + ' && ' run_command += setup_command + ' && '
run_command += '\'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring) run_command += '\'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings) run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)
else: else:
run_command += ' : && ' # Trick to prevent some issues in exporting variables run_command += ' : && ' # Trick to prevent some issues in exporting variables
if setup_command: if setup_command:
run_command += setup_command + ' && ' run_command += setup_command + ' && '
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT={} && {} '.format(task.port, authstring) run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT={} && {} '.format(task.port, authstring)
run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings) run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)
# Set registry # Set registry
if task.container.registry == 'docker_local': if task.container.registry == 'docker_local':
......
services/webapp/code/rosetta/core_app/models.py
+ 1
− 1
View file @ c450b440
...@@ -272,7 +272,7 @@ class Task(models.Model): ...@@ -272,7 +272,7 @@ class Task(models.Model):
port = models.IntegerField('Task port', blank=True, null=True) port = models.IntegerField('Task port', blank=True, null=True)
ip = models.CharField('Task ip address', max_length=36, blank=True, null=True) ip = models.CharField('Task ip address', max_length=36, blank=True, null=True)
tunnel_port = models.IntegerField('Task tunnel port', blank=True, null=True) tunnel_port = models.IntegerField('Task tunnel port', blank=True, null=True)
extra_volumes = models.CharField('Extra volumes', max_length=4096, blank=True, null=True) extra_binds = models.CharField('Extra binds', max_length=4096, blank=True, null=True)
# Links # Links
computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE) computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
......
services/webapp/code/rosetta/core_app/templates/components/task.html
+ 2
− 2
View file @ c450b440
...@@ -47,8 +47,8 @@ ...@@ -47,8 +47,8 @@
</tr> </tr>
<tr> <tr>
<td><b>Extra volumes</b></td> <td><b>Extra binds</b></td>
<td>{{ task.extra_volumes }}</td> <td>{{ task.extra_binds }}</td>
</tr> </tr>
<tr> <tr>
......
services/webapp/code/rosetta/core_app/templates/create_task.html
+ 3
− 3
View file @ c450b440
...@@ -153,10 +153,10 @@ ...@@ -153,10 +153,10 @@
{% if data.task_container.type == 'singularity' %} {% if data.task_container.type == 'singularity' %}
<tr> <tr>
<td valign="top" style="width:180px"><b>Extra volumes</b></td> <td valign="top" style="width:180px"><b>Extra binds</b></td>
<td> <td>
<input type="text" name="extra_volumes" value="" placeholder="" size="23" /><br> <input type="text" name="extra_volumes" value="" placeholder="" size="23" /><br>
<p style="line-height: 0.95"><font size=-1>Here you can set extra volume bindings on top of the ones already define by the administrator. Format is <i>host_directory:container_directory</i>, comma separated.</font></p> <p style="line-height: 0.95"><font size=-1>Here you can set extra binds on top of the ones already define by the administrator. Format is <i>host_directory:container_directory</i>, comma separated.</font></p>
</td> </td>
</tr> </tr>
{% endif %} {% endif %}
...@@ -195,7 +195,7 @@ ...@@ -195,7 +195,7 @@
<tr><td colspan=2> <tr><td colspan=2>
<table><tr><td style="border: 1px solid lightgray;" > <table><tr><td style="border: 1px solid lightgray;" >
I understand that files saved or modified in this container, if not explicitly saved to a persistent volume, will be LOST when the task ends. I understand that files saved or modified in this container, if not explicitly saved to a persistent bind, will be LOST when the task ends.
</td><td style="border: 1px solid lightgray;" > </td><td style="border: 1px solid lightgray;" >
<input class="form-check-input" type="checkbox" value="" id="invalidCheck" required> <input class="form-check-input" type="checkbox" value="" id="invalidCheck" required>
</td></table> </td></table>
......
services/webapp/code/rosetta/core_app/views.py
+ 13
− 7
View file @ c450b440
...@@ -392,9 +392,12 @@ def tasks(request): ...@@ -392,9 +392,12 @@ def tasks(request):
else: else:
logger.debug('Task "{}" has no running tunnel, creating it'.format(task)) logger.debug('Task "{}" has no running tunnel, creating it'.format(task))
# Get user keys
user_keys = KeyPair.objects.get(user=task.user, default=True)
# Tunnel command # Tunnel command
if task.computing.type == 'remotehop': if task.computing.type == 'remotehop':
# Get computing params # Get computing params
first_host = task.computing.get_conf_param('first_host') first_host = task.computing.get_conf_param('first_host')
first_user = task.computing.get_conf_param('first_user') first_user = task.computing.get_conf_param('first_user')
...@@ -402,11 +405,14 @@ def tasks(request): ...@@ -402,11 +405,14 @@ def tasks(request):
#second_user = task.computing.get_conf_param('second_user') #second_user = task.computing.get_conf_param('second_user')
#setup_command = task.computing.get_conf_param('setup_command') #setup_command = task.computing.get_conf_param('setup_command')
#base_port = task.computing.get_conf_param('base_port') #base_port = task.computing.get_conf_param('base_port')
tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} {}@{} & '.format(task.tunnel_port, task.ip, task.port, first_user, first_host) tunnel_command= 'ssh -4 i {} -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} {}@{} & '.format(user_keys.private_key_file, task.tunnel_port, task.ip, task.port, first_user, first_host)
else: elif task.computing.type == 'local':
tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} localhost & '.format(task.tunnel_port, task.ip, task.port) tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} localhost & '.format(task.tunnel_port, task.ip, task.port)
else:
tunnel_command= 'ssh -4 i {} -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} localhost & '.format(user_keys.private_key_file, task.tunnel_port, task.ip, task.port)
background_tunnel_command = 'nohup {} >/dev/null 2>&1 &'.format(tunnel_command) background_tunnel_command = 'nohup {} >/dev/null 2>&1 &'.format(tunnel_command)
...@@ -561,7 +567,7 @@ def create_task(request): ...@@ -561,7 +567,7 @@ def create_task(request):
computing_cpus = request.POST.get('computing_cpus', None) computing_cpus = request.POST.get('computing_cpus', None)
computing_memory = request.POST.get('computing_memory', None) computing_memory = request.POST.get('computing_memory', None)
computing_partition = request.POST.get('computing_partition', None) computing_partition = request.POST.get('computing_partition', None)
extra_volumes = request.POST.get('extra_volumes', None) extra_binds = request.POST.get('extra_binds', None)
computing_options = {} computing_options = {}
if computing_cpus: if computing_cpus:
...@@ -588,8 +594,8 @@ def create_task(request): ...@@ -588,8 +594,8 @@ def create_task(request):
if task.container.ports: if task.container.ports:
task.port = task.container.port task.port = task.container.port
# Set exttra volumes if any: # Set extra binds if any:
task.extra_volumes = extra_volumes task.extra_binds = extra_binds
# Save the task before starting it, or the computing manager will not be able to work properly # Save the task before starting it, or the computing manager will not be able to work properly
task.save() task.save()
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment