Skip to content
Snippets Groups Projects
Commit 73c88b8f authored by Stefano Alberto Russo's avatar Stefano Alberto Russo
Browse files

Added hopped, remote computing support. Minor fixes.

parent 9e551c06
No related branches found
No related tags found
No related merge requests found
...@@ -242,7 +242,6 @@ class RemoteComputingManager(ComputingManager): ...@@ -242,7 +242,6 @@ class RemoteComputingManager(ComputingManager):
# Stop the task remotely # Stop the task remotely
stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.pid) stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.pid)
logger.debug(stop_command)
out = os_shell(stop_command, capture=True) out = os_shell(stop_command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
if not 'No such process' in out.stderr: if not 'No such process' in out.stderr:
...@@ -265,7 +264,7 @@ class RemoteComputingManager(ComputingManager): ...@@ -265,7 +264,7 @@ class RemoteComputingManager(ComputingManager):
host = task.computing.get_conf_param('host') host = task.computing.get_conf_param('host')
user = task.computing.get_conf_param('user') user = task.computing.get_conf_param('user')
# Stop the task remotely # View log remotely
view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid) view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
out = os_shell(view_log_command, capture=True) out = os_shell(view_log_command, capture=True)
...@@ -400,7 +399,6 @@ class SlurmComputingManager(ComputingManager): ...@@ -400,7 +399,6 @@ class SlurmComputingManager(ComputingManager):
# Stop the task remotely # Stop the task remotely
stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid) stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid)
logger.debug(stop_command)
out = os_shell(stop_command, capture=True) out = os_shell(stop_command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
raise Exception(out.stderr) raise Exception(out.stderr)
...@@ -422,7 +420,7 @@ class SlurmComputingManager(ComputingManager): ...@@ -422,7 +420,7 @@ class SlurmComputingManager(ComputingManager):
host = task.computing.get_conf_param('master') host = task.computing.get_conf_param('master')
user = task.computing.get_conf_param('user') user = task.computing.get_conf_param('user')
# Stop the task remotely # View log remotely
view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid) view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
out = os_shell(view_log_command, capture=True) out = os_shell(view_log_command, capture=True)
...@@ -433,6 +431,160 @@ class SlurmComputingManager(ComputingManager): ...@@ -433,6 +431,160 @@ class SlurmComputingManager(ComputingManager):
class RemotehopComputingManager(ComputingManager):
def _start_task(self, task, **kwargs):
logger.debug('Starting a remote task "{}"'.format(task.computing))
# Get computing params
first_host = task.computing.get_conf_param('first_host')
first_user = task.computing.get_conf_param('first_user')
second_host = task.computing.get_conf_param('second_host')
second_user = task.computing.get_conf_param('second_user')
setup_command = task.computing.get_conf_param('setup_command')
# De hard-code
use_agent = False
# Get user keys
if task.computing.requires_user_keys:
user_keys = KeyPair.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
# Get webapp conn string
from.utils import get_webapp_conn_string
webapp_conn_string = get_webapp_conn_string()
# Run the container on the host (non blocking)
if task.container.type == 'singularity':
task.tid = task.uuid
task.save()
# Set pass if any
if task.auth_pass:
authstring = ' export SINGULARITYENV_AUTH_PASS={} && '.format(task.auth_pass)
else:
authstring = ''
run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
run_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} /bin/bash -c \''.format(second_user, second_host)
if use_agent:
run_command += '\'wget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> /dev/null && export BASE_PORT=\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
if setup_command:
run_command += setup_command + ' && '
run_command += '\'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
run_command += 'exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv '
else:
run_command += ' : && ' # Trick to prevent some issues in exporting variables
if setup_command:
run_command += setup_command + ' && '
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT={} && {} '.format(task.port, authstring)
run_command += 'exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv '
# Set registry
if task.container.registry == 'docker_local':
raise Exception('This computing resource does not support local Docker registries yet')
# Get local Docker registry conn string
from.utils import get_local_docker_registry_conn_string
local_docker_registry_conn_string = get_local_docker_registry_conn_string()
registry = 'docker://{}/'.format(local_docker_registry_conn_string)
elif task.container.registry == 'docker_hub':
registry = 'docker://'
else:
raise NotImplementedError('Registry {} not supported'.format(task.container.registry))
run_command+='{}{} &>> \$HOME/{}.log & echo \$!\'"'.format(registry, task.container.image, task.uuid)
else:
raise NotImplementedError('Container {} not supported'.format(task.container.type))
out = os_shell(run_command, capture=True)
if out.exit_code != 0:
raise Exception(out.stderr)
# Log
logger.debug('Shell exec output: "{}"'.format(out))
# Load back the task to avoid concurrency problems in the agent call
task_uuid = task.uuid
task = Task.objects.get(uuid=task_uuid)
# Save pid echoed by the command above
task_pid = out.stdout
# Set fields
task.status = TaskStatuses.running
task.pid = task_pid
task.ip = second_host
# Save
task.save()
def _stop_task(self, task, **kwargs):
# Get user keys
if task.computing.requires_user_keys:
user_keys = KeyPair.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
# Get computing params
first_host = task.computing.get_conf_param('first_host')
first_user = task.computing.get_conf_param('first_user')
second_host = task.computing.get_conf_param('second_host')
second_user = task.computing.get_conf_param('second_user')
# Stop the task remotely
stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
stop_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
stop_command += 'kill -9 {}"'.format(task.pid)
out = os_shell(stop_command, capture=True)
if out.exit_code != 0:
if not 'No such process' in out.stderr:
raise Exception(out.stderr)
# Set task as stopped
task.status = TaskStatuses.stopped
task.save()
def _get_task_log(self, task, **kwargs):
# Get user keys
if task.computing.requires_user_keys:
user_keys = KeyPair.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
# Get computing params
first_host = task.computing.get_conf_param('first_host')
first_user = task.computing.get_conf_param('first_user')
second_host = task.computing.get_conf_param('second_host')
second_user = task.computing.get_conf_param('second_user')
# View log remotely
view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
view_log_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
view_log_command += 'cat \\\\\\$HOME/{}.log"'.format(task.uuid)
out = os_shell(view_log_command, capture=True)
if out.exit_code != 0:
raise Exception(out.stderr)
else:
return out.stdout
......
import uuid import uuid
import json
from django.conf import settings from django.conf import settings
from django.db import models from django.db import models
from django.contrib.auth.models import User from django.contrib.auth.models import User
...@@ -158,6 +159,12 @@ class Computing(models.Model): ...@@ -158,6 +159,12 @@ class Computing(models.Model):
return ComputingSysConf.objects.get(computing=self).data return ComputingSysConf.objects.get(computing=self).data
except ComputingSysConf.DoesNotExist: except ComputingSysConf.DoesNotExist:
return None return None
@property
def sys_conf_data_json(self):
return json.dumps(self.sys_conf_data)
@property @property
def user_conf_data(self): def user_conf_data(self):
...@@ -166,6 +173,11 @@ class Computing(models.Model): ...@@ -166,6 +173,11 @@ class Computing(models.Model):
except AttributeError: except AttributeError:
raise AttributeError('User conf data is not yet attached, please attach it before accessing.') raise AttributeError('User conf data is not yet attached, please attach it before accessing.')
@property
def user_conf_data_json(self):
return json.dumps(self.user_conf_data)
def attach_user_conf_data(self, user): def attach_user_conf_data(self, user):
if self.user and self.user != user: if self.user and self.user != user:
......
...@@ -56,9 +56,8 @@ ...@@ -56,9 +56,8 @@
</tr> </tr>
<tr> <tr>
<td><b>Ports</b></td> <td colspan=2><b>Default port(s)</b>
<td> &nbsp; &nbsp;<input type="text" name="container_ports" value="" placeholder="" size="5" />
<input type="text" name="container_ports" value="" placeholder="" size="5" />
</td> </td>
</tr> </tr>
......
...@@ -77,7 +77,7 @@ ...@@ -77,7 +77,7 @@
<div style="padding:10px;"> <div style="padding:10px;">
<b>Type:</b> {{ computing.type.title }}<br/> <b>Type:</b> {{ computing.type.title }}<br/>
<b>Owner:</b> {% if computing.user %}{{ data.computing.user }}{% else %}Platform{% endif %}<br/> <b>Owner:</b> {% if computing.user %}{{ computing.user }}{% else %}Platform{% endif %}<br/>
<b>Supports:</b> <b>Supports:</b>
{% if computing.supports_docker %}Docker <img src="/static/img/docker-logo.svg" style="height:18px; width:18px; margin-bottom:2px" />{% endif %} {% if computing.supports_docker %}Docker <img src="/static/img/docker-logo.svg" style="height:18px; width:18px; margin-bottom:2px" />{% endif %}
{% if computing.supports_singularity %}Singularity <img src="/static/img/singularity-logo.svg" style="height:18px; width:18px; margin-bottom:2px" />{% endif %} {% if computing.supports_singularity %}Singularity <img src="/static/img/singularity-logo.svg" style="height:18px; width:18px; margin-bottom:2px" />{% endif %}
......
...@@ -40,8 +40,8 @@ ...@@ -40,8 +40,8 @@
</tr> </tr>
<tr> <tr>
<td><b>Ports</b></td> <td colspan=2><b>Default port(s)</b>
<td>{{ container.ports }}</td> &nbsp; &nbsp;{{ container.ports }}</td>
</tr> </tr>
<tr> <tr>
......
...@@ -130,6 +130,16 @@ ...@@ -130,6 +130,16 @@
</tr> </tr>
{% endif %} {% endif %}
{% if data.task_container.supports_dynamic_ports and data.task_computing.type == 'remotehop' %}
<tr>
<td valign="top" style="width:180px"><b>Set custom port </b></td>
<td>
<input type="text" name="task_base_port" value="" placeholder="" size="23" style="margin-bottom:5px"/><br>
<p style="line-height: 0.9"><font size=-1>This container supports dynamic ports and you can thus set a custom port (>5900) to avoid clashes with services already running on the computing resource.</font></p>
</td>
</tr>
{% endif %}
{% if data.task_container.supports_pass_auth %} {% if data.task_container.supports_pass_auth %}
<tr> <tr>
<td valign="top"><b>Set task password</b></td> <td valign="top"><b>Set task password</b></td>
......
...@@ -23,7 +23,7 @@ ...@@ -23,7 +23,7 @@
<tr> <tr>
<td> <td>
<textarea name="new_conf" style="height:300px; width:500px">{{ data.computing_conf_data_json}}</textarea> <textarea name="new_conf" style="height:300px; width:500px">{{ data.computing_conf_data_json }}</textarea>
</td> </td>
</tr> </tr>
......
...@@ -393,7 +393,21 @@ def tasks(request): ...@@ -393,7 +393,21 @@ def tasks(request):
logger.debug('Task "{}" has no running tunnel, creating it'.format(task)) logger.debug('Task "{}" has no running tunnel, creating it'.format(task))
# Tunnel command # Tunnel command
tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} localhost & '.format(task.tunnel_port, task.ip, task.port) if task.computing.type == 'remotehop':
# Get computing params
first_host = task.computing.get_conf_param('first_host')
first_user = task.computing.get_conf_param('first_user')
#second_host = task.computing.get_conf_param('second_host')
#second_user = task.computing.get_conf_param('second_user')
#setup_command = task.computing.get_conf_param('setup_command')
#base_port = task.computing.get_conf_param('base_port')
tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} {}@{} & '.format(task.tunnel_port, task.ip, task.port, first_user, first_host)
else:
tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} localhost & '.format(task.tunnel_port, task.ip, task.port)
background_tunnel_command = 'nohup {} >/dev/null 2>&1 &'.format(tunnel_command) background_tunnel_command = 'nohup {} >/dev/null 2>&1 &'.format(tunnel_command)
# Log # Log
...@@ -534,6 +548,10 @@ def create_task(request): ...@@ -534,6 +548,10 @@ def create_task(request):
task.auth_user = request.POST.get('auth_user', None) task.auth_user = request.POST.get('auth_user', None)
task.auth_pass = request.POST.get('auth_password', None) task.auth_pass = request.POST.get('auth_password', None)
task.access_method = request.POST.get('access_method', None) task.access_method = request.POST.get('access_method', None)
task_base_port = request.POST.get('task_base_port', None)
if task_base_port:
task.port = task_base_port
# Cheks # Cheks
if task.auth_pass and len(task.auth_pass) < 6: if task.auth_pass and len(task.auth_pass) < 6:
...@@ -811,15 +829,17 @@ def computings(request): ...@@ -811,15 +829,17 @@ def computings(request):
data['computing'] = Computing.objects.get(uuid=computing_uuid, user=request.user) data['computing'] = Computing.objects.get(uuid=computing_uuid, user=request.user)
except Computing.DoesNotExist: except Computing.DoesNotExist:
data['computing'] = Computing.objects.get(uuid=computing_uuid, user=None) data['computing'] = Computing.objects.get(uuid=computing_uuid, user=None)
# Attach user conf in any
data['computing'].attach_user_conf_data(request.user)
else: else:
data['computings'] = list(Computing.objects.filter(user=None)) + list(Computing.objects.filter(user=request.user)) data['computings'] = list(Computing.objects.filter(user=None)) + list(Computing.objects.filter(user=request.user))
# Attach user conf in any # Attach user conf in any
for computing in data['computings']: for computing in data['computings']:
computing.attach_user_conf_data(request.user) computing.attach_user_conf_data(request.user)
computing.user_conf_data_json = json.dumps(computing.user_conf_data)
computing.sys_conf_data_json = json.dumps(computing.sys_conf_data)
return render(request, 'computings.html', {'data': data}) return render(request, 'computings.html', {'data': data})
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment