diff --git a/services/webapp/code/rosetta/core_app/api.py b/services/webapp/code/rosetta/core_app/api.py
index ab6e55d964bd3e95ad1f5fb5adb091697e90cc98..591b7b31e295efead33b71bee8eb73ba980cb5ee 100644
--- a/services/webapp/code/rosetta/core_app/api.py
+++ b/services/webapp/code/rosetta/core_app/api.py
@@ -372,13 +372,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
user_keys = KeyPair.objects.get(user=user, default=True)
# Get computing host
- computing_host = computing.get_conf_param('host')
+ computing_host = computing.conf.get('host')
# Trick for handling Slurm.. TODO: fix me!
if not computing_host:
- computing_host = computing.get_conf_param('master')
+ computing_host = computing.conf.get('master')
- computing_user = computing.get_conf_param('user')
+ computing_user = computing.conf.get('user')
if not computing_host:
raise Exception('No computing host?!')
@@ -404,13 +404,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
user_keys = KeyPair.objects.get(user=user, default=True)
# Get computing host
- computing_host = computing.get_conf_param('host')
+ computing_host = computing.conf.get('host')
# Trick for handling Slurm.. TODO: fix me!
if not computing_host:
- computing_host = computing.get_conf_param('master')
+ computing_host = computing.conf.get('master')
- computing_user = computing.get_conf_param('user')
+ computing_user = computing.conf.get('user')
if not computing_host:
raise Exception('No computing host?!')
@@ -457,7 +457,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
computing = computing[0]
# Attach user conf in any
- computing.attach_user_conf_data(request.user)
+ computing.attach_user_conf(request.user)
return computing
@@ -710,9 +710,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
computings = list(Computing.objects.filter(user=None)) + list(Computing.objects.filter(user=request.user))
for computing in computings:
-
+
+ # For now, we only support SSH-based computing resources
+ if not 'ssh' in computing.access_method:
+ continue
+
# Attach user conf in any
- computing.attach_user_conf_data(request.user)
+ computing.attach_user_conf(request.user)
data['data'].append({
'id': '/{}/'.format(computing.name),
@@ -733,9 +737,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
# TODO: we can remove this and just always filter agains bind probably...
if len(path.split('/')) == 3:
if computing.user != request.user:
- binds = computing.get_conf_param('binds', from_sys_only=True )
+ binds = computing.sys_conf.get('binds')
else:
- binds = computing.get_conf_param('binds')
+ binds = computing.conf.get('binds')
if binds:
binds = binds.split(',')
diff --git a/services/webapp/code/rosetta/core_app/computing_managers.py b/services/webapp/code/rosetta/core_app/computing_managers.py
index b3cb204f68aa18681caa95018a00feae402aa7d3..08cd271d37dc19fa660b9efb6ee75383c4019c84 100644
--- a/services/webapp/code/rosetta/core_app/computing_managers.py
+++ b/services/webapp/code/rosetta/core_app/computing_managers.py
@@ -10,6 +10,9 @@ logger = logging.getLogger(__name__)
class ComputingManager(object):
+ def __init__(self, computing):
+ self.computing = computing
+
def start_task(self, task, **kwargs):
# Check for run task logic implementation
@@ -70,8 +73,21 @@ class ComputingManager(object):
return self._get_task_log(task, **kwargs)
+class SingleNodeComputingManager(ComputingManager):
+ pass
+
+
+class ClusterComputingManager(ComputingManager):
+ pass
+
-class LocalComputingManager(ComputingManager):
+class SSHComputingManager(ComputingManager):
+ # SSH-f + keys utils here
+ pass
+
+
+
+class InternalSingleNodeComputingManager(SingleNodeComputingManager):
def _start_task(self, task):
@@ -158,17 +174,21 @@ class LocalComputingManager(ComputingManager):
-class RemoteComputingManager(ComputingManager):
+
+
+
+
+class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingManager):
def _start_task(self, task, **kwargs):
- logger.debug('Starting a remote task "{}"'.format(task.computing))
+ logger.debug('Starting a remote task "{}"'.format(self.computing))
# Get computing host
- host = task.computing.get_conf_param('host')
- user = task.computing.get_conf_param('user')
+ host = self.computing.conf.get('host')
+ user = self.computing.conf.get('user')
# Get user keys
- if task.computing.requires_user_keys:
+ if self.computing.requires_user_keys:
user_keys = KeyPair.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
@@ -190,10 +210,10 @@ class RemoteComputingManager(ComputingManager):
authstring = ''
# Set binds, only from sys config if the resource is not owned by the user
- if task.computing.user != task.user:
- binds = task.computing.get_conf_param('binds', from_sys_only=True )
+ if self.computing.user != task.user:
+ binds = self.computing.sys_conf.get('binds')
else:
- binds = task.computing.get_conf_param('binds')
+ binds = self.computing.conf.get('binds')
if not binds:
binds = ''
else:
@@ -253,14 +273,14 @@ class RemoteComputingManager(ComputingManager):
def _stop_task(self, task, **kwargs):
# Get user keys
- if task.computing.requires_user_keys:
+ if self.computing.requires_user_keys:
user_keys = KeyPair.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
# Get computing host
- host = task.computing.get_conf_param('host')
- user = task.computing.get_conf_param('user')
+ host = self.computing.conf.get('host')
+ user = self.computing.conf.get('user')
# Stop the task remotely
stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.pid)
@@ -277,14 +297,14 @@ class RemoteComputingManager(ComputingManager):
def _get_task_log(self, task, **kwargs):
# Get user keys
- if task.computing.requires_user_keys:
+ if self.computing.requires_user_keys:
user_keys = KeyPair.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
# Get computing host
- host = task.computing.get_conf_param('host')
- user = task.computing.get_conf_param('user')
+ host = self.computing.conf.get('host')
+ user = self.computing.conf.get('user')
# View log remotely
view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}_data/task.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
@@ -297,17 +317,17 @@ class RemoteComputingManager(ComputingManager):
-class SlurmComputingManager(ComputingManager):
+class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManager):
def _start_task(self, task, **kwargs):
- logger.debug('Starting a remote task "{}"'.format(task.computing))
+ logger.debug('Starting a remote task "{}"'.format(self.computing))
# Get computing host
- host = task.computing.get_conf_param('master')
- user = task.computing.get_conf_param('user')
+ host = self.computing.conf.get('master')
+ user = self.computing.conf.get('user')
# Get user keys
- if task.computing.requires_user_keys:
+ if self.computing.requires_user_keys:
user_keys = KeyPair.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
@@ -349,10 +369,10 @@ class SlurmComputingManager(ComputingManager):
authstring = ''
# Set binds, only from sys config if the resource is not owned by the user
- if task.computing.user != task.user:
- binds = task.computing.get_conf_param('binds', from_sys_only=True )
+ if self.computing.user != task.user:
+ binds = self.computing.sys_conf.get('binds')
else:
- binds = task.computing.get_conf_param('binds')
+ binds = self.computing.conf.get('binds')
if not binds:
binds = ''
else:
@@ -422,14 +442,14 @@ class SlurmComputingManager(ComputingManager):
def _stop_task(self, task, **kwargs):
# Get user keys
- if task.computing.requires_user_keys:
+ if self.computing.requires_user_keys:
user_keys = KeyPair.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
# Get computing host
- host = task.computing.get_conf_param('master')
- user = task.computing.get_conf_param('user')
+ host = self.computing.conf.get('master')
+ user = self.computing.conf.get('user')
# Stop the task remotely
stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid)
@@ -445,14 +465,14 @@ class SlurmComputingManager(ComputingManager):
def _get_task_log(self, task, **kwargs):
# Get user keys
- if task.computing.requires_user_keys:
+ if self.computing.requires_user_keys:
user_keys = KeyPair.objects.get(user=task.user, default=True)
else:
raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
# Get computing host
- host = task.computing.get_conf_param('master')
- user = task.computing.get_conf_param('user')
+ host = self.computing.conf.get('master')
+ user = self.computing.conf.get('user')
# View log remotely
view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
@@ -464,173 +484,173 @@ class SlurmComputingManager(ComputingManager):
return out.stdout
-
-class RemotehopComputingManager(ComputingManager):
-
- def _start_task(self, task, **kwargs):
- logger.debug('Starting a remote task "{}"'.format(task.computing))
-
- # Get computing params
- first_host = task.computing.get_conf_param('first_host')
- first_user = task.computing.get_conf_param('first_user')
- second_host = task.computing.get_conf_param('second_host')
- second_user = task.computing.get_conf_param('second_user')
- setup_command = task.computing.get_conf_param('setup_command')
-
- # TODO: De hard-code
- use_agent = False
-
- # Get user keys
- if task.computing.requires_user_keys:
- user_keys = KeyPair.objects.get(user=task.user, default=True)
- else:
- raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
-
- # Get webapp conn string
- from.utils import get_webapp_conn_string
- webapp_conn_string = get_webapp_conn_string()
-
- # Run the container on the host (non blocking)
- if task.container.type == 'singularity':
-
- task.tid = task.uuid
- task.save()
-
- # Set pass if any
- if task.auth_pass:
- authstring = ' export SINGULARITYENV_AUTH_PASS={} && '.format(task.auth_pass)
- else:
- authstring = ''
-
- # Set binds, only from sys config if the resource is not owned by the user
- if task.computing.user != task.user:
- binds = task.computing.get_conf_param('binds', from_sys_only=True )
- else:
- binds = task.computing.get_conf_param('binds')
- if not binds:
- binds = ''
- else:
- binds = '-B {}'.format(binds)
-
- # Manage task extra binds
- if task.extra_binds:
- if not binds:
- binds = '-B {}'.format(task.extra_binds)
- else:
- binds += ',{}'.format(task.extra_binds)
-
- run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
- run_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} /bin/bash -c \''.format(second_user, second_host)
-
- if use_agent:
- run_command += '\'wget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> /dev/null && export BASE_PORT=\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
- if setup_command:
- run_command += setup_command + ' && '
- run_command += '\'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
- run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
- run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)
- else:
- run_command += ' : && ' # Trick to prevent some issues in exporting variables
- if setup_command:
- run_command += setup_command + ' && '
- run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT={} && {} '.format(task.port, authstring)
- run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
- run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)
-
- # Set registry
- if task.container.registry == 'docker_local':
- raise Exception('This computing resource does not support local Docker registries yet')
- # Get local Docker registry conn string
- from.utils import get_local_docker_registry_conn_string
- local_docker_registry_conn_string = get_local_docker_registry_conn_string()
- registry = 'docker://{}/'.format(local_docker_registry_conn_string)
- elif task.container.registry == 'docker_hub':
- registry = 'docker://'
- else:
- raise NotImplementedError('Registry {} not supported'.format(task.container.registry))
-
- run_command+='{}{} &>> \$HOME/{}.log & echo \$!\'"'.format(registry, task.container.image, task.uuid)
-
- else:
- raise NotImplementedError('Container {} not supported'.format(task.container.type))
-
- out = os_shell(run_command, capture=True)
- if out.exit_code != 0:
- raise Exception(out.stderr)
-
- # Log
- logger.debug('Shell exec output: "{}"'.format(out))
-
-
- # Load back the task to avoid concurrency problems in the agent call
- task_uuid = task.uuid
- task = Task.objects.get(uuid=task_uuid)
-
- # Save pid echoed by the command above
- task_pid = out.stdout
-
- # Set fields
- task.status = TaskStatuses.running
- task.pid = task_pid
- task.ip = second_host
-
- # Save
- task.save()
-
-
- def _stop_task(self, task, **kwargs):
-
- # Get user keys
- if task.computing.requires_user_keys:
- user_keys = KeyPair.objects.get(user=task.user, default=True)
- else:
- raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
-
- # Get computing params
- first_host = task.computing.get_conf_param('first_host')
- first_user = task.computing.get_conf_param('first_user')
- second_host = task.computing.get_conf_param('second_host')
- second_user = task.computing.get_conf_param('second_user')
-
- # Stop the task remotely
- stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
- stop_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
- stop_command += 'kill -9 {}"'.format(task.pid)
-
- out = os_shell(stop_command, capture=True)
- if out.exit_code != 0:
- if not 'No such process' in out.stderr:
- raise Exception(out.stderr)
-
- # Set task as stopped
- task.status = TaskStatuses.stopped
- task.save()
-
-
- def _get_task_log(self, task, **kwargs):
-
- # Get user keys
- if task.computing.requires_user_keys:
- user_keys = KeyPair.objects.get(user=task.user, default=True)
- else:
- raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
-
- # Get computing params
- first_host = task.computing.get_conf_param('first_host')
- first_user = task.computing.get_conf_param('first_user')
- second_host = task.computing.get_conf_param('second_host')
- second_user = task.computing.get_conf_param('second_user')
-
- # View log remotely
- view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
- view_log_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
- view_log_command += 'cat \\\\\\$HOME/{}.log"'.format(task.uuid)
-
- out = os_shell(view_log_command, capture=True)
- if out.exit_code != 0:
- raise Exception(out.stderr)
- else:
- return out.stdout
+# TODO: rename the following as "ssh+ssh" access mode? Ore somethign similar?
+# class RemotehopComputingManager(ComputingManager):
+#
+# def _start_task(self, task, **kwargs):
+# logger.debug('Starting a remote task "{}"'.format(self.computing))
+#
+# # Get computing params
+# first_host = self.computing.conf.get('first_host')
+# first_user = self.computing.conf.get('first_user')
+# second_host = self.computing.conf.get('second_host')
+# second_user = self.computing.conf.get('second_user')
+# setup_command = self.computing.conf.get('setup_command')
+#
+# # TODO: De hard-code
+# use_agent = False
+#
+# # Get user keys
+# if self.computing.requires_user_keys:
+# user_keys = KeyPair.objects.get(user=task.user, default=True)
+# else:
+# raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
+#
+# # Get webapp conn string
+# from.utils import get_webapp_conn_string
+# webapp_conn_string = get_webapp_conn_string()
+#
+# # Run the container on the host (non blocking)
+# if task.container.type == 'singularity':
+#
+# task.tid = task.uuid
+# task.save()
+#
+# # Set pass if any
+# if task.auth_pass:
+# authstring = ' export SINGULARITYENV_AUTH_PASS={} && '.format(task.auth_pass)
+# else:
+# authstring = ''
+#
+# # Set binds, only from sys config if the resource is not owned by the user
+# if self.computing.user != task.user:
+# binds = self.computing.sys_conf.get('binds')
+# else:
+# binds = self.computing.conf.get('binds')
+# if not binds:
+# binds = ''
+# else:
+# binds = '-B {}'.format(binds)
+#
+# # Manage task extra binds
+# if task.extra_binds:
+# if not binds:
+# binds = '-B {}'.format(task.extra_binds)
+# else:
+# binds += ',{}'.format(task.extra_binds)
+#
+# run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
+# run_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} /bin/bash -c \''.format(second_user, second_host)
+#
+# if use_agent:
+# run_command += '\'wget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> /dev/null && export BASE_PORT=\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
+# if setup_command:
+# run_command += setup_command + ' && '
+# run_command += '\'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
+# run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
+# run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)
+# else:
+# run_command += ' : && ' # Trick to prevent some issues in exporting variables
+# if setup_command:
+# run_command += setup_command + ' && '
+# run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT={} && {} '.format(task.port, authstring)
+# run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
+# run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)
+#
+# # Set registry
+# if task.container.registry == 'docker_local':
+# raise Exception('This computing resource does not support local Docker registries yet')
+# # Get local Docker registry conn string
+# from.utils import get_local_docker_registry_conn_string
+# local_docker_registry_conn_string = get_local_docker_registry_conn_string()
+# registry = 'docker://{}/'.format(local_docker_registry_conn_string)
+# elif task.container.registry == 'docker_hub':
+# registry = 'docker://'
+# else:
+# raise NotImplementedError('Registry {} not supported'.format(task.container.registry))
+#
+# run_command+='{}{} &>> \$HOME/{}.log & echo \$!\'"'.format(registry, task.container.image, task.uuid)
+#
+# else:
+# raise NotImplementedError('Container {} not supported'.format(task.container.type))
+#
+# out = os_shell(run_command, capture=True)
+# if out.exit_code != 0:
+# raise Exception(out.stderr)
+#
+# # Log
+# logger.debug('Shell exec output: "{}"'.format(out))
+#
+#
+# # Load back the task to avoid concurrency problems in the agent call
+# task_uuid = task.uuid
+# task = Task.objects.get(uuid=task_uuid)
+#
+# # Save pid echoed by the command above
+# task_pid = out.stdout
+#
+# # Set fields
+# task.status = TaskStatuses.running
+# task.pid = task_pid
+# task.ip = second_host
+#
+# # Save
+# task.save()
+#
+#
+# def _stop_task(self, task, **kwargs):
+#
+# # Get user keys
+# if self.computing.requires_user_keys:
+# user_keys = KeyPair.objects.get(user=task.user, default=True)
+# else:
+# raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
+#
+# # Get computing params
+# first_host = self.computing.conf.get('first_host')
+# first_user = self.computing.conf.get('first_user')
+# second_host = self.computing.conf.get('second_host')
+# second_user = self.computing.conf.get('second_user')
+#
+# # Stop the task remotely
+# stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
+# stop_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
+# stop_command += 'kill -9 {}"'.format(task.pid)
+#
+# out = os_shell(stop_command, capture=True)
+# if out.exit_code != 0:
+# if not 'No such process' in out.stderr:
+# raise Exception(out.stderr)
+#
+# # Set task as stopped
+# task.status = TaskStatuses.stopped
+# task.save()
+#
+#
+# def _get_task_log(self, task, **kwargs):
+#
+# # Get user keys
+# if self.computing.requires_user_keys:
+# user_keys = KeyPair.objects.get(user=task.user, default=True)
+# else:
+# raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
+#
+# # Get computing params
+# first_host = self.computing.conf.get('first_host')
+# first_user = self.computing.conf.get('first_user')
+# second_host = self.computing.conf.get('second_host')
+# second_user = self.computing.conf.get('second_user')
+#
+# # View log remotely
+# view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
+# view_log_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
+# view_log_command += 'cat \\\\\\$HOME/{}.log"'.format(task.uuid)
+#
+# out = os_shell(view_log_command, capture=True)
+# if out.exit_code != 0:
+# raise Exception(out.stderr)
+# else:
+# return out.stdout
diff --git a/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py b/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py
index 8bcdfd0f20bb8f5e7f170c64861ffcf84226ffd6..feadcdfb88fd27dce06dde834639558d9ad8b1b2 100644
--- a/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py
+++ b/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py
@@ -209,11 +209,12 @@ class Command(BaseCommand):
print('Creating demo computing resources containers...')
#==============================
- # Local remote computing
+ # Demo Internal computing
#==============================
Computing.objects.create(user = None,
- name = 'Local',
- type = 'local',
+ name = 'Demo Internal',
+ type = 'singlenode',
+ access_method = 'internal',
requires_sys_conf = False,
requires_user_conf = False,
requires_user_keys = False,
@@ -222,32 +223,34 @@ class Command(BaseCommand):
#==============================
- # Demo remote computing
+ # Demo Single Node computing
#==============================
- demo_remote_auth_computing = Computing.objects.create(user = None,
- name = 'Demo remote',
- type = 'remote',
- requires_sys_conf = True,
- requires_user_conf = True,
- requires_user_keys = True,
- supports_docker = True,
- supports_singularity = True)
+ demo_singlenode_computing = Computing.objects.create(user = None,
+ name = 'Demo Single Node',
+ type = 'singlenode',
+ access_method = 'ssh',
+ requires_sys_conf = True,
+ requires_user_conf = True,
+ requires_user_keys = True,
+ supports_docker = True,
+ supports_singularity = True)
- ComputingSysConf.objects.create(computing = demo_remote_auth_computing,
+ ComputingSysConf.objects.create(computing = demo_singlenode_computing,
data = {'host': 'slurmclusterworker-one',
'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'})
ComputingUserConf.objects.create(user = testuser,
- computing = demo_remote_auth_computing,
+ computing = demo_singlenode_computing,
data = {'user': 'slurmtestuser'})
#==============================
- # Demo Slurm computing
+ # Demo Cluster computing
#==============================
demo_slurm_computing = Computing.objects.create(user = None,
- name = 'Demo Slurm',
- type = 'slurm',
+ name = 'Demo Cluster',
+ type = 'cluster',
+ access_method = 'slurm+ssh',
requires_sys_conf = True,
requires_user_conf = True,
requires_user_keys = True,
diff --git a/services/webapp/code/rosetta/core_app/migrations/0004_auto_20210408_1041.py b/services/webapp/code/rosetta/core_app/migrations/0004_auto_20210408_1041.py
new file mode 100644
index 0000000000000000000000000000000000000000..2df6120032df977c623724de00afbb7e77290333
--- /dev/null
+++ b/services/webapp/code/rosetta/core_app/migrations/0004_auto_20210408_1041.py
@@ -0,0 +1,30 @@
+# Generated by Django 2.2.1 on 2021-04-08 10:41
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core_app', '0003_text'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='computing',
+ name='access_method',
+ field=models.CharField(default='NA', max_length=255, verbose_name='Computing Access method'),
+ preserve_default=False,
+ ),
+ migrations.AlterField(
+ model_name='computingsysconf',
+ name='computing',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='related_sys_conf', to='core_app.Computing'),
+ ),
+ migrations.AlterField(
+ model_name='computinguserconf',
+ name='computing',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='related_user_conf', to='core_app.Computing'),
+ ),
+ ]
diff --git a/services/webapp/code/rosetta/core_app/models.py b/services/webapp/code/rosetta/core_app/models.py
index a3dffde50d8ad1b33f8f25fc884e7efffaf62d8a..3ec486ff8a8b6126b04e9ce44e3ba4df6d12baa4 100644
--- a/services/webapp/code/rosetta/core_app/models.py
+++ b/services/webapp/code/rosetta/core_app/models.py
@@ -5,6 +5,7 @@ from django.db import models
from django.contrib.auth.models import User
from django.utils import timezone
from .utils import os_shell, color_map, hash_string_to_int
+from .exceptions import ConsistencyException
if 'sqlite' in settings.DATABASES['default']['ENGINE']:
from .fields import JSONField
@@ -138,6 +139,7 @@ class Computing(models.Model):
name = models.CharField('Computing Name', max_length=255, blank=False, null=False)
type = models.CharField('Computing Type', max_length=255, blank=False, null=False)
+ access_method = models.CharField('Computing Access method', max_length=255, blank=False, null=False)
requires_sys_conf = models.BooleanField(default=False)
requires_user_conf = models.BooleanField(default=False)
@@ -146,50 +148,72 @@ class Computing(models.Model):
supports_docker = models.BooleanField(default=False)
supports_singularity = models.BooleanField(default=False)
+ @property
+ def type_str(self):
+ # TODO: improve me?
+ if self.type == 'cluster':
+ return 'Cluster'
+ elif self.type == 'singlenode':
+ return 'Single Node'
+ else:
+ raise ConsistencyException('Unknown computing resource type "{}"'.format(self.type))
+
+ @property
+ def access_method_str(self):
+ # TODO: improve me?
+ access_method = self.access_method
+ access_method = access_method.replace('ssh', 'SSH')
+ access_method = access_method.replace('slurm', 'Slurm')
+ return access_method
class Meta:
ordering = ['name']
-
def __str__(self):
if self.user:
return str('Computing "{}" of user "{}"'.format(self.name, self.user))
else:
return str('Computing "{}"'.format(self.name))
-
@property
def id(self):
return str(self.uuid).split('-')[0]
+ @property
+ def color(self):
+ string_int_hash = hash_string_to_int(self.name)
+ color_map_index = string_int_hash % len(color_map)
+ return color_map[color_map_index]
- @property
- def sys_conf_data(self):
- try:
- return ComputingSysConf.objects.get(computing=self).data
- except ComputingSysConf.DoesNotExist:
- return None
-
-
- @property
- def sys_conf_data_json(self):
- return json.dumps(self.sys_conf_data)
+ #=======================
+ # Computing manager
+ #=======================
- @property
- def user_conf_data(self):
+ @property
+ def manager(self):
+ from . import computing_managers
+
+ # Instantiate the computing manager based on type (if not already done)
try:
- return self._user_conf_data
+ return self._manager
except AttributeError:
- raise AttributeError('User conf data is not yet attached, please attach it before accessing.')
-
-
- @property
- def user_conf_data_json(self):
- return json.dumps(self.user_conf_data)
-
+ if self.type == 'cluster' and self.access_method == 'slurm+ssh':
+ self._manager = computing_managers.SlurmSSHClusterComputingManager(self)
+ elif self.type == 'singlenode' and self.access_method == 'ssh':
+ self._manager = computing_managers.SSHSingleNodeComputingManager(self)
+ elif self.type == 'singlenode' and self.access_method == 'internal':
+ self._manager = computing_managers.InternalSingleNodeComputingManager(self)
+ else:
+ raise ConsistencyException('Don\'t know how to instantiate a computing manager for computing resource of type "{}" and access mode "{}"'.format(self.type, self.access_method))
+ return self._manager
+
- def attach_user_conf_data(self, user):
+ #=======================
+ # Sys & user conf
+ #=======================
+
+ def attach_user_conf(self, user):
if self.user and self.user != user:
raise Exception('Cannot attach a conf data for another user (my user="{}", another user="{}"'.format(self.user, user))
try:
@@ -197,47 +221,50 @@ class Computing(models.Model):
except ComputingUserConf.DoesNotExist:
self._user_conf_data = None
+ @property
+ def sys_conf(self):
+ return self.related_sys_conf.get().data
- def get_conf_param(self, param, from_sys_only=False):
+ @property
+ def user_conf(self):
try:
- param_value = self.sys_conf_data[param]
- except (TypeError, KeyError):
- if not from_sys_only:
- try:
- param_value = self.user_conf_data[param]
- except (TypeError, KeyError):
- return None
- else:
- return None
- return param_value
+ return self._user_conf_data
+ except AttributeError:
+ raise ConsistencyException('User conf has to been attached, cannot proceed.')
- @property
- def conf_params(self):
- class ConfParams():
- def __init__(self, computing):
- self.computing = computing
- def __getitem__(self, key):
- return self.computing.get_conf_param(key)
- return ConfParams(self)
+ @property
+ def sys_conf_as_json(self):
+ return json.dumps(self.sys_conf)
+
+ @property
+ def user_conf_as_json(self):
+ return json.dumps(self.user_conf)
@property
- def manager(self):
- from . import computing_managers
- ComputingManager = getattr(computing_managers, '{}ComputingManager'.format(self.type.title()))
- return ComputingManager()
-
- @ property
- def color(self):
- string_int_hash = hash_string_to_int(self.name)
- color_map_index = string_int_hash % len(color_map)
- return color_map[color_map_index]
+ def conf(self):
+
+ if not self.requires_user_conf:
+ conf_tmp = self.sys_conf
+ else:
+ try:
+ # Copy the conf or the original user conf will be affected by the overwrite below
+ conf_tmp = {key:value for key, value in self._user_conf_data.items()}
+ except AttributeError:
+ raise ConsistencyException('User conf has not been attached, cannot proceed.')
+
+ # Now add (overwrite) with the sys conf
+ sys_conf = self.sys_conf
+ for key in sys_conf:
+ conf_tmp[key] = sys_conf[key]
+ return conf_tmp
+
class ComputingSysConf(models.Model):
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
- computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
+ computing = models.ForeignKey(Computing, related_name='related_sys_conf', on_delete=models.CASCADE)
data = JSONField(blank=True, null=True)
@@ -255,7 +282,7 @@ class ComputingUserConf(models.Model):
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True)
- computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
+ computing = models.ForeignKey(Computing, related_name='related_user_conf', on_delete=models.CASCADE)
data = JSONField(blank=True, null=True)
@property
diff --git a/services/webapp/code/rosetta/core_app/templates/components/computing.html b/services/webapp/code/rosetta/core_app/templates/components/computing.html
index 34799c0be528362ec383cbfd6101e8110ca63a71..a8e5284a4b30cb397ae50e63cb8f51b67e7ed3eb 100644
--- a/services/webapp/code/rosetta/core_app/templates/components/computing.html
+++ b/services/webapp/code/rosetta/core_app/templates/components/computing.html
@@ -24,13 +24,18 @@
<td>{{ data.computing.type }}</td>
</tr>
+ <tr>
+ <td><b>Access method</b></td>
+ <td>{{ data.computing.access_method }}</td>
+ </tr>
+
<tr>
<td><b>Owner</b></td>
<td>{% if data.computing.user %}{{ data.computing.user }}{% else %}platform{% endif %}</td>
</tr>
<tr>
- <td><b>Require</b></td>
+ <td><b>Requires</b></td>
<td>
Sys conf: {{ data.computing.requires_sys_conf }} <br/>
User conf: {{ data.computing.requires_user_conf }} <br/>
@@ -48,12 +53,12 @@
<tr>
<td><b>Sys Conf</b></td>
- <td>{{ data.computing.sys_conf_data_json }} {% if request.user.is_superuser %} [<a href="/edit_computing_conf?type=sys&computing_uuid={{ data.computing.uuid}}">Edit</a>] {% endif %}</td>
+ <td>{{ data.computing.sys_conf_as_json }} {% if request.user.is_superuser %} [<a href="/edit_computing_conf?type=sys&computing_uuid={{ data.computing.uuid}}">Edit</a>] {% endif %}</td>
</tr>
<tr>
<td><b>User Conf</b></td>
- <td>{{ data.computing.user_conf_data_json }} [<a href="/edit_computing_conf?type=user&computing_uuid={{ data.computing.uuid}}">Edit</a>]</td>
+ <td>{{ data.computing.user_conf_as_json }} [<a href="/edit_computing_conf?type=user&computing_uuid={{ data.computing.uuid}}">Edit</a>]</td>
</tr>
@@ -76,8 +81,9 @@
</div>
<div style="padding:10px;">
- <b>Type:</b> {{ computing.type.title }}<br/>
- <b>Owner:</b> {% if computing.user %}{{ computing.user }}{% else %}Platform{% endif %}<br/>
+ <b>Type:</b> {{ computing.type_str }}<br/>
+ <b>Access:</b> {{ computing.access_method_str }}<br/>
+ <!-- <b>Owner:</b> {% if computing.user %}{{ computing.user }}{% else %}Platform{% endif %}<br/> -->
<b>Supports:</b>
{% if computing.supports_docker %}Docker <img src="/static/img/docker-logo.svg" style="height:18px; width:18px; margin-bottom:2px" />{% endif %}
{% if computing.supports_singularity %}Singularity <img src="/static/img/singularity-logo.svg" style="height:18px; width:18px; margin-bottom:2px" />{% endif %}
diff --git a/services/webapp/code/rosetta/core_app/templates/create_task.html b/services/webapp/code/rosetta/core_app/templates/create_task.html
index c22487addd4b2dff567e2cba17ed96af4c21a807..3bfd578400fade9c634358cf97042f5c9479f839 100644
--- a/services/webapp/code/rosetta/core_app/templates/create_task.html
+++ b/services/webapp/code/rosetta/core_app/templates/create_task.html
@@ -185,9 +185,9 @@
<td><b>Computing options</b></td>
<td>
<table>
- <tr><td>Partition</td><td><input type="text" name="computing_partition" value="{{ data.task_computing.conf_params.default_partition }}" placeholder="" size="20" /></td></tr>
- <tr><td>Cpus</td><td><input type="text" name="computing_cpus" value="{{ data.task_computing.conf_params.default_cpus }}" placeholder="" size="5" /></td></tr>
- <tr><td>Memory</td><td><input type="text" name="computing_memory" value="{{ data.task_computing.conf_params.default_memory }}" placeholder="" size="5" /></td></tr>
+ <tr><td>Partition</td><td><input type="text" name="computing_partition" value="{{ data.task_computing.conf.default_partition }}" placeholder="" size="20" /></td></tr>
+ <tr><td>Cpus</td><td><input type="text" name="computing_cpus" value="{{ data.task_computing.conf.default_cpus }}" placeholder="" size="5" /></td></tr>
+ <tr><td>Memory</td><td><input type="text" name="computing_memory" value="{{ data.task_computing.conf.default_memory }}" placeholder="" size="5" /></td></tr>
</table>
</td>
</tr>
diff --git a/services/webapp/code/rosetta/core_app/utils.py b/services/webapp/code/rosetta/core_app/utils.py
index 977cf9534dd6511f1eea55ebd8a7267d01b38142..71931ed9104414a299e09203ad40f49392983082 100644
--- a/services/webapp/code/rosetta/core_app/utils.py
+++ b/services/webapp/code/rosetta/core_app/utils.py
@@ -564,12 +564,12 @@ def setup_tunnel(task):
if task.computing.type == 'remotehop':
# Get computing params
- first_host = task.computing.get_conf_param('first_host')
- first_user = task.computing.get_conf_param('first_user')
- #second_host = task.computing.get_conf_param('second_host')
- #second_user = task.computing.get_conf_param('second_user')
- #setup_command = task.computing.get_conf_param('setup_command')
- #base_port = task.computing.get_conf_param('base_port')
+ first_host = task.computing.conf.get('first_host')
+ first_user = task.computing.conf.get('first_user')
+ #second_host = task.computing.conf.get('second_host')
+ #second_user = task.computing.conf.get('second_user')
+ #setup_command = task.computing.conf.get('setup_command')
+ #base_port = task.computing.conf.get('base_port')
tunnel_command= 'ssh -4 -i {} -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} {}@{} & '.format(user_keys.private_key_file, task.tunnel_port, task.ip, task.port, first_user, first_host)
diff --git a/services/webapp/code/rosetta/core_app/views.py b/services/webapp/code/rosetta/core_app/views.py
index 3de2dd6e1a811aaa4e7aafb95041f9f87647b1fa..cedc871b02c9b89acad99384d2f64b3f7fe5158b 100644
--- a/services/webapp/code/rosetta/core_app/views.py
+++ b/services/webapp/code/rosetta/core_app/views.py
@@ -314,7 +314,7 @@ def tasks(request):
data['task'] = task
# Attach user config to computing
- task.computing.attach_user_conf_data(task.user)
+ task.computing.attach_user_conf(task.user)
# Task actions
if action=='delete':
@@ -448,7 +448,8 @@ def create_task(request):
try:
task_computing = Computing.objects.get(uuid=task_computing_uuid, user=request.user)
except Computing.DoesNotExist:
- raise Exception('Consistency error, computing with uuid "{}" does not exists or user "{}" does not have access rights'.format(task_computing_uuid, request.user.email))
+ raise Exception('Consistency error, computing with uuid "{}" does not exists or user "{}" does not have access rights'.format(task_computing_uuid, request.user.email))
+ task_computing.attach_user_conf(request.user)
data['task_computing'] = task_computing
# Handle step one/two
@@ -507,7 +508,7 @@ def create_task(request):
task.computing_options = computing_options
# Attach user config to computing
- task.computing.attach_user_conf_data(task.user)
+ task.computing.attach_user_conf(task.user)
# Set port if not dynamic ports
if not task.container.supports_dynamic_ports:
@@ -569,8 +570,8 @@ def task_log(request):
data['refresh'] = refresh
# Attach user conf in any
- task.computing.attach_user_conf_data(request.user)
-
+ task.computing.attach_user_conf(request.user)
+
# Get the log
try:
@@ -783,7 +784,7 @@ def computings(request):
data['computing'] = Computing.objects.get(uuid=computing_uuid, user=None)
# Attach user conf in any
- data['computing'].attach_user_conf_data(request.user)
+ data['computing'].attach_user_conf(request.user)
else:
@@ -791,7 +792,7 @@ def computings(request):
# Attach user conf in any
for computing in data['computings']:
- computing.attach_user_conf_data(request.user)
+ computing.attach_user_conf(request.user)
return render(request, 'computings.html', {'data': data})