diff --git a/services/webapp/code/rosetta/core_app/admin.py b/services/webapp/code/rosetta/core_app/admin.py
index 0edd07f57dd33f616b95bea77f6c3fe4a632ffbf..1e09ed48918f33f19babb36ecb10ec470a4b1c8c 100644
--- a/services/webapp/code/rosetta/core_app/admin.py
+++ b/services/webapp/code/rosetta/core_app/admin.py
@@ -1,13 +1,13 @@
from django.contrib import admin
-from .models import Profile, LoginToken, Task, Container, Computing, ComputingSysConf, ComputingUserConf, Storage, KeyPair, Text
+from .models import Profile, LoginToken, Task, Container, Computing, ComputingConf, ComputingUserConf, Storage, KeyPair, Text
admin.site.register(Profile)
admin.site.register(LoginToken)
admin.site.register(Task)
admin.site.register(Container)
admin.site.register(Computing)
-admin.site.register(ComputingSysConf)
+admin.site.register(ComputingConf)
admin.site.register(ComputingUserConf)
admin.site.register(Storage)
admin.site.register(KeyPair)
diff --git a/services/webapp/code/rosetta/core_app/api.py b/services/webapp/code/rosetta/core_app/api.py
index 4633c79327742196d75a8409acd429e6736a43c4..8f8f4e4bd74f8c3885b23245d76bda383029313a 100644
--- a/services/webapp/code/rosetta/core_app/api.py
+++ b/services/webapp/code/rosetta/core_app/api.py
@@ -11,7 +11,7 @@ from django.conf import settings
from rest_framework.response import Response
from rest_framework import status, serializers, viewsets
from rest_framework.views import APIView
-from .utils import format_exception, send_email, os_shell, now_t
+from .utils import format_exception, send_email, os_shell, now_t, get_ssh_access_mode_credentials
from .models import Profile, Task, TaskStatuses, Computing, Storage, KeyPair
from .exceptions import ConsistencyException
import json
@@ -368,51 +368,28 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
source = source.replace('\ ', '\\\\\\ ')
else:
dest = dest.replace('\ ', '\\\\\\ ')
-
- # Get user key
- user_keys = KeyPair.objects.get(user=user, default=True)
-
- # Get computing host
- computing.attach_user_conf(user)
- computing_host = computing.conf.get('host')
- computing_user = computing.conf.get('user')
-
- if not computing_host:
- raise Exception('No computing host?!')
-
- if not computing_user:
- raise Exception('No computing user?!')
+
+ # Get credentials
+ computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(computing, user)
# Command
if mode=='get':
- command = 'scp -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{}:{} {}'.format(user_keys.private_key_file, computing_user, computing_host, source, dest)
+ command = 'scp -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{}:{} {}'.format(computing_keys.private_key_file, computing_user, computing_host, source, dest)
elif mode == 'put':
- command = 'scp -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {} {}@{}:{}'.format(user_keys.private_key_file, source, computing_user, computing_host, dest)
+ command = 'scp -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {} {}@{}:{}'.format(computing_keys.private_key_file, source, computing_user, computing_host, dest)
else:
raise ValueError('Unknown mode "{}"'.format(mode))
return command
-
def ssh_command(self, command, user, computing):
- # Get user key
- user_keys = KeyPair.objects.get(user=user, default=True)
-
- # Get computing host
- computing.attach_user_conf(user)
- computing_host = computing.conf.get('host')
- computing_user = computing.conf.get('user')
-
- if not computing_host:
- raise Exception('No computing host?!')
-
- if not computing_user:
- raise Exception('No computing user?!')
+ # Get credentials
+ computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(computing, user)
# Command
- command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} "{}"'.format(user_keys.private_key_file, computing_user, computing_host, command)
+ command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} "{}"'.format(computing_keys.private_key_file, computing_user, computing_host, command)
return command
@@ -441,7 +418,11 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
if storage.access_through_computing:
computing = storage.computing
computing.attach_user_conf(user)
- base_path_expanded = base_path_expanded.replace('$SSH_USER', computing.conf.get('user'))
+ if computing.auth_mode == 'user_keys':
+ base_path_expanded = base_path_expanded.replace('$SSH_USER', computing.user_conf.get('user'))
+ else:
+ base_path_expanded = base_path_expanded.replace('$SSH_USER', computing.conf.get('user'))
+
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in base_path_expanded:
diff --git a/services/webapp/code/rosetta/core_app/computing_managers.py b/services/webapp/code/rosetta/core_app/computing_managers.py
index a5a211659f75e60154d88029cde303eb50468767..2ce933f1b19eea309d9446ac569c2db64b78835a 100644
--- a/services/webapp/code/rosetta/core_app/computing_managers.py
+++ b/services/webapp/code/rosetta/core_app/computing_managers.py
@@ -1,5 +1,5 @@
from .models import TaskStatuses, KeyPair, Task, Storage
-from .utils import os_shell
+from .utils import os_shell, get_ssh_access_mode_credentials
from .exceptions import ErrorMessage, ConsistencyException
from django.conf import settings
@@ -180,15 +180,8 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
def _start_task(self, task, **kwargs):
logger.debug('Starting a remote task "{}"'.format(self.computing))
- # Get computing user and host
- computing_user = self.computing.conf.get('user')
- computing_host = self.computing.conf.get('host')
-
- # Get user keys
- if self.computing.requires_user_keys:
- user_keys = KeyPair.objects.get(user=task.user, default=True)
- else:
- raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
+ # Get credentials
+ computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)
# Get webapp conn string
from.utils import get_webapp_conn_string
@@ -238,7 +231,7 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
else:
binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)
- run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, computing_user, computing_host)
+ run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)
run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp && mkdir -p /tmp/{}_data/home && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'wget {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export BASE_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
@@ -271,18 +264,11 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
def _stop_task(self, task, **kwargs):
- # Get user keys
- if self.computing.requires_user_keys:
- user_keys = KeyPair.objects.get(user=task.user, default=True)
- else:
- raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
-
- # Get computing host
- host = self.computing.conf.get('host')
- user = self.computing.conf.get('user')
+ # Get credentials
+ computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)
# Stop the task remotely
- stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.id)
+ stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(computing_keys.private_key_file, computing_user, computing_host, task.id)
out = os_shell(stop_command, capture=True)
if out.exit_code != 0:
if not 'No such process' in out.stderr:
@@ -294,19 +280,12 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
def _get_task_log(self, task, **kwargs):
-
- # Get user keys
- if self.computing.requires_user_keys:
- user_keys = KeyPair.objects.get(user=task.user, default=True)
- else:
- raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
-
- # Get computing host
- host = self.computing.conf.get('host')
- user = self.computing.conf.get('user')
+ # Get credentials
+ computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)
+
# View log remotely
- view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}_data/task.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
+ view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}_data/task.log"\''.format(computing_keys.private_key_file, computing_user, computing_host, task.uuid)
out = os_shell(view_log_command, capture=True)
if out.exit_code != 0:
@@ -321,15 +300,8 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
def _start_task(self, task, **kwargs):
logger.debug('Starting a remote task "{}"'.format(self.computing))
- # Get computing host
- computing_host = self.computing.conf.get('host')
- computing_user = self.computing.conf.get('user')
-
- # Get user keys
- if self.computing.requires_user_keys:
- user_keys = KeyPair.objects.get(user=task.user, default=True)
- else:
- raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
+ # Get credentials
+ computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)
# Get webapp conn string
from.utils import get_webapp_conn_string
@@ -399,7 +371,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
else:
binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)
- run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, computing_user, computing_host)
+ run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)
run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT && {} '.format(authstring)
run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
@@ -442,18 +414,11 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
def _stop_task(self, task, **kwargs):
- # Get user keys
- if self.computing.requires_user_keys:
- user_keys = KeyPair.objects.get(user=task.user, default=True)
- else:
- raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
-
- # Get computing host
- host = self.computing.conf.get('host')
- user = self.computing.conf.get('user')
+ # Get credentials
+ computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)
# Stop the task remotely
- stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.id)
+ stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(computing_keys.private_key_file, computing_user, computing_host, task.id)
out = os_shell(stop_command, capture=True)
if out.exit_code != 0:
raise Exception(out.stderr)
@@ -465,18 +430,11 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
def _get_task_log(self, task, **kwargs):
- # Get user keys
- if self.computing.requires_user_keys:
- user_keys = KeyPair.objects.get(user=task.user, default=True)
- else:
- raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
-
- # Get computing host
- host = self.computing.conf.get('host')
- user = self.computing.conf.get('user')
+ # Get credentials
+ computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)
# View log remotely
- view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
+ view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(computing_keys.private_key_file, computing_user, computing_host, task.uuid)
out = os_shell(view_log_command, capture=True)
if out.exit_code != 0:
@@ -485,183 +443,3 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
return out.stdout
-# TODO: rename the following as "ssh+ssh" access mode? Ore somethign similar?
-# class RemotehopComputingManager(ComputingManager):
-#
-# def _start_task(self, task, **kwargs):
-# logger.debug('Starting a remote task "{}"'.format(self.computing))
-#
-# # Get computing params
-# first_host = self.computing.conf.get('first_host')
-# first_user = self.computing.conf.get('first_user')
-# second_host = self.computing.conf.get('second_host')
-# second_user = self.computing.conf.get('second_user')
-# setup_command = self.computing.conf.get('setup_command')
-#
-# # TODO: De hard-code
-# use_agent = False
-#
-# # Get user keys
-# if self.computing.requires_user_keys:
-# user_keys = KeyPair.objects.get(user=task.user, default=True)
-# else:
-# raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
-#
-# # Get webapp conn string
-# from.utils import get_webapp_conn_string
-# webapp_conn_string = get_webapp_conn_string()
-#
-# # Run the container on the host (non blocking)
-# if task.container.type == 'singularity':
-#
-# task.tid = task.uuid
-# task.save()
-#
-# # Set pass if any
-# if task.auth_pass:
-# authstring = ' export SINGULARITYENV_AUTH_PASS={} && '.format(task.auth_pass)
-# else:
-# authstring = ''
-#
-# # Set binds, only from sys config if the resource is not owned by the user
-# if self.computing.user != task.user:
-# binds = self.computing.sys_conf.get('binds')
-# else:
-# binds = self.computing.conf.get('binds')
-# if not binds:
-# binds = ''
-# else:
-# binds = '-B {}'.format(binds)
-#
-# # Manage task extra binds
-# if task.extra_binds:
-# if not binds:
-# binds = '-B {}'.format(task.extra_binds)
-# else:
-# binds += ',{}'.format(task.extra_binds)
-#
-# run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
-# run_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} /bin/bash -c \''.format(second_user, second_host)
-#
-# if use_agent:
-# run_command += '\'wget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> /dev/null && export BASE_PORT=\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
-# if setup_command:
-# run_command += setup_command + ' && '
-# run_command += '\'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
-# run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
-# run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)
-# else:
-# run_command += ' : && ' # Trick to prevent some issues in exporting variables
-# if setup_command:
-# run_command += setup_command + ' && '
-# run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT={} && {} '.format(task.port, authstring)
-# run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
-# run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)
-#
-# # Set registry
-# if task.container.registry == 'docker_local':
-# raise Exception('This computing resource does not support local Docker registries yet')
-# # Get local Docker registry conn string
-# from.utils import get_local_docker_registry_conn_string
-# local_docker_registry_conn_string = get_local_docker_registry_conn_string()
-# registry = 'docker://{}/'.format(local_docker_registry_conn_string)
-# elif task.container.registry == 'docker_hub':
-# registry = 'docker://'
-# else:
-# raise NotImplementedError('Registry {} not supported'.format(task.container.registry))
-#
-# run_command+='{}{} &>> \$HOME/{}.log & echo \$!\'"'.format(registry, task.container.image, task.uuid)
-#
-# else:
-# raise NotImplementedError('Container {} not supported'.format(task.container.type))
-#
-# out = os_shell(run_command, capture=True)
-# if out.exit_code != 0:
-# raise Exception(out.stderr)
-#
-# # Log
-# logger.debug('Shell exec output: "{}"'.format(out))
-#
-#
-# # Load back the task to avoid concurrency problems in the agent call
-# task_uuid = task.uuid
-# task = Task.objects.get(uuid=task_uuid)
-#
-# # Save pid echoed by the command above
-# task_pid = out.stdout
-#
-# # Set fields
-# task.status = TaskStatuses.running
-# task.pid = task_pid
-# task.ip = second_host
-#
-# # Save
-# task.save()
-#
-#
-# def _stop_task(self, task, **kwargs):
-#
-# # Get user keys
-# if self.computing.requires_user_keys:
-# user_keys = KeyPair.objects.get(user=task.user, default=True)
-# else:
-# raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
-#
-# # Get computing params
-# first_host = self.computing.conf.get('first_host')
-# first_user = self.computing.conf.get('first_user')
-# second_host = self.computing.conf.get('second_host')
-# second_user = self.computing.conf.get('second_user')
-#
-# # Stop the task remotely
-# stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
-# stop_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
-# stop_command += 'kill -9 {}"'.format(task.pid)
-#
-# out = os_shell(stop_command, capture=True)
-# if out.exit_code != 0:
-# if not 'No such process' in out.stderr:
-# raise Exception(out.stderr)
-#
-# # Set task as stopped
-# task.status = TaskStatuses.stopped
-# task.save()
-#
-#
-# def _get_task_log(self, task, **kwargs):
-#
-# # Get user keys
-# if self.computing.requires_user_keys:
-# user_keys = KeyPair.objects.get(user=task.user, default=True)
-# else:
-# raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
-#
-# # Get computing params
-# first_host = self.computing.conf.get('first_host')
-# first_user = self.computing.conf.get('first_user')
-# second_host = self.computing.conf.get('second_host')
-# second_user = self.computing.conf.get('second_user')
-#
-# # View log remotely
-# view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
-# view_log_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
-# view_log_command += 'cat \\\\\\$HOME/{}.log"'.format(task.uuid)
-#
-# out = os_shell(view_log_command, capture=True)
-# if out.exit_code != 0:
-# raise Exception(out.stderr)
-# else:
-# return out.stdout
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py b/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py
index 46f32b81f4f07e43a28c9b85a1454a1301073fd8..e54bc154e6d35ba458e430e357f9301d65363146 100644
--- a/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py
+++ b/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py
@@ -1,6 +1,6 @@
from django.core.management.base import BaseCommand
from django.contrib.auth.models import User
-from ...models import Profile, Container, Computing, ComputingSysConf, ComputingUserConf, Storage, KeyPair, Text
+from ...models import Profile, Container, Computing, ComputingConf, ComputingUserConf, Storage, KeyPair, Text
class Command(BaseCommand):
help = 'Adds the admin superuser with \'a\' password.'
@@ -24,7 +24,7 @@ class Command(BaseCommand):
#=====================
try:
testuser = User.objects.get(username='testuser')
- print('Not creating test user as it already exist')
+ print('Not creating test user as it already exists')
except User.DoesNotExist:
print('Creating test user with default password')
@@ -38,11 +38,28 @@ class Command(BaseCommand):
Profile.objects.create(user=testuser, authtoken='129aac94-284a-4476-953c-ffa4349b4a50')
# Create default keys
- print('Creating testuser defualt keys')
+ print('Creating testuser default keys')
KeyPair.objects.create(user = testuser,
- default = True,
- private_key_file = '/rosetta/.ssh/id_rsa',
- public_key_file = '/rosetta/.ssh/id_rsa.pub')
+ default = True,
+ private_key_file = '/rosetta/.ssh/id_rsa',
+ public_key_file = '/rosetta/.ssh/id_rsa.pub')
+
+
+ #=====================
+ # Platform keys
+ #=====================
+ # TODO: create a different pair
+ try:
+ testuser = KeyPair.objects.get(user=None, default=True)
+ print('Not creating default platform keys as they already exist')
+
+ except KeyPair.DoesNotExist:
+ print('Creating platform default keys')
+ KeyPair.objects.create(user = None,
+ default = True,
+ private_key_file = '/rosetta/.ssh/id_rsa',
+ public_key_file = '/rosetta/.ssh/id_rsa.pub')
+
#=====================
# Default home text
@@ -183,9 +200,6 @@ class Command(BaseCommand):
access_mode = 'internal',
auth_mode = 'internal',
wms = None,
- requires_sys_conf = False,
- requires_user_conf = False,
- requires_user_keys = False,
container_runtimes = 'docker')
@@ -197,14 +211,11 @@ class Command(BaseCommand):
access_mode = 'ssh+cli',
auth_mode = 'user_keys',
wms = None,
- requires_sys_conf = True,
- requires_user_conf = True,
- requires_user_keys = True,
container_runtimes = 'singularity')
- ComputingSysConf.objects.create(computing = demo_singlenode_computing,
- data = {'host': 'slurmclusterworker-one',
- 'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'})
+ ComputingConf.objects.create(computing = demo_singlenode_computing,
+ data = {'host': 'slurmclusterworker-one',
+ 'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'})
ComputingUserConf.objects.create(user = testuser,
computing = demo_singlenode_computing,
@@ -219,14 +230,11 @@ class Command(BaseCommand):
access_mode = 'ssh+cli',
auth_mode = 'user_keys',
wms = 'slurm',
- requires_sys_conf = True,
- requires_user_conf = True,
- requires_user_keys = True,
container_runtimes = 'singularity')
- ComputingSysConf.objects.create(computing = demo_slurm_computing,
- data = {'host': 'slurmclustermaster-main', 'default_partition': 'partition1',
- 'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'})
+ ComputingConf.objects.create(computing = demo_slurm_computing,
+ data = {'host': 'slurmclustermaster-main', 'default_partition': 'partition1',
+ 'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'})
ComputingUserConf.objects.create(user = testuser,
computing = demo_slurm_computing,
diff --git a/services/webapp/code/rosetta/core_app/migrations/0012_remove_task_extra_binds.py b/services/webapp/code/rosetta/core_app/migrations/0012_remove_task_extra_binds.py
new file mode 100644
index 0000000000000000000000000000000000000000..d76bc5fe573621ce4884adeb51608b11971b0628
--- /dev/null
+++ b/services/webapp/code/rosetta/core_app/migrations/0012_remove_task_extra_binds.py
@@ -0,0 +1,17 @@
+# Generated by Django 2.2.1 on 2021-11-08 14:52
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core_app', '0011_storage'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='task',
+ name='extra_binds',
+ ),
+ ]
diff --git a/services/webapp/code/rosetta/core_app/migrations/0013_auto_20211108_1529.py b/services/webapp/code/rosetta/core_app/migrations/0013_auto_20211108_1529.py
new file mode 100644
index 0000000000000000000000000000000000000000..10d1fbb181ee8891a40f0e7faea4a6955196740a
--- /dev/null
+++ b/services/webapp/code/rosetta/core_app/migrations/0013_auto_20211108_1529.py
@@ -0,0 +1,43 @@
+# Generated by Django 2.2.1 on 2021-11-08 15:29
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core_app', '0012_remove_task_extra_binds'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='ComputingConf',
+ fields=[
+ ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+ ('data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True)),
+ ],
+ ),
+ migrations.RemoveField(
+ model_name='computing',
+ name='requires_sys_conf',
+ ),
+ migrations.RemoveField(
+ model_name='computing',
+ name='requires_user_conf',
+ ),
+ migrations.RemoveField(
+ model_name='computing',
+ name='requires_user_keys',
+ ),
+ migrations.DeleteModel(
+ name='ComputingSysConf',
+ ),
+ migrations.AddField(
+ model_name='computingconf',
+ name='computing',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='related_computing_conf', to='core_app.Computing'),
+ ),
+ ]
diff --git a/services/webapp/code/rosetta/core_app/migrations/0014_auto_20211108_1548.py b/services/webapp/code/rosetta/core_app/migrations/0014_auto_20211108_1548.py
new file mode 100644
index 0000000000000000000000000000000000000000..98aa509849d51bc5964c7ad1a6c4e10d4f437709
--- /dev/null
+++ b/services/webapp/code/rosetta/core_app/migrations/0014_auto_20211108_1548.py
@@ -0,0 +1,20 @@
+# Generated by Django 2.2.1 on 2021-11-08 15:48
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('core_app', '0013_auto_20211108_1529'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='keypair',
+ name='user',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL),
+ ),
+ ]
diff --git a/services/webapp/code/rosetta/core_app/models.py b/services/webapp/code/rosetta/core_app/models.py
index a5c5caab6b3027557ab79c18481c5730acd62c8b..eee49ad033c79d2efd24224dc66006df897460db 100644
--- a/services/webapp/code/rosetta/core_app/models.py
+++ b/services/webapp/code/rosetta/core_app/models.py
@@ -135,13 +135,9 @@ class Computing(models.Model):
name = models.CharField('Name', max_length=255, blank=False, null=False)
description = models.TextField('Description', blank=True, null=True)
- # Standalone / sluster
+ # Tye (standalone / cluster)
type = models.CharField('Type', max_length=255, blank=False, null=False)
- requires_sys_conf = models.BooleanField(default=False)
- requires_user_conf = models.BooleanField(default=False)
- requires_user_keys = models.BooleanField(default=False)
-
# Interfce and interaction definition
access_mode = models.CharField('Access (control) mode', max_length=36, blank=False, null=False)
auth_mode = models.CharField('Auth mode', max_length=36, blank=False, null=False)
@@ -198,7 +194,7 @@ class Computing(models.Model):
#=======================
- # Sys & user conf
+ # Conf & user conf
#=======================
def attach_user_conf(self, user):
@@ -210,9 +206,17 @@ class Computing(models.Model):
self._user_conf_data = None
@property
- def sys_conf(self):
- return self.related_sys_conf.get().data
+ def conf(self):
+ try:
+ return self.related_computing_conf.get().data
+ except:
+ return {}
+ #TODO: add a setter and start removing the ComputingConf model
+ @property
+ def conf_as_json(self):
+ return json.dumps(self.conf)
+
@property
def user_conf(self):
try:
@@ -220,39 +224,17 @@ class Computing(models.Model):
except AttributeError:
raise ConsistencyException('User conf has to been attached, cannot proceed.')
- @property
- def sys_conf_as_json(self):
- return json.dumps(self.sys_conf)
-
@property
def user_conf_as_json(self):
return json.dumps(self.user_conf)
- @property
- def conf(self):
-
- if not self.requires_user_conf:
- conf_tmp = self.sys_conf
- else:
- try:
- # Copy the conf or the original user conf will be affected by the overwrite below
- conf_tmp = {key:value for key, value in self._user_conf_data.items()}
- except AttributeError:
- raise ConsistencyException('User conf has not been attached, cannot proceed.')
-
- # Now add (overwrite) with the sys conf
- sys_conf = self.sys_conf
- for key in sys_conf:
- conf_tmp[key] = sys_conf[key]
-
- return conf_tmp
-class ComputingSysConf(models.Model):
+class ComputingConf(models.Model):
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
- computing = models.ForeignKey(Computing, related_name='related_sys_conf', on_delete=models.CASCADE)
+ computing = models.ForeignKey(Computing, related_name='related_computing_conf', on_delete=models.CASCADE)
data = JSONField(blank=True, null=True)
@@ -316,13 +298,11 @@ class Task(models.Model):
computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
container = models.ForeignKey('Container', on_delete=models.CASCADE, related_name='+')
- # Extra
- extra_binds = models.CharField('Extra binds', max_length=4096, blank=True, null=True)
+ # Computing options
+ # TODO: add the option for selecting the runtime as advanced option when creating the task?
computing_options = JSONField('Computing options', blank=True, null=True) # i.e. CPUs, RAM, cluster partition etc. TODO: why here?
- # TODO: add the option for selecting the runtime as advanced option when creating the task
- #container_runtime
-
+
class Meta:
ordering = ['-created']
@@ -440,7 +420,7 @@ class Storage(models.Model):
class KeyPair(models.Model):
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
- user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=False)
+ user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, blank=True, null=True)
private_key_file = models.CharField('Private key file', max_length=4096, blank=False, null=False)
public_key_file = models.CharField('Public key file', max_length=4096, blank=False, null=False)
diff --git a/services/webapp/code/rosetta/core_app/templates/components/computing.html b/services/webapp/code/rosetta/core_app/templates/components/computing.html
index f6f647a949d692278ac8954f402bfc9606e518cc..d906e9592bb65b367f6e32dc87a8dcf84f304727 100644
--- a/services/webapp/code/rosetta/core_app/templates/components/computing.html
+++ b/services/webapp/code/rosetta/core_app/templates/components/computing.html
@@ -49,15 +49,6 @@
<td>{% if data.computing.user %}{{ data.computing.user }}{% else %}platform{% endif %}</td>
</tr>
- <tr>
- <td><b>Requires</b></td>
- <td>
- Sys conf: {{ data.computing.requires_sys_conf }} <br/>
- User conf: {{ data.computing.requires_user_conf }} <br/>
- User keys: {{ data.computing.requires_user_keys }}
- </td>
- </tr>
-
<tr>
<td><b>Container runtimes</b></td>
<td>{{ data.computing.container_runtimes }}</td>
@@ -65,7 +56,7 @@
<tr>
<td><b>Sys Conf</b></td>
- <td>{{ data.computing.sys_conf_as_json }} {% if request.user.is_superuser %} [<a href="/edit_computing_conf?type=sys&computing_uuid={{ data.computing.uuid}}">Edit</a>] {% endif %}</td>
+ <td>{{ data.computing.conf_as_json }} {% if request.user.is_superuser %} [<a href="/edit_computing_conf?type=sys&computing_uuid={{ data.computing.uuid}}">Edit</a>] {% endif %}</td>
</tr>
<tr>
diff --git a/services/webapp/code/rosetta/core_app/templates/components/task.html b/services/webapp/code/rosetta/core_app/templates/components/task.html
index 1dcb6d83269541c45f5b2f1af73c506d683272b4..8508df3ffbf58f5a6154d619a92c093f1dfb4d04 100644
--- a/services/webapp/code/rosetta/core_app/templates/components/task.html
+++ b/services/webapp/code/rosetta/core_app/templates/components/task.html
@@ -120,11 +120,6 @@
<td>{{ task.created }}</td>
</tr>
- <tr>
- <td><b>Extra binds</b></td>
- <td>{{ task.extra_binds }}</td>
- </tr>
-
<tr>
<td><b>Interface IP</b></td>
<td>{{ task.interface_ip }}</td>
diff --git a/services/webapp/code/rosetta/core_app/templates/create_task.html b/services/webapp/code/rosetta/core_app/templates/create_task.html
index 9ff63a05c03cb401a935f8af5cf5b03a6c85097f..8eedd2ca85aff248dbfa210bd3264af1b13044b3 100644
--- a/services/webapp/code/rosetta/core_app/templates/create_task.html
+++ b/services/webapp/code/rosetta/core_app/templates/create_task.html
@@ -108,16 +108,6 @@
</td>
</tr>
- <!-- {% if data.task_computing.supports_custom_binds %}
- <tr>
- <td valign="top" style="width:180px"><b>Extra binds</b></td>
- <td>
- <input type="text" name="extra_binds" value="" placeholder="" size="40" /><br>
-<p style="line-height: 0.95"><font size=-1>For this computing resource, you can set extra FS binds on top of the ones already define by the administrators. Format is <i>host_directory:container_directory</i>, comma separated.</font></p>
- </td>
- </tr>
- {% endif %}
- -->
{% if request.user.profile.is_power_user %}
<tr>
diff --git a/services/webapp/code/rosetta/core_app/templates/files.html b/services/webapp/code/rosetta/core_app/templates/files.html
index 6790061998a3d79ca3c4dc34a5a1fbc061e8a7b5..fee103b589546dd3866b3ba9b9a63138c4ede4dc 100644
--- a/services/webapp/code/rosetta/core_app/templates/files.html
+++ b/services/webapp/code/rosetta/core_app/templates/files.html
@@ -10,7 +10,7 @@
<div class="span8 offset2" style="height:100%;">
- <h1>Files</h1>
+ <h1>Storages</h1>
<hr/>
diff --git a/services/webapp/code/rosetta/core_app/templates/navigation.html b/services/webapp/code/rosetta/core_app/templates/navigation.html
index f55251056ff62ee8d2a2119ebe919bd5ab4831a1..4d80f42adebd3f570419a6929bdc1a7bbabaaf83 100644
--- a/services/webapp/code/rosetta/core_app/templates/navigation.html
+++ b/services/webapp/code/rosetta/core_app/templates/navigation.html
@@ -23,12 +23,15 @@
<li>
<a href="/computings" onclick = $("#menu-close").click(); >Computing</a>
</li>
+
<li>
- <a href="/tasks" onclick = $("#menu-close").click(); >Tasks</a>
+ <a href="/files" onclick = $("#menu-close").click(); >Storage</a>
</li>
+
<li>
- <a href="/files" onclick = $("#menu-close").click(); >Files</a>
+ <a href="/tasks" onclick = $("#menu-close").click(); >Tasks</a>
</li>
+
<li>
<a href="/account" onclick = $("#menu-close").click(); >Account</a>
</li>
diff --git a/services/webapp/code/rosetta/core_app/tests/test_models.py b/services/webapp/code/rosetta/core_app/tests/test_models.py
index 0654b17555d8a3c2bc036b913dfefec6fdc09941..508ff70901119a82f4766c29e1591b2f32b79801 100644
--- a/services/webapp/code/rosetta/core_app/tests/test_models.py
+++ b/services/webapp/code/rosetta/core_app/tests/test_models.py
@@ -3,7 +3,7 @@ import json
from django.contrib.auth.models import User
from .common import BaseAPITestCase
-from ..models import Profile, Computing, ComputingSysConf
+from ..models import Profile, Computing, ComputingConf
class Modeltest(BaseAPITestCase):
@@ -22,8 +22,8 @@ class Modeltest(BaseAPITestCase):
computing = Computing.objects.create(name='MyComp', type='remote')
- computingSysConf = ComputingSysConf.objects.create(computing=computing, data={'myvar':42})
+ computingSysConf = ComputingConf.objects.create(computing=computing, data={'myvar':42})
- self.assertEqual(ComputingSysConf.objects.all()[0].data, {'myvar':42})
+ self.assertEqual(ComputingConf.objects.all()[0].data, {'myvar':42})
diff --git a/services/webapp/code/rosetta/core_app/utils.py b/services/webapp/code/rosetta/core_app/utils.py
index 2e7801640e3e4628661fd337e59bf444753932b5..057675338720817e7279e2f69b3fd37779a26bf6 100644
--- a/services/webapp/code/rosetta/core_app/utils.py
+++ b/services/webapp/code/rosetta/core_app/utils.py
@@ -696,7 +696,27 @@ Listen '''+str(task.tcp_tunnel_port)+'''
-
+def get_ssh_access_mode_credentials(computing, user):
+ from .models import KeyPair
+ # Get computing host
+ computing_host = computing.conf.get('host')
+ if not computing_host:
+ raise Exception('No computing host?!')
+
+ # Get computing user and keys
+ if computing.auth_mode == 'user_keys':
+ computing.attach_user_conf(user)
+ computing_user = computing.user_conf.get('user')
+ # Get user key
+ computing_keys = KeyPair.objects.get(user=user, default=True)
+ elif computing.auth_mode == 'platform_keys':
+ computing_user = computing.conf.get('user')
+ computing_keys = KeyPair.objects.get(user=None, default=True)
+ else:
+ raise NotImplementedError('Auth modes other than user_keys and platform_keys not supported.')
+ if not computing_user:
+ raise Exception('No computing user?!')
+ return (computing_user, computing_host, computing_keys)
diff --git a/services/webapp/code/rosetta/core_app/views.py b/services/webapp/code/rosetta/core_app/views.py
index 685b60cd0e22e4b47c9fcf66bc4fe6f9ec5dfba4..ebb3eded12357a1c994b645d4ff506ac47bac85e 100644
--- a/services/webapp/code/rosetta/core_app/views.py
+++ b/services/webapp/code/rosetta/core_app/views.py
@@ -9,7 +9,7 @@ from django.http import HttpResponse, HttpResponseRedirect
from django.contrib.auth.models import User
from django.shortcuts import redirect
from django.db.models import Q
-from .models import Profile, LoginToken, Task, TaskStatuses, Container, Computing, KeyPair, ComputingSysConf, ComputingUserConf, Text
+from .models import Profile, LoginToken, Task, TaskStatuses, Container, Computing, KeyPair, ComputingConf, ComputingUserConf, Text
from .utils import send_email, format_exception, timezonize, os_shell, booleanize, debug_param, get_task_tunnel_host, get_task_proxy_host, random_username, setup_tunnel_and_proxy, finalize_user_creation
from .decorators import public_view, private_view
from .exceptions import ErrorMessage
@@ -551,7 +551,6 @@ def create_task(request):
computing_cpus = request.POST.get('computing_cpus', None)
computing_memory = request.POST.get('computing_memory', None)
computing_partition = request.POST.get('computing_partition', None)
- extra_binds = request.POST.get('extra_binds', None)
computing_options = {}
if computing_cpus:
@@ -573,9 +572,6 @@ def create_task(request):
# Attach user config to computing
task.computing.attach_user_conf(task.user)
- # Set extra binds if any:
- task.extra_binds = extra_binds
-
# Save the task before starting it, or the computing manager will not be able to work properly
task.save()
@@ -924,11 +920,11 @@ def edit_computing_conf(request):
try:
computing = Computing.objects.get(uuid=computing_uuid)
data['computing'] = computing
- except ComputingSysConf.DoesNotExist:
+ except ComputingConf.DoesNotExist:
raise Exception('Unknown computing "{}"'.format(computing_uuid))
# Get computing conf
- computingSysConf, _ = ComputingSysConf.objects.get_or_create(computing=computing)
+ computingSysConf, _ = ComputingConf.objects.get_or_create(computing=computing)
# Edit conf?
if new_conf: