diff --git a/services/webapp/code/rosetta/core_app/api.py b/services/webapp/code/rosetta/core_app/api.py index ab6e55d964bd3e95ad1f5fb5adb091697e90cc98..d2dd69172e38a46cfc20dc6421cfa99b3f8578e6 100644 --- a/services/webapp/code/rosetta/core_app/api.py +++ b/services/webapp/code/rosetta/core_app/api.py @@ -372,13 +372,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): user_keys = KeyPair.objects.get(user=user, default=True) # Get computing host - computing_host = computing.get_conf_param('host') + computing_host = computing.conf.get('host') # Trick for handling Slurm.. TODO: fix me! if not computing_host: - computing_host = computing.get_conf_param('master') + computing_host = computing.conf.get('master') - computing_user = computing.get_conf_param('user') + computing_user = computing.conf.get('user') if not computing_host: raise Exception('No computing host?!') @@ -404,13 +404,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): user_keys = KeyPair.objects.get(user=user, default=True) # Get computing host - computing_host = computing.get_conf_param('host') + computing_host = computing.conf.get('host') # Trick for handling Slurm.. TODO: fix me! if not computing_host: - computing_host = computing.get_conf_param('master') + computing_host = computing.conf.get('master') - computing_user = computing.get_conf_param('user') + computing_user = computing.conf.get('user') if not computing_host: raise Exception('No computing host?!') @@ -457,7 +457,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): computing = computing[0] # Attach user conf in any - computing.attach_user_conf_data(request.user) + computing.attach_user_conf(request.user) return computing @@ -712,7 +712,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): for computing in computings: # Attach user conf in any - computing.attach_user_conf_data(request.user) + computing.attach_user_conf(request.user) data['data'].append({ 'id': '/{}/'.format(computing.name), @@ -733,9 +733,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): # TODO: we can remove this and just always filter agains bind probably... if len(path.split('/')) == 3: if computing.user != request.user: - binds = computing.get_conf_param('binds', from_sys_only=True ) + binds = computing.sys_conf.get('binds') else: - binds = computing.get_conf_param('binds') + binds = computing.conf.get('binds') if binds: binds = binds.split(',') diff --git a/services/webapp/code/rosetta/core_app/computing_managers.py b/services/webapp/code/rosetta/core_app/computing_managers.py index b3cb204f68aa18681caa95018a00feae402aa7d3..e485f332e7f379c3b4e290d294dc4f111ca37878 100644 --- a/services/webapp/code/rosetta/core_app/computing_managers.py +++ b/services/webapp/code/rosetta/core_app/computing_managers.py @@ -10,6 +10,9 @@ logger = logging.getLogger(__name__) class ComputingManager(object): + def __init__(self, computing): + self.computing = computing + def start_task(self, task, **kwargs): # Check for run task logic implementation @@ -161,14 +164,14 @@ class LocalComputingManager(ComputingManager): class RemoteComputingManager(ComputingManager): def _start_task(self, task, **kwargs): - logger.debug('Starting a remote task "{}"'.format(task.computing)) + logger.debug('Starting a remote task "{}"'.format(self.computing)) # Get computing host - host = task.computing.get_conf_param('host') - user = task.computing.get_conf_param('user') + host = self.computing.conf.get('host') + user = self.computing.conf.get('user') # Get user keys - if task.computing.requires_user_keys: + if self.computing.requires_user_keys: user_keys = KeyPair.objects.get(user=task.user, default=True) else: raise NotImplementedError('Remote tasks not requiring keys are not yet supported') @@ -190,10 +193,10 @@ class RemoteComputingManager(ComputingManager): authstring = '' # Set binds, only from sys config if the resource is not owned by the user - if task.computing.user != task.user: - binds = task.computing.get_conf_param('binds', from_sys_only=True ) + if self.computing.user != task.user: + binds = self.computing.sys_conf.get('binds') else: - binds = task.computing.get_conf_param('binds') + binds = self.computing.conf.get('binds') if not binds: binds = '' else: @@ -253,14 +256,14 @@ class RemoteComputingManager(ComputingManager): def _stop_task(self, task, **kwargs): # Get user keys - if task.computing.requires_user_keys: + if self.computing.requires_user_keys: user_keys = KeyPair.objects.get(user=task.user, default=True) else: raise NotImplementedError('Remote tasks not requiring keys are not yet supported') # Get computing host - host = task.computing.get_conf_param('host') - user = task.computing.get_conf_param('user') + host = self.computing.conf.get('host') + user = self.computing.conf.get('user') # Stop the task remotely stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.pid) @@ -277,14 +280,14 @@ class RemoteComputingManager(ComputingManager): def _get_task_log(self, task, **kwargs): # Get user keys - if task.computing.requires_user_keys: + if self.computing.requires_user_keys: user_keys = KeyPair.objects.get(user=task.user, default=True) else: raise NotImplementedError('Remote tasks not requiring keys are not yet supported') # Get computing host - host = task.computing.get_conf_param('host') - user = task.computing.get_conf_param('user') + host = self.computing.conf.get('host') + user = self.computing.conf.get('user') # View log remotely view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}_data/task.log"\''.format(user_keys.private_key_file, user, host, task.uuid) @@ -300,14 +303,14 @@ class RemoteComputingManager(ComputingManager): class SlurmComputingManager(ComputingManager): def _start_task(self, task, **kwargs): - logger.debug('Starting a remote task "{}"'.format(task.computing)) + logger.debug('Starting a remote task "{}"'.format(self.computing)) # Get computing host - host = task.computing.get_conf_param('master') - user = task.computing.get_conf_param('user') + host = self.computing.conf.get('master') + user = self.computing.conf.get('user') # Get user keys - if task.computing.requires_user_keys: + if self.computing.requires_user_keys: user_keys = KeyPair.objects.get(user=task.user, default=True) else: raise NotImplementedError('Remote tasks not requiring keys are not yet supported') @@ -349,10 +352,10 @@ class SlurmComputingManager(ComputingManager): authstring = '' # Set binds, only from sys config if the resource is not owned by the user - if task.computing.user != task.user: - binds = task.computing.get_conf_param('binds', from_sys_only=True ) + if self.computing.user != task.user: + binds = self.computing.sys_conf.get('binds') else: - binds = task.computing.get_conf_param('binds') + binds = self.computing.conf.get('binds') if not binds: binds = '' else: @@ -422,14 +425,14 @@ class SlurmComputingManager(ComputingManager): def _stop_task(self, task, **kwargs): # Get user keys - if task.computing.requires_user_keys: + if self.computing.requires_user_keys: user_keys = KeyPair.objects.get(user=task.user, default=True) else: raise NotImplementedError('Remote tasks not requiring keys are not yet supported') # Get computing host - host = task.computing.get_conf_param('master') - user = task.computing.get_conf_param('user') + host = self.computing.conf.get('master') + user = self.computing.conf.get('user') # Stop the task remotely stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid) @@ -445,14 +448,14 @@ class SlurmComputingManager(ComputingManager): def _get_task_log(self, task, **kwargs): # Get user keys - if task.computing.requires_user_keys: + if self.computing.requires_user_keys: user_keys = KeyPair.objects.get(user=task.user, default=True) else: raise NotImplementedError('Remote tasks not requiring keys are not yet supported') # Get computing host - host = task.computing.get_conf_param('master') - user = task.computing.get_conf_param('user') + host = self.computing.conf.get('master') + user = self.computing.conf.get('user') # View log remotely view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid) @@ -468,20 +471,20 @@ class SlurmComputingManager(ComputingManager): class RemotehopComputingManager(ComputingManager): def _start_task(self, task, **kwargs): - logger.debug('Starting a remote task "{}"'.format(task.computing)) + logger.debug('Starting a remote task "{}"'.format(self.computing)) # Get computing params - first_host = task.computing.get_conf_param('first_host') - first_user = task.computing.get_conf_param('first_user') - second_host = task.computing.get_conf_param('second_host') - second_user = task.computing.get_conf_param('second_user') - setup_command = task.computing.get_conf_param('setup_command') + first_host = self.computing.conf.get('first_host') + first_user = self.computing.conf.get('first_user') + second_host = self.computing.conf.get('second_host') + second_user = self.computing.conf.get('second_user') + setup_command = self.computing.conf.get('setup_command') # TODO: De hard-code use_agent = False # Get user keys - if task.computing.requires_user_keys: + if self.computing.requires_user_keys: user_keys = KeyPair.objects.get(user=task.user, default=True) else: raise NotImplementedError('Remote tasks not requiring keys are not yet supported') @@ -503,10 +506,10 @@ class RemotehopComputingManager(ComputingManager): authstring = '' # Set binds, only from sys config if the resource is not owned by the user - if task.computing.user != task.user: - binds = task.computing.get_conf_param('binds', from_sys_only=True ) + if self.computing.user != task.user: + binds = self.computing.sys_conf.get('binds') else: - binds = task.computing.get_conf_param('binds') + binds = self.computing.conf.get('binds') if not binds: binds = '' else: @@ -581,16 +584,16 @@ class RemotehopComputingManager(ComputingManager): def _stop_task(self, task, **kwargs): # Get user keys - if task.computing.requires_user_keys: + if self.computing.requires_user_keys: user_keys = KeyPair.objects.get(user=task.user, default=True) else: raise NotImplementedError('Remote tasks not requiring keys are not yet supported') # Get computing params - first_host = task.computing.get_conf_param('first_host') - first_user = task.computing.get_conf_param('first_user') - second_host = task.computing.get_conf_param('second_host') - second_user = task.computing.get_conf_param('second_user') + first_host = self.computing.conf.get('first_host') + first_user = self.computing.conf.get('first_user') + second_host = self.computing.conf.get('second_host') + second_user = self.computing.conf.get('second_user') # Stop the task remotely stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host) @@ -610,16 +613,16 @@ class RemotehopComputingManager(ComputingManager): def _get_task_log(self, task, **kwargs): # Get user keys - if task.computing.requires_user_keys: + if self.computing.requires_user_keys: user_keys = KeyPair.objects.get(user=task.user, default=True) else: raise NotImplementedError('Remote tasks not requiring keys are not yet supported') # Get computing params - first_host = task.computing.get_conf_param('first_host') - first_user = task.computing.get_conf_param('first_user') - second_host = task.computing.get_conf_param('second_host') - second_user = task.computing.get_conf_param('second_user') + first_host = self.computing.conf.get('first_host') + first_user = self.computing.conf.get('first_user') + second_host = self.computing.conf.get('second_host') + second_user = self.computing.conf.get('second_user') # View log remotely view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host) diff --git a/services/webapp/code/rosetta/core_app/models.py b/services/webapp/code/rosetta/core_app/models.py index a3dffde50d8ad1b33f8f25fc884e7efffaf62d8a..c374c393f7fbf3721239b3354725700db0d4f802 100644 --- a/services/webapp/code/rosetta/core_app/models.py +++ b/services/webapp/code/rosetta/core_app/models.py @@ -5,6 +5,7 @@ from django.db import models from django.contrib.auth.models import User from django.utils import timezone from .utils import os_shell, color_map, hash_string_to_int +from .exceptions import ConsistencyException if 'sqlite' in settings.DATABASES['default']['ENGINE']: from .fields import JSONField @@ -146,50 +147,54 @@ class Computing(models.Model): supports_docker = models.BooleanField(default=False) supports_singularity = models.BooleanField(default=False) - class Meta: ordering = ['name'] - def __str__(self): if self.user: return str('Computing "{}" of user "{}"'.format(self.name, self.user)) else: return str('Computing "{}"'.format(self.name)) - @property def id(self): return str(self.uuid).split('-')[0] - - @property - def sys_conf_data(self): - try: - return ComputingSysConf.objects.get(computing=self).data - except ComputingSysConf.DoesNotExist: - return None + @property + def color(self): + string_int_hash = hash_string_to_int(self.name) + color_map_index = string_int_hash % len(color_map) + return color_map[color_map_index] - @property - def sys_conf_data_json(self): - return json.dumps(self.sys_conf_data) - + #======================= + # Computing manager + #======================= - @property - def user_conf_data(self): + @property + def manager(self): + from . import computing_managers + + # Instantiate the computing manager based on type (if not already done) try: - return self._user_conf_data + return self._manager except AttributeError: - raise AttributeError('User conf data is not yet attached, please attach it before accessing.') - - - @property - def user_conf_data_json(self): - return json.dumps(self.user_conf_data) - + if self.type == 'local': + self._manager = computing_managers.LocalComputingManager(self) + elif self.type == 'remote': + self._manager = computing_managers.RemoteComputingManager(self) + elif self.type == 'slurm': + self._manager = computing_managers.SlurmComputingManager(self) + else: + raise ConsistencyException('Don\'t know how to instantiate a computing manager for computing resource of type "{}"'.format(self.type)) + return self._manager - def attach_user_conf_data(self, user): + + #======================= + # Sys & user conf + #======================= + + def attach_user_conf(self, user): if self.user and self.user != user: raise Exception('Cannot attach a conf data for another user (my user="{}", another user="{}"'.format(self.user, user)) try: @@ -197,47 +202,50 @@ class Computing(models.Model): except ComputingUserConf.DoesNotExist: self._user_conf_data = None + @property + def sys_conf(self): + return self.related_sys_conf.get().data - def get_conf_param(self, param, from_sys_only=False): + @property + def user_conf(self): try: - param_value = self.sys_conf_data[param] - except (TypeError, KeyError): - if not from_sys_only: - try: - param_value = self.user_conf_data[param] - except (TypeError, KeyError): - return None - else: - return None - return param_value + return self._user_conf_data + except AttributeError: + raise ConsistencyException('User conf has to been attached, cannot proceed.') - @property - def conf_params(self): - class ConfParams(): - def __init__(self, computing): - self.computing = computing - def __getitem__(self, key): - return self.computing.get_conf_param(key) - return ConfParams(self) + @property + def sys_conf_as_json(self): + return json.dumps(self.sys_conf) + + @property + def user_conf_as_json(self): + return json.dumps(self.user_conf) @property - def manager(self): - from . import computing_managers - ComputingManager = getattr(computing_managers, '{}ComputingManager'.format(self.type.title())) - return ComputingManager() - - @ property - def color(self): - string_int_hash = hash_string_to_int(self.name) - color_map_index = string_int_hash % len(color_map) - return color_map[color_map_index] + def conf(self): + + if not self.requires_user_conf: + conf_tmp = self.sys_conf + else: + try: + # Copy the conf or the original user conf will be affected by the overwrite below + conf_tmp = {key:value for key, value in self._user_conf_data.items()} + except AttributeError: + raise ConsistencyException('User conf has not been attached, cannot proceed.') + + # Now add (overwrite) with the sys conf + sys_conf = self.sys_conf + for key in sys_conf: + conf_tmp[key] = sys_conf[key] + return conf_tmp + class ComputingSysConf(models.Model): uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE) + computing = models.ForeignKey(Computing, related_name='related_sys_conf', on_delete=models.CASCADE) data = JSONField(blank=True, null=True) @@ -255,7 +263,7 @@ class ComputingUserConf(models.Model): uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True) - computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE) + computing = models.ForeignKey(Computing, related_name='related_user_conf', on_delete=models.CASCADE) data = JSONField(blank=True, null=True) @property diff --git a/services/webapp/code/rosetta/core_app/templates/components/computing.html b/services/webapp/code/rosetta/core_app/templates/components/computing.html index 34799c0be528362ec383cbfd6101e8110ca63a71..df1d1d4a239110688bf251e816fafde86eac50b4 100644 --- a/services/webapp/code/rosetta/core_app/templates/components/computing.html +++ b/services/webapp/code/rosetta/core_app/templates/components/computing.html @@ -48,12 +48,12 @@ <tr> <td><b>Sys Conf</b></td> - <td>{{ data.computing.sys_conf_data_json }} {% if request.user.is_superuser %} [<a href="/edit_computing_conf?type=sys&computing_uuid={{ data.computing.uuid}}">Edit</a>] {% endif %}</td> + <td>{{ data.computing.sys_conf_as_json }} {% if request.user.is_superuser %} [<a href="/edit_computing_conf?type=sys&computing_uuid={{ data.computing.uuid}}">Edit</a>] {% endif %}</td> </tr> <tr> <td><b>User Conf</b></td> - <td>{{ data.computing.user_conf_data_json }} [<a href="/edit_computing_conf?type=user&computing_uuid={{ data.computing.uuid}}">Edit</a>]</td> + <td>{{ data.computing.user_conf_as_json }} [<a href="/edit_computing_conf?type=user&computing_uuid={{ data.computing.uuid}}">Edit</a>]</td> </tr> diff --git a/services/webapp/code/rosetta/core_app/templates/create_task.html b/services/webapp/code/rosetta/core_app/templates/create_task.html index c22487addd4b2dff567e2cba17ed96af4c21a807..3bfd578400fade9c634358cf97042f5c9479f839 100644 --- a/services/webapp/code/rosetta/core_app/templates/create_task.html +++ b/services/webapp/code/rosetta/core_app/templates/create_task.html @@ -185,9 +185,9 @@ <td><b>Computing options</b></td> <td> <table> - <tr><td>Partition</td><td><input type="text" name="computing_partition" value="{{ data.task_computing.conf_params.default_partition }}" placeholder="" size="20" /></td></tr> - <tr><td>Cpus</td><td><input type="text" name="computing_cpus" value="{{ data.task_computing.conf_params.default_cpus }}" placeholder="" size="5" /></td></tr> - <tr><td>Memory</td><td><input type="text" name="computing_memory" value="{{ data.task_computing.conf_params.default_memory }}" placeholder="" size="5" /></td></tr> + <tr><td>Partition</td><td><input type="text" name="computing_partition" value="{{ data.task_computing.conf.default_partition }}" placeholder="" size="20" /></td></tr> + <tr><td>Cpus</td><td><input type="text" name="computing_cpus" value="{{ data.task_computing.conf.default_cpus }}" placeholder="" size="5" /></td></tr> + <tr><td>Memory</td><td><input type="text" name="computing_memory" value="{{ data.task_computing.conf.default_memory }}" placeholder="" size="5" /></td></tr> </table> </td> </tr> diff --git a/services/webapp/code/rosetta/core_app/utils.py b/services/webapp/code/rosetta/core_app/utils.py index 977cf9534dd6511f1eea55ebd8a7267d01b38142..71931ed9104414a299e09203ad40f49392983082 100644 --- a/services/webapp/code/rosetta/core_app/utils.py +++ b/services/webapp/code/rosetta/core_app/utils.py @@ -564,12 +564,12 @@ def setup_tunnel(task): if task.computing.type == 'remotehop': # Get computing params - first_host = task.computing.get_conf_param('first_host') - first_user = task.computing.get_conf_param('first_user') - #second_host = task.computing.get_conf_param('second_host') - #second_user = task.computing.get_conf_param('second_user') - #setup_command = task.computing.get_conf_param('setup_command') - #base_port = task.computing.get_conf_param('base_port') + first_host = task.computing.conf.get('first_host') + first_user = task.computing.conf.get('first_user') + #second_host = task.computing.conf.get('second_host') + #second_user = task.computing.conf.get('second_user') + #setup_command = task.computing.conf.get('setup_command') + #base_port = task.computing.conf.get('base_port') tunnel_command= 'ssh -4 -i {} -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} {}@{} & '.format(user_keys.private_key_file, task.tunnel_port, task.ip, task.port, first_user, first_host) diff --git a/services/webapp/code/rosetta/core_app/views.py b/services/webapp/code/rosetta/core_app/views.py index 3de2dd6e1a811aaa4e7aafb95041f9f87647b1fa..cedc871b02c9b89acad99384d2f64b3f7fe5158b 100644 --- a/services/webapp/code/rosetta/core_app/views.py +++ b/services/webapp/code/rosetta/core_app/views.py @@ -314,7 +314,7 @@ def tasks(request): data['task'] = task # Attach user config to computing - task.computing.attach_user_conf_data(task.user) + task.computing.attach_user_conf(task.user) # Task actions if action=='delete': @@ -448,7 +448,8 @@ def create_task(request): try: task_computing = Computing.objects.get(uuid=task_computing_uuid, user=request.user) except Computing.DoesNotExist: - raise Exception('Consistency error, computing with uuid "{}" does not exists or user "{}" does not have access rights'.format(task_computing_uuid, request.user.email)) + raise Exception('Consistency error, computing with uuid "{}" does not exists or user "{}" does not have access rights'.format(task_computing_uuid, request.user.email)) + task_computing.attach_user_conf(request.user) data['task_computing'] = task_computing # Handle step one/two @@ -507,7 +508,7 @@ def create_task(request): task.computing_options = computing_options # Attach user config to computing - task.computing.attach_user_conf_data(task.user) + task.computing.attach_user_conf(task.user) # Set port if not dynamic ports if not task.container.supports_dynamic_ports: @@ -569,8 +570,8 @@ def task_log(request): data['refresh'] = refresh # Attach user conf in any - task.computing.attach_user_conf_data(request.user) - + task.computing.attach_user_conf(request.user) + # Get the log try: @@ -783,7 +784,7 @@ def computings(request): data['computing'] = Computing.objects.get(uuid=computing_uuid, user=None) # Attach user conf in any - data['computing'].attach_user_conf_data(request.user) + data['computing'].attach_user_conf(request.user) else: @@ -791,7 +792,7 @@ def computings(request): # Attach user conf in any for computing in data['computings']: - computing.attach_user_conf_data(request.user) + computing.attach_user_conf(request.user) return render(request, 'computings.html', {'data': data})