diff --git a/services/webapp/code/rosetta/core_app/admin.py b/services/webapp/code/rosetta/core_app/admin.py index 1e09ed48918f33f19babb36ecb10ec470a4b1c8c..c22b65d72d4704efdd5b46076c44fbb56cc36493 100644 --- a/services/webapp/code/rosetta/core_app/admin.py +++ b/services/webapp/code/rosetta/core_app/admin.py @@ -1,14 +1,12 @@ from django.contrib import admin -from .models import Profile, LoginToken, Task, Container, Computing, ComputingConf, ComputingUserConf, Storage, KeyPair, Text +from .models import Profile, LoginToken, Task, Container, Computing, Storage, KeyPair, Text admin.site.register(Profile) admin.site.register(LoginToken) admin.site.register(Task) admin.site.register(Container) admin.site.register(Computing) -admin.site.register(ComputingConf) -admin.site.register(ComputingUserConf) admin.site.register(Storage) admin.site.register(KeyPair) admin.site.register(Text) diff --git a/services/webapp/code/rosetta/core_app/api.py b/services/webapp/code/rosetta/core_app/api.py index 8f8f4e4bd74f8c3885b23245d76bda383029313a..300d2ee5f321a36d05a1a17e51eb9dfcbdcb8f84 100644 --- a/services/webapp/code/rosetta/core_app/api.py +++ b/services/webapp/code/rosetta/core_app/api.py @@ -417,9 +417,11 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): if '$SSH_USER' in base_path_expanded: if storage.access_through_computing: computing = storage.computing - computing.attach_user_conf(user) if computing.auth_mode == 'user_keys': - base_path_expanded = base_path_expanded.replace('$SSH_USER', computing.user_conf.get('user')) + computing_user = user.profile.get_extra_conf('computing_user', storage.computing) + if not computing_user: + raise Exception('Computing resource \'{}\' user is not configured'.format(storage.computing.name)) + base_path_expanded = base_path_expanded.replace('$SSH_USER', computing_user) else: base_path_expanded = base_path_expanded.replace('$SSH_USER', computing.conf.get('user')) @@ -444,16 +446,23 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): except IndexError: computing_name = None - # Get all the storages for this name: - storages = Storage.objects.filter(name=storage_name, user=None) + # Get all the storages this user has access to: + storages = list(Storage.objects.filter(group=None, name=storage_name)) + list(Storage.objects.filter(group__user=request.user, name=storage_name)) - # Filter by computing resource name + # Filter by computing resource name (or None) if computing_name: unfiltered_storages = storages storages = [] for storage in unfiltered_storages: if storage.computing.name == computing_name: storages.append(storage) + else: + unfiltered_storages = storages + storages = [] + for storage in unfiltered_storages: + if storage.computing is None: + storages.append(storage) + # Check that we had at least and no more than one storage in the end if len(storages) == 0: @@ -714,7 +723,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): data = {'data':[]} # Get storages - storages = list(Storage.objects.filter(user=None)) + list(Storage.objects.filter(user=request.user)) + storages = list(Storage.objects.filter(group=None)) + list(Storage.objects.filter(group__user=request.user)) for storage in storages: diff --git a/services/webapp/code/rosetta/core_app/computing_managers.py b/services/webapp/code/rosetta/core_app/computing_managers.py index 2ce933f1b19eea309d9446ac569c2db64b78835a..0ab1affc42269aceea72d577f021079f3292fd90 100644 --- a/services/webapp/code/rosetta/core_app/computing_managers.py +++ b/services/webapp/code/rosetta/core_app/computing_managers.py @@ -171,10 +171,6 @@ class InternalSingleNodeComputingManager(SingleNodeComputingManager): - - - - class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingManager): def _start_task(self, task, **kwargs): @@ -208,7 +204,6 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana expanded_base_path = storage.base_path if '$SSH_USER' in expanded_base_path: if storage.access_through_computing: - self.computing.attach_user_conf(self.computing.user) expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user) else: raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented') @@ -348,7 +343,6 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag expanded_base_path = storage.base_path if '$SSH_USER' in expanded_base_path: if storage.access_through_computing: - self.computing.attach_user_conf(self.computing.user) expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user) else: raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented') diff --git a/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py b/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py index e54bc154e6d35ba458e430e357f9301d65363146..038f06402e091c1df52d713581de436ec859d093 100644 --- a/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py +++ b/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py @@ -1,6 +1,6 @@ from django.core.management.base import BaseCommand from django.contrib.auth.models import User -from ...models import Profile, Container, Computing, ComputingConf, ComputingUserConf, Storage, KeyPair, Text +from ...models import Profile, Container, Computing, Storage, KeyPair, Text class Command(BaseCommand): help = 'Adds the admin superuser with \'a\' password.' @@ -50,7 +50,7 @@ class Command(BaseCommand): #===================== # TODO: create a different pair try: - testuser = KeyPair.objects.get(user=None, default=True) + KeyPair.objects.get(user=None, default=True) print('Not creating default platform keys as they already exist') except KeyPair.DoesNotExist: @@ -193,8 +193,7 @@ class Command(BaseCommand): print('Creating demo computing resources...') # Demo internal computing - Computing.objects.create(user = None, - name = 'Demo Internal', + Computing.objects.create(name = 'Demo Internal', description = 'A demo internal computing resource.', type = 'standalone', access_mode = 'internal', @@ -204,43 +203,30 @@ class Command(BaseCommand): # Demo standalone computing plus conf - demo_singlenode_computing = Computing.objects.create(user = None, - name = 'Demo Standalone', + demo_singlenode_computing = Computing.objects.create(name = 'Demo Standalone', description = 'A demo standalone computing resource.', type = 'standalone', access_mode = 'ssh+cli', auth_mode = 'user_keys', wms = None, + conf = {'host': 'slurmclusterworker-one'}, container_runtimes = 'singularity') - ComputingConf.objects.create(computing = demo_singlenode_computing, - data = {'host': 'slurmclusterworker-one', - 'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'}) - - ComputingUserConf.objects.create(user = testuser, - computing = demo_singlenode_computing, - data = {'user': 'slurmtestuser'}) - + # Add testuser extra conf for this computing resource + testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_singlenode_computing, value= 'slurmtestuser') # Demo cluster computing plus conf - demo_slurm_computing = Computing.objects.create(user = None, - name = 'Demo Cluster', + demo_slurm_computing = Computing.objects.create(name = 'Demo Cluster', description = 'A demo cluster computing resource.', type = 'cluster', access_mode = 'ssh+cli', auth_mode = 'user_keys', wms = 'slurm', + conf = {'host': 'slurmclustermaster-main', 'default_partition': 'partition1'}, container_runtimes = 'singularity') - - ComputingConf.objects.create(computing = demo_slurm_computing, - data = {'host': 'slurmclustermaster-main', 'default_partition': 'partition1', - 'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'}) - - ComputingUserConf.objects.create(user = testuser, - computing = demo_slurm_computing, - data = {'user': 'slurmtestuser'}) - - + + # Add testuser extra conf for this computing resource + testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_slurm_computing, value= 'slurmtestuser') #===================== # Storages @@ -267,26 +253,24 @@ class Command(BaseCommand): for computing in demo_computing_resources: # Demo shared computing plus conf - Storage.objects.create(user = None, - computing = computing, - access_through_computing = True, - name = 'Shared', - type = 'generic_posix', - access_mode = 'ssh+cli', - auth_mode = 'user_keys', - base_path = '/shared/data/shared', - bind_path = '/storages/shared') + Storage.objects.create(computing = computing, + access_through_computing = True, + name = 'Shared', + type = 'generic_posix', + access_mode = 'ssh+cli', + auth_mode = 'user_keys', + base_path = '/shared/data/shared', + bind_path = '/storages/shared') # Demo shared computing plus conf - Storage.objects.create(user = None, - computing = computing, - access_through_computing = True, - name = 'Personal', - type = 'generic_posix', - access_mode = 'ssh+cli', - auth_mode = 'user_keys', - base_path = '/shared/data/users/$SSH_USER', - bind_path = '/storages/personal') + Storage.objects.create(computing = computing, + access_through_computing = True, + name = 'Personal', + type = 'generic_posix', + access_mode = 'ssh+cli', + auth_mode = 'user_keys', + base_path = '/shared/data/users/$SSH_USER', + bind_path = '/storages/personal') diff --git a/services/webapp/code/rosetta/core_app/migrations/0015_auto_20211108_1639.py b/services/webapp/code/rosetta/core_app/migrations/0015_auto_20211108_1639.py new file mode 100644 index 0000000000000000000000000000000000000000..e2a20f7630972948900aba1e1379bf741e756221 --- /dev/null +++ b/services/webapp/code/rosetta/core_app/migrations/0015_auto_20211108_1639.py @@ -0,0 +1,83 @@ +# Generated by Django 2.2.1 on 2021-11-08 16:39 + +from django.conf import settings +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('auth', '0011_update_proxy_permissions'), + ('core_app', '0014_auto_20211108_1548'), + ] + + operations = [ + migrations.RenameField( + model_name='storage', + old_name='config', + new_name='conf', + ), + migrations.RemoveField( + model_name='computing', + name='user', + ), + migrations.RemoveField( + model_name='storage', + name='user', + ), + migrations.AddField( + model_name='computing', + name='conf', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name='computing', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='computings', to='auth.Group'), + ), + migrations.AddField( + model_name='container', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='containers', to='auth.Group'), + ), + migrations.AddField( + model_name='storage', + name='group', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='storages', to='auth.Group'), + ), + migrations.AlterField( + model_name='computinguserconf', + name='computing', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='user_confs', to='core_app.Computing'), + ), + migrations.AlterField( + model_name='computinguserconf', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='computing_confs', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='container', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='containers', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='keypair', + name='user', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='key_pairs', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='task', + name='computing', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to='core_app.Computing'), + ), + migrations.AlterField( + model_name='task', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to=settings.AUTH_USER_MODEL), + ), + migrations.DeleteModel( + name='ComputingConf', + ), + ] diff --git a/services/webapp/code/rosetta/core_app/migrations/0016_profile_extra_conf.py b/services/webapp/code/rosetta/core_app/migrations/0016_profile_extra_conf.py new file mode 100644 index 0000000000000000000000000000000000000000..0c6a1a9adecfd2c1cdc7aebe7bab215830e8f411 --- /dev/null +++ b/services/webapp/code/rosetta/core_app/migrations/0016_profile_extra_conf.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.1 on 2021-11-08 17:53 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core_app', '0015_auto_20211108_1639'), + ] + + operations = [ + migrations.AddField( + model_name='profile', + name='extra_conf', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + ] diff --git a/services/webapp/code/rosetta/core_app/migrations/0017_auto_20211108_1759.py b/services/webapp/code/rosetta/core_app/migrations/0017_auto_20211108_1759.py new file mode 100644 index 0000000000000000000000000000000000000000..171e8e5b05bf76609426900a47d49aaf279b9bf3 --- /dev/null +++ b/services/webapp/code/rosetta/core_app/migrations/0017_auto_20211108_1759.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.1 on 2021-11-08 17:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core_app', '0016_profile_extra_conf'), + ] + + operations = [ + migrations.RenameField( + model_name='profile', + old_name='extra_conf', + new_name='extra_confs', + ), + ] diff --git a/services/webapp/code/rosetta/core_app/migrations/0018_delete_computinguserconf.py b/services/webapp/code/rosetta/core_app/migrations/0018_delete_computinguserconf.py new file mode 100644 index 0000000000000000000000000000000000000000..718f76aa53ba2f783c4f2101302a7c01f38f024c --- /dev/null +++ b/services/webapp/code/rosetta/core_app/migrations/0018_delete_computinguserconf.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2.1 on 2021-11-08 19:30 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core_app', '0017_auto_20211108_1759'), + ] + + operations = [ + migrations.DeleteModel( + name='ComputingUserConf', + ), + ] diff --git a/services/webapp/code/rosetta/core_app/models.py b/services/webapp/code/rosetta/core_app/models.py index eee49ad033c79d2efd24224dc66006df897460db..6c5433025c68a585b6396e437a258956d58253dd 100644 --- a/services/webapp/code/rosetta/core_app/models.py +++ b/services/webapp/code/rosetta/core_app/models.py @@ -2,7 +2,7 @@ import uuid import json from django.conf import settings from django.db import models -from django.contrib.auth.models import User +from django.contrib.auth.models import User, Group from django.utils import timezone from .utils import os_shell, color_map, hash_string_to_int, get_task_tunnel_host from .exceptions import ConsistencyException @@ -46,6 +46,7 @@ class Profile(models.Model): timezone = models.CharField('User Timezone', max_length=36, default='UTC') authtoken = models.CharField('User auth token', max_length=36, blank=True, null=True) is_power_user = models.BooleanField('Power user status', default=False) + extra_confs = JSONField(blank=True, null=True) def save(self, *args, **kwargs): @@ -54,9 +55,32 @@ class Profile(models.Model): super(Profile, self).save(*args, **kwargs) - def __unicode__(self): - return str('Profile of user "{}"'.format(self.user.username)) + def __str__(self): + return str('Profile of user "{}"'.format(self.user.email)) + + def add_extra_conf(self, conf_type, object=None, value=None): + if value in [None, '']: # TODO: improve me? + raise ValueError('Empty value') + if self.extra_confs is None: + self.extra_confs = {} + self.extra_confs[str(uuid.uuid4())] = {'type': conf_type, 'object_uuid': str(object.uuid), 'value': value} + self.save() + + + def get_extra_conf(self, conf_type, object=None): + + if self.extra_confs: + for extra_conf in self.extra_confs: + if conf_type == self.extra_confs[extra_conf]['type']: + if object: + #logger.debug("{} vs {}".format(self.extra_confs[extra_conf]['object_uuid'], str(object.uuid))) + if self.extra_confs[extra_conf]['object_uuid'] == str(object.uuid): + return self.extra_confs[extra_conf]['value'] + else: + return self.extra_confs[extra_conf]['value'] + return None + #========================= @@ -69,6 +93,9 @@ class LoginToken(models.Model): user = models.OneToOneField(User, on_delete=models.CASCADE) token = models.CharField('Login token', max_length=36) + def __str__(self): + return str('Login token of user "{}"'.format(self.user.email)) + #========================= @@ -77,8 +104,11 @@ class LoginToken(models.Model): class Container(models.Model): uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, blank=True, null=True) + user = models.ForeignKey(User, related_name='containers', on_delete=models.CASCADE, blank=True, null=True) # If a container has no user, it will be available to anyone. Can be created, edited and deleted only by admins. + group = models.ForeignKey(Group, related_name='containers', on_delete=models.CASCADE, blank=True, null=True) + # If a container has no group, it will be available to anyone. Can be created, edited and deleted only by admins. + # Generic attributes name = models.CharField('Name', max_length=255, blank=False, null=False) @@ -109,11 +139,9 @@ class Container(models.Model): ordering = ['name'] def __str__(self): - return str('Container "{}" with image "{}" and tag "{}" of user "{}" on registry "{}" '.format(self.name, self.image, self.tag, self.user, self.registry)) + user_str = self.user.email if self.user else None + return str('Container "{}" of user "{}" with image "{}" and tag "{}" on registry "{}" '.format(self.name, user_str, self.image, self.tag, self.registry)) - #@property - #def id(self): - # return str(self.uuid).split('-')[0] @ property def color(self): @@ -129,8 +157,8 @@ class Container(models.Model): class Computing(models.Model): uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, blank=True, null=True) - # If a compute resource has no user, it will be available to anyone. Can be created, edited and deleted only by admins. + group = models.ForeignKey(Group, related_name='computings', on_delete=models.CASCADE, blank=True, null=True) + # If a compute resource has no group, it will be available to anyone. Can be created, edited and deleted only by admins. name = models.CharField('Name', max_length=255, blank=False, null=False) description = models.TextField('Description', blank=True, null=True) @@ -146,18 +174,23 @@ class Computing(models.Model): # Supported container runtimes container_runtimes = models.CharField('Container runtimes', max_length=256, blank=False, null=False) + # Conf + conf = JSONField(blank=True, null=True) + + class Meta: ordering = ['name'] + def __str__(self): - if self.user: - return str('Computing "{}" of user "{}"'.format(self.name, self.user)) + if self.group: + return str('Computing "{}" of group "{}"'.format(self.name, self.group)) else: return str('Computing "{}"'.format(self.name)) @property - def id(self): - return str(self.uuid).split('-')[0] + def uuid_as_str(self): + return str(self.uuid) @property def color(self): @@ -192,78 +225,6 @@ class Computing(models.Model): raise ConsistencyException('Don\'t know how to instantiate a computing manager for computing resource of type "{}", access mode "{}" and WMS "{}"'.format(self.type, self.access_mode, self.wms)) return self._manager - - #======================= - # Conf & user conf - #======================= - - def attach_user_conf(self, user): - if self.user and self.user != user: - raise Exception('Cannot attach a conf data for another user (my user="{}", another user="{}"'.format(self.user, user)) - try: - self._user_conf_data = ComputingUserConf.objects.get(computing=self, user=user).data - except ComputingUserConf.DoesNotExist: - self._user_conf_data = None - - @property - def conf(self): - try: - return self.related_computing_conf.get().data - except: - return {} - #TODO: add a setter and start removing the ComputingConf model - - @property - def conf_as_json(self): - return json.dumps(self.conf) - - @property - def user_conf(self): - try: - return self._user_conf_data - except AttributeError: - raise ConsistencyException('User conf has to been attached, cannot proceed.') - - @property - def user_conf_as_json(self): - return json.dumps(self.user_conf) - - - - -class ComputingConf(models.Model): - - uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - computing = models.ForeignKey(Computing, related_name='related_computing_conf', on_delete=models.CASCADE) - data = JSONField(blank=True, null=True) - - - @property - def id(self): - return str(self.uuid).split('-')[0] - - - def __str__(self): - return 'Computing sys conf for {} with id "{}"'.format(self.computing, self.id) - - - -class ComputingUserConf(models.Model): - - uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True) - computing = models.ForeignKey(Computing, related_name='related_user_conf', on_delete=models.CASCADE) - data = JSONField(blank=True, null=True) - - @property - def id(self): - return str(self.uuid).split('-')[0] - - def __str__(self): - return 'Computing user conf for {} with id "{}" of user "{}"'.format(self.computing, self.id, self.user) - - - #========================= # Tasks @@ -272,7 +233,7 @@ class ComputingUserConf(models.Model): class Task(models.Model): uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE) + user = models.ForeignKey(User, related_name='tasks', on_delete=models.CASCADE) name = models.CharField('Name', max_length=36, blank=False, null=False) # Task management @@ -295,7 +256,7 @@ class Task(models.Model): auth_token = models.CharField('Auth token', max_length=36, blank=True, null=True) # A one-time token for proxy or interface authentication # Links - computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE) + computing = models.ForeignKey(Computing, related_name='tasks', on_delete=models.CASCADE) container = models.ForeignKey('Container', on_delete=models.CASCADE, related_name='+') # Computing options @@ -338,13 +299,9 @@ class Task(models.Model): self.save() - #@property - #def id(self): - # return str(self.uuid).split('-')[0] - def __str__(self): - return str('Task "{}" of user "{}" running on "{}" in status "{}" created at "{}"'.format(self.name, self.user, self.computing, self.status, self.created)) + return str('Task "{}" of user "{}" running on "{}" in status "{}" created at "{}"'.format(self.name, self.user.email, self.computing, self.status, self.created)) @property def color(self): @@ -371,7 +328,7 @@ class Task(models.Model): class Storage(models.Model): uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, blank=True, null=True) + group = models.ForeignKey(Group, related_name='storages', on_delete=models.CASCADE, blank=True, null=True) name = models.CharField('Name', max_length=255, blank=False, null=False) #description = models.TextField('Description', blank=True, null=True) @@ -393,15 +350,15 @@ class Storage(models.Model): # If the above is linked, some configuration can be taken from the linked computing resource (i.e. the hostname) # Configuration - config = JSONField(blank=True, null=True) + conf = JSONField(blank=True, null=True) class Meta: ordering = ['name'] def __str__(self): - if self.user: - return str('Storage "{}" of user "{}"'.format(self.id, self.user)) + if self.group: + return str('Storage "{}" of group "{}"'.format(self.id, self.group)) else: return str('Storage "{}"'.format(self.id)) @@ -420,7 +377,7 @@ class Storage(models.Model): class KeyPair(models.Model): uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, blank=True, null=True) + user = models.ForeignKey(User, related_name='key_pairs', on_delete=models.CASCADE, blank=True, null=True) private_key_file = models.CharField('Private key file', max_length=4096, blank=False, null=False) public_key_file = models.CharField('Public key file', max_length=4096, blank=False, null=False) @@ -429,12 +386,7 @@ class KeyPair(models.Model): def __str__(self): - return str('KeyPair with id "{}" of user "{}"'.format(self.id, self.user)) - - - @property - def id(self): - return str(self.uuid).split('-')[0] + return str('KeyPair of user "{}" (default={})'.format( self.user.email, self.default)) diff --git a/services/webapp/code/rosetta/core_app/templates/account.html b/services/webapp/code/rosetta/core_app/templates/account.html index 909cc3e610696c971cb9ee8b7064f6271cf98836..42c52eafa46c949f3687331d117097951fd853c2 100644 --- a/services/webapp/code/rosetta/core_app/templates/account.html +++ b/services/webapp/code/rosetta/core_app/templates/account.html @@ -62,11 +62,11 @@ </td> </tr> - </table> - <br /> + </table> + <br /> - <h3>Profile</h3> - <table class="dashboard"> + <h3>Profile</h3> + <table class="dashboard"> <tr> <td> @@ -83,11 +83,38 @@ {% endif %} </td> </tr> + + + <tr> + <td valign=top> + <b>Extra configurations</b> + </td> + <td> + {% for conf_uuid, conf_data in data.profile.extra_confs.items %} + <code>{{ conf_data.type }}</code> + {% if conf_data.object_uuid %} + {% for computing in data.computings %} + {% if conf_data.object_uuid == computing.uuid_as_str %} + <font style="font-size:0.9em"> @ {{ computing.name }}</font> + {% endif %} + {% endfor %} + {% endif %} + + : <code>{{conf_data.value}}</code> | <a href='?delete_extra_conf_uuid={{conf_uuid}}'>delete</a> + <br/> + {% endfor %} + + + + <span style="margin:3px"><a href="/add_profile_conf" style="line-height:2em">Add new...</a></span> + </td> + </tr> + </table> <br /> - <h3>KeyPair</h3> - <table class="dashboard"> + <h3>KeyPair</h3> + <table class="dashboard"> <tr> <td valign="top"> diff --git a/services/webapp/code/rosetta/core_app/templates/add_profile_conf.html b/services/webapp/code/rosetta/core_app/templates/add_profile_conf.html new file mode 100644 index 0000000000000000000000000000000000000000..1b1369a316ac04e149095106a025108fbce36776 --- /dev/null +++ b/services/webapp/code/rosetta/core_app/templates/add_profile_conf.html @@ -0,0 +1,91 @@ +{% load static %} +{% include "header.html" %} +{% include "navigation.html" with main_path='/main/' %} + +<br/> +<br/> + +<div class="container"> + <div class="dashboard"> + <div class="span8 offset2"> + <h1>Add extra profile configuration</h1> + <hr> + + <h4>Chose the configuration type and add the values</h4> + + <br/> + + <form action="/add_profile_conf/" method="POST"> + {% csrf_token %} + + <table class="dashboard" style="max-width:430px"> + <tr> + + <td> + {%if data.conf_type %} + <select name="conf_type"> + <option value="{{data.conf_type}}" selected>{{data.conf_type}}</option> + </select> + {% else %} + <select name="conf_type"> + {% for conf_type in data.conf_types %} + <option value="{{conf_type}}">{{conf_type}}</option> + {% endfor %} + </select> + {% endif %} + </td> + + {%if data.conf_type %} + <td> + {%if data.computing %} + <select name="computing_uuid"> + <option value="{{data.computing.uuid}}" selected>{{data.computing.name}}</option> + </select> + {% else %} + <select name="computing_uuid"> + {% for computing in data.computings %} + <option value="{{computing.uuid}}">{{computing.name}}</option> + {% endfor %} + </select> + {% endif %} + </td> + {% endif %} + + + {% if data.last_step %} + <td colspan=2 align=center style="padding:20px"> + <input name="value" type="text" value=""> + </td> + <td colspan=2 align=center style="padding:20px"> + <input type="submit" value="Save"> + </td> + {% else %} + <td colspan=2 align=center style="padding:20px"> + <input type="submit" value="Next"> + </td> + {% endif %} + + </table> + </form> + + <br/> + <br/> + <br/> + <br/> + <br/> + <br/> + <br/> + <br/> + <br/> + + </div> + </div> +</div> + +{% include "footer.html" %} + + + + + + diff --git a/services/webapp/code/rosetta/core_app/templates/components/computing.html b/services/webapp/code/rosetta/core_app/templates/components/computing.html index d906e9592bb65b367f6e32dc87a8dcf84f304727..0a56384b893090a8395eb06092135c0b07b0a20f 100644 --- a/services/webapp/code/rosetta/core_app/templates/components/computing.html +++ b/services/webapp/code/rosetta/core_app/templates/components/computing.html @@ -54,23 +54,6 @@ <td>{{ data.computing.container_runtimes }}</td> </tr> - <tr> - <td><b>Sys Conf</b></td> - <td>{{ data.computing.conf_as_json }} {% if request.user.is_superuser %} [<a href="/edit_computing_conf?type=sys&computing_uuid={{ data.computing.uuid}}">Edit</a>] {% endif %}</td> - </tr> - - <tr> - <td><b>User Conf</b></td> - <td>{{ data.computing.user_conf_as_json }} [<a href="/edit_computing_conf?type=user&computing_uuid={{ data.computing.uuid}}">Edit</a>]</td> - </tr> - - - {% if data.computing.user %} - <tr> - <td><b>Operations</b></td> - <td><a href="?action=delete&uuid={{ data.computing.uuid }}">Delete</a></td> - </tr> - {% endif %} </table> </div> diff --git a/services/webapp/code/rosetta/core_app/templates/components/task.html b/services/webapp/code/rosetta/core_app/templates/components/task.html index 8508df3ffbf58f5a6154d619a92c093f1dfb4d04..2c269fbd051526f7763288886f89210b1a60ad38 100644 --- a/services/webapp/code/rosetta/core_app/templates/components/task.html +++ b/services/webapp/code/rosetta/core_app/templates/components/task.html @@ -3,7 +3,7 @@ <center> <div style="width:350px; display:block; border: #e0e0e0 solid 1px; margin:10px; background:#f8f8f8; margin-bottom:20px"> <div style="margin-top:5px; padding:10px; text-align:center; border-bottom: #f8f8f8 solid 1px; "> - <b>Task {{data.task.name}} summary</b> + Task <b>{{data.task.name}}</b> summary </div> {% else %} <div style="width:350px; float:left; border: #e0e0e0 solid 1px; margin:10px; background:#f8f8f8; margin-bottom:20px"> @@ -143,8 +143,8 @@ {% endif %} - <tr><td style="padding-right:0"><b>Direct link</b> - <td>{% if task.status == "running" %}<a href="{{ task.sharable_link }}">{{ task.sharable_link }}</a>{% else %}N.A. (task not running) {% endif %}</td> + <!-- <tr><td style="padding-right:0"><b>Direct link</b> + <td>{% if task.status == "running" %}<a href="{{ task.sharable_link }}">{{ task.sharable_link }}</a>{% else %}N.A. (task not running) {% endif %}</td> --> </tr> diff --git a/services/webapp/code/rosetta/core_app/templates/edit_computing_conf.html b/services/webapp/code/rosetta/core_app/templates/edit_computing_conf.html deleted file mode 100644 index 10e0cb6bdf157e31abda67eb1606c2ab7288f164..0000000000000000000000000000000000000000 --- a/services/webapp/code/rosetta/core_app/templates/edit_computing_conf.html +++ /dev/null @@ -1,67 +0,0 @@ -{% load static %} -{% include "header.html" %} -{% include "navigation.html" with main_path='/main/' %} - -<br/> -<br/> - -<div class="container"> - <div class="dashboard"> - <div class="span8 offset2"> - <h1>Edit computing conf</h1> - <hr> - - <h4>Edit the configuration in JSON format for {{ data.computing }}</h4> - - <br/> - - <form action="/edit_computing_conf/" method="POST"> - {% csrf_token %} - <input type="hidden" name="type" value="{{ data.type }}"> - <input type="hidden" name="computing_uuid" value="{{ data.computing.uuid }}"> - <table class="dashboard" style="max-width:430px"> - - <tr> - <td> - <textarea name="new_conf" style="height:300px; width:500px">{{ data.computing_conf_data_json }}</textarea> - </td> - </tr> - - {% if data.saved %} - <tr> - <td colspan=2 align=center style="padding:0px"> - <font color="green">Saved</font> - </td> - </tr> - {% endif %} - - <tr> - <td colspan=2 align=center style="padding:20px"> - <input type="submit" value="Save"> - </td> - </tr> - - </table> - </form> - - <br/> - <br/> - <br/> - <br/> - <br/> - <br/> - <br/> - <br/> - <br/> - - </div> - </div> -</div> - -{% include "footer.html" %} - - - - - - diff --git a/services/webapp/code/rosetta/core_app/templates/main.html b/services/webapp/code/rosetta/core_app/templates/main.html index b134900b8ce969187999b72a9f037206a6abee9b..e91e4bfcc1d58be4eca4754e627adb9c850db786 100644 --- a/services/webapp/code/rosetta/core_app/templates/main.html +++ b/services/webapp/code/rosetta/core_app/templates/main.html @@ -7,7 +7,7 @@ <div style="display:table-row"> <div class="text-vertical-center"> <h1> Rosetta <img src="/static/img/emoji_u1f6f0.png" style="height:84px; width:64px; padding-bottom:20px"></h1> - <h2 style="margin-top:10px; margin-left:25px; margin-right:25px; font-weight:100; line-height: 30px;"><i>A Science Platform for interactive data analysis<br></i></h2> + <h2 style="margin-top:10px; margin-left:25px; margin-right:25px; font-weight:100; line-height: 30px;"><i>A container-centric Science Platform<br></i></h2> </div> </div> diff --git a/services/webapp/code/rosetta/core_app/templates/navigation.html b/services/webapp/code/rosetta/core_app/templates/navigation.html index 4d80f42adebd3f570419a6929bdc1a7bbabaaf83..77ff62d697b2140b20180630f03e3ae7eff5dba0 100644 --- a/services/webapp/code/rosetta/core_app/templates/navigation.html +++ b/services/webapp/code/rosetta/core_app/templates/navigation.html @@ -25,7 +25,7 @@ </li> <li> - <a href="/files" onclick = $("#menu-close").click(); >Storage</a> + <a href="/files" onclick = $("#menu-close").click(); >Storages</a> </li> <li> diff --git a/services/webapp/code/rosetta/core_app/utils.py b/services/webapp/code/rosetta/core_app/utils.py index 057675338720817e7279e2f69b3fd37779a26bf6..04c0e1f26a8ef7a7a8d32d1d189fbeb645095570 100644 --- a/services/webapp/code/rosetta/core_app/utils.py +++ b/services/webapp/code/rosetta/core_app/utils.py @@ -699,14 +699,18 @@ Listen '''+str(task.tcp_tunnel_port)+''' def get_ssh_access_mode_credentials(computing, user): from .models import KeyPair # Get computing host - computing_host = computing.conf.get('host') + try: + computing_host = computing.conf.get('host') + except AttributeError: + computing_host = None if not computing_host: raise Exception('No computing host?!') # Get computing user and keys if computing.auth_mode == 'user_keys': - computing.attach_user_conf(user) - computing_user = computing.user_conf.get('user') + computing_user = user.profile.get_extra_conf('computing_user', computing) + if not computing_user: + raise Exception('Computing resource \'{}\' user is not configured'.format(computing.name)) # Get user key computing_keys = KeyPair.objects.get(user=user, default=True) elif computing.auth_mode == 'platform_keys': diff --git a/services/webapp/code/rosetta/core_app/views.py b/services/webapp/code/rosetta/core_app/views.py index ebb3eded12357a1c994b645d4ff506ac47bac85e..1f929d110dafc434941771e4cc1bcecc612cb32f 100644 --- a/services/webapp/code/rosetta/core_app/views.py +++ b/services/webapp/code/rosetta/core_app/views.py @@ -9,7 +9,7 @@ from django.http import HttpResponse, HttpResponseRedirect from django.contrib.auth.models import User from django.shortcuts import redirect from django.db.models import Q -from .models import Profile, LoginToken, Task, TaskStatuses, Container, Computing, KeyPair, ComputingConf, ComputingUserConf, Text +from .models import Profile, LoginToken, Task, TaskStatuses, Container, Computing, KeyPair, Text from .utils import send_email, format_exception, timezonize, os_shell, booleanize, debug_param, get_task_tunnel_host, get_task_proxy_host, random_username, setup_tunnel_and_proxy, finalize_user_creation from .decorators import public_view, private_view from .exceptions import ErrorMessage @@ -231,6 +231,10 @@ def account(request): with open(KeyPair.objects.get(user=request.user, default=True).public_key_file) as f: data['default_public_key'] = f.read() + # Add computings (for extra confs) + if request.user.profile.extra_confs: + data['computings'] = list(Computing.objects.filter(group=None)) + list(Computing.objects.filter(group__user=request.user)) + # Edit values if edit and value: try: @@ -273,6 +277,21 @@ def account(request): data['error'] = 'The property "{}" does not exists or the value "{}" is not valid.'.format(edit, value) return render(request, 'error.html', {'data': data}) + # Lastly, do we have to remove an extra conf? + + delete_extra_conf_uuid = request.GET.get('delete_extra_conf_uuid', None) + if delete_extra_conf_uuid: + #logger.debug('Deleting extra conf "{}"'.format(delete_extra_conf_uuid)) + new_extra_confs = {} + for extra_conf_uuid in profile.extra_confs: + if extra_conf_uuid != delete_extra_conf_uuid: + new_extra_confs[extra_conf_uuid] = profile.extra_confs[extra_conf_uuid] + profile.extra_confs = new_extra_confs + profile.save() + return redirect('/account') + + + return render(request, 'account.html', {'data': data}) @@ -310,9 +329,6 @@ def tasks(request): raise ErrorMessage('Task does not exists or no access rights') data['task'] = task - # Attach user config to computing - task.computing.attach_user_conf(task.user) - # Task actions if action=='delete': if task.status not in [TaskStatuses.stopped, TaskStatuses.exited]: @@ -431,13 +447,12 @@ def create_task(request): def get_task_computing(request): task_computing_uuid = request.POST.get('task_computing_uuid', None) try: - task_computing = Computing.objects.get(uuid=task_computing_uuid, user=None) + task_computing = Computing.objects.get(uuid=task_computing_uuid, group=None) except Computing.DoesNotExist: try: - task_computing = Computing.objects.get(uuid=task_computing_uuid, user=request.user) + task_computing = Computing.objects.get(uuid=task_computing_uuid, group__user=request.user) except Computing.DoesNotExist: raise Exception('Consistency error, computing with uuid "{}" does not exists or user "{}" does not have access rights'.format(task_computing_uuid, request.user.email)) - task_computing.attach_user_conf(request.user) return task_computing # Get task name helper function @@ -465,7 +480,7 @@ def create_task(request): data['task_container'] = get_task_container(request) # List all computing resources - data['computings'] = list(Computing.objects.filter(user=None)) + list(Computing.objects.filter(user=request.user)) + data['computings'] = list(Computing.objects.filter(group=None)) + list(Computing.objects.filter(group__user=request.user)) data['step'] = 'two' data['next_step'] = 'three' @@ -569,9 +584,6 @@ def create_task(request): if computing_options: task.computing_options = computing_options - # Attach user config to computing - task.computing.attach_user_conf(task.user) - # Save the task before starting it, or the computing manager will not be able to work properly task.save() @@ -630,9 +642,6 @@ def task_log(request): data['task'] = task data['refresh'] = refresh - # Attach user conf in any - task.computing.attach_user_conf(request.user) - # Get the log try: @@ -846,135 +855,57 @@ def computings(request): if details and computing_uuid: try: - data['computing'] = Computing.objects.get(uuid=computing_uuid, user=request.user) + data['computing'] = Computing.objects.get(uuid=computing_uuid, group__user=request.user) except Computing.DoesNotExist: - data['computing'] = Computing.objects.get(uuid=computing_uuid, user=None) - - # Attach user conf in any - data['computing'].attach_user_conf(request.user) - - + data['computing'] = Computing.objects.get(uuid=computing_uuid, group=None) else: - data['computings'] = list(Computing.objects.filter(user=None)) + list(Computing.objects.filter(user=request.user)) + data['computings'] = list(Computing.objects.filter(group=None)) + list(Computing.objects.filter(group__user=request.user)) - # Attach user conf in any - for computing in data['computings']: - computing.attach_user_conf(request.user) - return render(request, 'computings.html', {'data': data}) -#========================= -# Add Computing view -#========================= - -@private_view -def add_computing(request): - - # Init data - data={} - data['user'] = request.user - data['profile'] = Profile.objects.get(user=request.user) - data['title'] = 'Add computing' - data['name'] = request.POST.get('name',None) - - - return render(request, 'add_computing.html', {'data': data}) - #========================= -# Edit Computing conf view +# Add profile conf view #========================= - + @private_view -def edit_computing_conf(request): - +def add_profile_conf(request): + # Init data data={} data['user'] = request.user - data['profile'] = Profile.objects.get(user=request.user) - data['title'] = 'Add computing' - - # Get computing conf type - computing_conf_type = request.GET.get('type', request.POST.get('type', None)) - if not computing_conf_type: - raise Exception('Missing type') - - # Get computing uuid - computing_uuid = request.GET.get('computing_uuid', request.POST.get('computing_uuid', None)) - if not computing_uuid: - raise Exception('Missing computing_uuid') - - new_conf = request.POST.get('new_conf', None) - - - if computing_conf_type == 'sys': - - data['type'] = 'sys' - - if not request.user.is_superuser: - raise Exception('Cannot edit sys conf as not superuser') - # Get computing - try: - computing = Computing.objects.get(uuid=computing_uuid) - data['computing'] = computing - except ComputingConf.DoesNotExist: - raise Exception('Unknown computing "{}"'.format(computing_uuid)) - - # Get computing conf - computingSysConf, _ = ComputingConf.objects.get_or_create(computing=computing) - - # Edit conf? - if new_conf: - new_conf_data = json.loads(new_conf) - logger.debug('Setting new conf data for sys conf "{}": "{}"'.format(computingSysConf.uuid, new_conf_data)) - computingSysConf.data = new_conf_data - computingSysConf.save() - data['saved'] = True - return HttpResponseRedirect('/computings') - - - # Dump conf data for the webpage - if computingSysConf.data: - data['computing_conf_data'] = computingSysConf.data - data['computing_conf_data_json'] = json.dumps(computingSysConf.data) + # Set conf types we can add + data['conf_types'] = ['computing_user'] #,'computing_custom_binds'] - elif computing_conf_type == 'user': - - data['type'] = 'user' - - # Get computing - try: - computing = Computing.objects.get(uuid=computing_uuid) - data['computing'] = computing - except ComputingUserConf.DoesNotExist: - raise Exception('Unknown computing "{}"'.format(computing_uuid)) - - # Get computing conf - computingUserConf, _ = ComputingUserConf.objects.get_or_create(computing=computing, user=request.user) - - # Edit conf? - if new_conf: - new_conf_data = json.loads(new_conf) - logger.debug('Setting new conf data for user conf "{}": "{}"'.format(computingUserConf.uuid, new_conf_data)) - computingUserConf.data = new_conf_data - computingUserConf.save() - data['saved'] = True - return HttpResponseRedirect('/computings') - - # Dump conf data for the webpage - if computingUserConf.data: - data['computing_conf_data'] = computingUserConf.data - data['computing_conf_data_json'] = json.dumps(computingUserConf.data) - - - else: - raise Exception('Unknown computing conf type "{}"'.format(computing_conf_type)) + # Process adding the new conf + conf_type = request.POST.get('conf_type', None) + if conf_type: + data['conf_type'] = conf_type + if conf_type in ['computing_user']: + computing_uuid = request.POST.get('computing_uuid', None) + if computing_uuid: + try: + computing = Computing.objects.get(uuid=computing_uuid, group__user=request.user) + except Computing.DoesNotExist: + computing = Computing.objects.get(uuid=computing_uuid, group=None) + data['computing'] = computing + data['last_step'] = True + value = request.POST.get('value', None) + if value: + request.user.profile.add_extra_conf(conf_type=conf_type, object=computing, value=value) + # Now redirect to site + return HttpResponseRedirect('/account/') + + else: + data['computings'] = list(Computing.objects.filter(group=None)) + list(Computing.objects.filter(group__user=request.user)) + else: + raise ErrorMessage('Unknown conf type \'{}\''.format(conf_type)) - - return render(request, 'edit_computing_conf.html', {'data': data}) + + return render(request, 'add_profile_conf.html', {'data': data}) #========================= diff --git a/services/webapp/code/rosetta/urls.py b/services/webapp/code/rosetta/urls.py index 77353f90d592049706ab961d1c0b483604a18626..4807d439fa0c151a45f637ca0860b79b741046c8 100644 --- a/services/webapp/code/rosetta/urls.py +++ b/services/webapp/code/rosetta/urls.py @@ -46,13 +46,13 @@ urlpatterns = [ path('logout/', core_app_views.logout_view), url(r'^register/$', core_app_views.register_view), url(r'^account/$', core_app_views.account), + url(r'^add_profile_conf/$', core_app_views.add_profile_conf), + url(r'^tasks/$', core_app_views.tasks), url(r'^create_task/$', core_app_views.create_task), url(r'^task_log/$', core_app_views.task_log), url(r'^task_connect/$', core_app_views.task_connect), url(r'^computings/$', core_app_views.computings), - url(r'^add_computing/$', core_app_views.add_computing), - url(r'^edit_computing_conf/$', core_app_views.edit_computing_conf), url(r'^containers/$', core_app_views.containers), url(r'^add_container/$', core_app_views.add_container), url(r'^files/$', core_app_views.files_view),