Skip to content
Snippets Groups Projects
Commit bdfe2f7c authored by Stefano Alberto Russo's avatar Stefano Alberto Russo
Browse files

Added the Storage model.

parent 09a66221
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 390 additions and 193 deletions
services/slurmclustermaster/prestartup_slurmclustermaster.sh
+ 5
− 1
View file @ bdfe2f7c
......@@ -7,8 +7,12 @@ mkdir -p /shared/rosetta && chown rosetta:rosetta /shared/rosetta
# Shared home for slurmtestuser to simulate a shared home folders filesystem
cp -a /home_slurmtestuser_vanilla /shared/home_slurmtestuser
# Create shared "data" and "scratch" directories
# Create shared data directories
mkdir -p /shared/scratch
chmod 777 /shared/scratch
mkdir -p /shared/data/shared
chmod 777 /shared/data/shared
mkdir -p /shared/data/users/slurmtestuser
chown slurmtestuser:slurmtestuser /shared/data/users/slurmtestuser
services/webapp/code/rosetta/core_app/admin.py
+ 2
− 1
View file @ bdfe2f7c
from django.contrib import admin
from .models import Profile, LoginToken, Task, Container, Computing, ComputingSysConf, ComputingUserConf, KeyPair, Text
from .models import Profile, LoginToken, Task, Container, Computing, ComputingSysConf, ComputingUserConf, Storage, KeyPair, Text
admin.site.register(Profile)
admin.site.register(LoginToken)
......@@ -9,5 +9,6 @@ admin.site.register(Container)
admin.site.register(Computing)
admin.site.register(ComputingSysConf)
admin.site.register(ComputingUserConf)
admin.site.register(Storage)
admin.site.register(KeyPair)
admin.site.register(Text)
services/webapp/code/rosetta/core_app/api.py
+ 167
− 148
View file @ bdfe2f7c
This diff is collapsed.
services/webapp/code/rosetta/core_app/computing_managers.py
+ 74
− 42
View file @ bdfe2f7c
from .models import TaskStatuses, KeyPair, Task
from .models import TaskStatuses, KeyPair, Task, Storage
from .utils import os_shell
from .exceptions import ErrorMessage, ConsistencyException
from django.conf import settings
......@@ -180,9 +180,9 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
def _start_task(self, task, **kwargs):
logger.debug('Starting a remote task "{}"'.format(self.computing))
# Get computing host
host = self.computing.conf.get('host')
user = self.computing.conf.get('user')
# Get computing user and host
computing_user = self.computing.conf.get('user')
computing_host = self.computing.conf.get('host')
# Get user keys
if self.computing.requires_user_keys:
......@@ -195,7 +195,7 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
webapp_conn_string = get_webapp_conn_string()
# Handle container runtime
if task.computing.default_container_runtime == 'singularity':
if self.computing.default_container_runtime == 'singularity':
#if not task.container.supports_custom_interface_port:
# raise Exception('This task does not support dynamic port allocation and is therefore not supported using singularity on Slurm')
......@@ -205,24 +205,40 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
if not task.requires_proxy_auth and task.password:
authstring = ' export SINGULARITYENV_AUTH_PASS={} && '.format(task.password)
# Set binds, only from sys config if the resource is not owned by the user
if self.computing.user != task.user:
binds = self.computing.sys_conf.get('binds')
else:
binds = self.computing.conf.get('binds')
if not binds:
binds = ''
else:
binds = '-B {}'.format(binds)
# Manage task extra binds
if task.extra_binds:
if not binds:
binds = '-B {}'.format(task.extra_binds)
else:
binds += ',{}'.format(task.extra_binds)
# Handle storages (binds)
binds = ''
storages = Storage.objects.filter(computing=self.computing)
for storage in storages:
if storage.type == 'generic_posix' and storage.bind_path:
# Expand the base path
expanded_base_path = storage.base_path
if '$SSH_USER' in expanded_base_path:
if storage.access_through_computing:
self.computing.attach_user_conf(self.computing.user)
expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user)
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_base_path:
expanded_base_path = expanded_base_path.replace('$USER', self.task.user.name)
# Expand the bind_path
expanded_bind_path = storage.bind_path
if '$SSH_USER' in expanded_bind_path:
if storage.access_through_computing:
expanded_bind_path = expanded_bind_path.replace('$SSH_USER', computing_user)
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_bind_path:
expanded_bind_path = expanded_bind_path.replace('$USER', self.task.user.name)
# Add the bind
if not binds:
binds = '-B {}:{}'.format(expanded_base_path, expanded_bind_path)
else:
binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)
run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, computing_user, computing_host)
run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp && mkdir -p /tmp/{}_data/home && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'wget {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export BASE_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
......@@ -306,8 +322,8 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
logger.debug('Starting a remote task "{}"'.format(self.computing))
# Get computing host
host = self.computing.conf.get('host')
user = self.computing.conf.get('user')
computing_host = self.computing.conf.get('host')
computing_user = self.computing.conf.get('user')
# Get user keys
if self.computing.requires_user_keys:
......@@ -350,24 +366,40 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
if not task.requires_proxy_auth and task.password:
authstring = ' export SINGULARITYENV_AUTH_PASS={} && '.format(task.password)
# Set binds, only from sys config if the resource is not owned by the user
if self.computing.user != task.user:
binds = self.computing.sys_conf.get('binds')
else:
binds = self.computing.conf.get('binds')
if not binds:
binds = ''
else:
binds = '-B {}'.format(binds)
# Manage task extra binds
if task.extra_binds:
if not binds:
binds = '-B {}'.format(task.extra_binds)
else:
binds += ',{}'.format(task.extra_binds)
run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
# Handle storages (binds)
binds = ''
storages = Storage.objects.filter(computing=self.computing)
for storage in storages:
if storage.type == 'generic_posix' and storage.bind_path:
# Expand the base path
expanded_base_path = storage.base_path
if '$SSH_USER' in expanded_base_path:
if storage.access_through_computing:
self.computing.attach_user_conf(self.computing.user)
expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user)
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_base_path:
expanded_base_path = expanded_base_path.replace('$USER', self.task.user.name)
# Expand the bind_path
expanded_bind_path = storage.bind_path
if '$SSH_USER' in expanded_bind_path:
if storage.access_through_computing:
expanded_bind_path = expanded_bind_path.replace('$SSH_USER', computing_user)
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_bind_path:
expanded_bind_path = expanded_bind_path.replace('$USER', self.task.user.name)
# Add the bind
if not binds:
binds = '-B {}:{}'.format(expanded_base_path, expanded_bind_path)
else:
binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)
run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, computing_user, computing_host)
run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT && {} '.format(authstring)
run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
......
services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py
+ 53
− 1
View file @ bdfe2f7c
from django.core.management.base import BaseCommand
from django.contrib.auth.models import User
from ...models import Profile, Container, Computing, ComputingSysConf, ComputingUserConf, KeyPair, Text
from ...models import Profile, Container, Computing, ComputingSysConf, ComputingUserConf, Storage, KeyPair, Text
class Command(BaseCommand):
help = 'Adds the admin superuser with \'a\' password.'
......@@ -233,3 +233,55 @@ class Command(BaseCommand):
data = {'user': 'slurmtestuser'})
#=====================
# Storages
#=====================
storages = Storage.objects.all()
if storages:
print('Not creating demo storages as they already exist')
else:
print('Creating demo storages...')
# Get demo computing resources
demo_computing_resources = []
try:
demo_slurm_computing = Computing.objects.get(name='Demo Cluster')
demo_computing_resources.append(demo_slurm_computing)
except:
pass
try:
demo_standalone_computing = Computing.objects.get(name='Demo Standalone')
demo_computing_resources.append(demo_standalone_computing)
except:
pass
for computing in demo_computing_resources:
# Demo shared computing plus conf
Storage.objects.create(user = None,
computing = computing,
access_through_computing = True,
name = 'Shared',
type = 'generic_posix',
access_mode = 'ssh+cli',
auth_mode = 'user_keys',
base_path = '/shared/data/shared',
bind_path = '/storages/shared')
# Demo shared computing plus conf
Storage.objects.create(user = None,
computing = computing,
access_through_computing = True,
name = 'Personal',
type = 'generic_posix',
access_mode = 'ssh+cli',
auth_mode = 'user_keys',
base_path = '/shared/data/users/$SSH_USER',
bind_path = '/storages/personal')
services/webapp/code/rosetta/core_app/migrations/0011_storage.py 0 → 100644
+ 37
− 0
View file @ bdfe2f7c
# Generated by Django 2.2.1 on 2021-11-08 14:45
from django.conf import settings
import django.contrib.postgres.fields.jsonb
from django.db import migrations, models
import django.db.models.deletion
import uuid
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('core_app', '0010_profile_is_power_user'),
]
operations = [
migrations.CreateModel(
name='Storage',
fields=[
('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('name', models.CharField(max_length=255, verbose_name='Name')),
('type', models.CharField(max_length=255, verbose_name='Type')),
('access_mode', models.CharField(max_length=36, verbose_name='Access (control) mode')),
('auth_mode', models.CharField(max_length=36, verbose_name='Auth mode')),
('base_path', models.CharField(max_length=4096, verbose_name='Base path')),
('bind_path', models.CharField(max_length=4096, verbose_name='Bind path')),
('access_through_computing', models.BooleanField(default=False, verbose_name='Access through linked computing resource?')),
('config', django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True)),
('computing', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='storages', to='core_app.Computing')),
('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='+', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['name'],
},
),
]
services/webapp/code/rosetta/core_app/models.py
+ 52
− 0
View file @ bdfe2f7c
......@@ -381,6 +381,58 @@ class Task(models.Model):
return get_task_tunnel_host()
#=========================
# Storages
#=========================
class Storage(models.Model):
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, blank=True, null=True)
name = models.CharField('Name', max_length=255, blank=False, null=False)
#description = models.TextField('Description', blank=True, null=True)
# Storage type
type = models.CharField('Type', max_length=255, blank=False, null=False)
# Access and auth mode
access_mode = models.CharField('Access (control) mode', max_length=36, blank=False, null=False)
auth_mode = models.CharField('Auth mode', max_length=36, blank=False, null=False)
# Paths
base_path = models.CharField('Base path', max_length=4096, blank=False, null=False)
bind_path = models.CharField('Bind path', max_length=4096, blank=False, null=False)
# Link with a computing resource
computing = models.ForeignKey(Computing, related_name='storages', on_delete=models.CASCADE, blank=True, null=True) # Make optional?
access_through_computing = models.BooleanField('Access through linked computing resource?', default=False)
# If the above is linked, some configuration can be taken from the linked computing resource (i.e. the hostname)
# Configuration
config = JSONField(blank=True, null=True)
class Meta:
ordering = ['name']
def __str__(self):
if self.user:
return str('Storage "{}" of user "{}"'.format(self.id, self.user))
else:
return str('Storage "{}"'.format(self.id))
@property
def id(self):
return (self.name if not self.computing else '{}@{}'.format(self.name,self.computing.name))
#=========================
# KeyPair
#=========================
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment