Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • exact/Rosetta
1 result
Select Git revision
Show changes
Commits on Source (2)
...@@ -13,7 +13,7 @@ from rest_framework import status, serializers, viewsets ...@@ -13,7 +13,7 @@ from rest_framework import status, serializers, viewsets
from rest_framework.views import APIView from rest_framework.views import APIView
from .utils import format_exception, send_email, os_shell, now_t, get_ssh_access_mode_credentials, get_or_create_container_from_repository, booleanize from .utils import format_exception, send_email, os_shell, now_t, get_ssh_access_mode_credentials, get_or_create_container_from_repository, booleanize
from .models import Profile, Task, TaskStatuses, Computing, Storage, KeyPair from .models import Profile, Task, TaskStatuses, Computing, Storage, KeyPair
from .exceptions import ConsistencyException from .exceptions import PermissionDenied
import json import json
# Setup logging # Setup logging
...@@ -153,8 +153,12 @@ class PrivatePOSTAPI(APIView): ...@@ -153,8 +153,12 @@ class PrivatePOSTAPI(APIView):
# Call API logic # Call API logic
return self._post(request) return self._post(request)
except Exception as e: except Exception as e:
logger.error(format_exception(e)) # TODO: refactor me
return error500('Got error in processing request: {}'.format(e)) if isinstance(e, PermissionDenied):
return error400(format(e))
else:
logger.error(format_exception(e))
return error500('Got error in processing request: {}'.format(e))
class PrivateGETAPI(APIView): class PrivateGETAPI(APIView):
'''Base private GET API class''' '''Base private GET API class'''
...@@ -174,9 +178,12 @@ class PrivateGETAPI(APIView): ...@@ -174,9 +178,12 @@ class PrivateGETAPI(APIView):
# Call API logic # Call API logic
return self._get(request) return self._get(request)
except Exception as e: except Exception as e:
logger.error(format_exception(e)) # TODO: refactor me
return error500('Got error in processing request: {}'.format(e)) if isinstance(e, PermissionDenied):
return error400(format(e))
else:
logger.error(format_exception(e))
return error500('Got error in processing request: {}'.format(e))
#============================== #==============================
...@@ -689,6 +696,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -689,6 +696,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
def delete(self, path, user, storage): def delete(self, path, user, storage):
if storage.read_only:
raise PermissionDenied('This storage is read-only')
if storage.type == 'generic_posix': if storage.type == 'generic_posix':
shell_path = self.sanitize_and_prepare_shell_path(path, user, storage) shell_path = self.sanitize_and_prepare_shell_path(path, user, storage)
...@@ -707,6 +717,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -707,6 +717,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
def mkdir(self, path, user, storage, force=False): def mkdir(self, path, user, storage, force=False):
if storage.read_only:
raise PermissionDenied('This storage is read-only')
path = self.sanitize_and_prepare_shell_path(path, user, storage) path = self.sanitize_and_prepare_shell_path(path, user, storage)
if storage.type == 'generic_posix': if storage.type == 'generic_posix':
...@@ -751,6 +764,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -751,6 +764,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
def rename(self, old, new, user, storage): def rename(self, old, new, user, storage):
if storage.read_only:
raise PermissionDenied('This storage is read-only')
if storage.type == 'generic_posix': if storage.type == 'generic_posix':
old = self.sanitize_and_prepare_shell_path(old, user, storage) old = self.sanitize_and_prepare_shell_path(old, user, storage)
...@@ -772,6 +788,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -772,6 +788,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
def copy(self, source, target, user, storage): def copy(self, source, target, user, storage):
if storage.read_only:
raise PermissionDenied('This storage is read-only')
if storage.type == 'generic_posix': if storage.type == 'generic_posix':
source = self.sanitize_and_prepare_shell_path(source, user, storage) source = self.sanitize_and_prepare_shell_path(source, user, storage)
...@@ -805,6 +824,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -805,6 +824,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
def scp_to(self, source, target, user, storage, mode='get'): def scp_to(self, source, target, user, storage, mode='get'):
if storage.read_only:
raise PermissionDenied('This storage is read-only')
source = self.sanitize_shell_path(source) # This is a folder on Rosetta (/tmp) source = self.sanitize_shell_path(source) # This is a folder on Rosetta (/tmp)
target = self.sanitize_and_prepare_shell_path(target, user, storage) target = self.sanitize_and_prepare_shell_path(target, user, storage)
......
...@@ -147,11 +147,17 @@ class InternalStandaloneComputingManager(StandaloneComputingManager): ...@@ -147,11 +147,17 @@ class InternalStandaloneComputingManager(StandaloneComputingManager):
if '$USER' in expanded_bind_path: if '$USER' in expanded_bind_path:
expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username) expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username)
# Read only?
if storage.read_only:
mode_string = ':ro'
else:
mode_string = ''
# Add the bind # Add the bind
if not binds: if not binds:
binds = '-v{}:{}'.format(expanded_base_path, expanded_bind_path) binds = '-v{}:{}{}'.format(expanded_base_path, expanded_bind_path, mode_string)
else: else:
binds += ' -v{}:{}'.format(expanded_base_path, expanded_bind_path) binds += ' -v{}:{}{}'.format(expanded_base_path, expanded_bind_path, mode_string)
# Host name, image entry command # Host name, image entry command
run_command += ' {} -h task-{} --name task-{} -d -t {}/{}:{}'.format(binds, task.short_uuid, task.short_uuid, task.container.registry, task.container.image_name, task.container.image_tag) run_command += ' {} -h task-{} --name task-{} -d -t {}/{}:{}'.format(binds, task.short_uuid, task.short_uuid, task.container.registry, task.container.image_name, task.container.image_tag)
...@@ -322,6 +328,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -322,6 +328,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
# Handle storages (binds) # Handle storages (binds)
binds = '' binds = ''
storages = Storage.objects.filter(computing=self.computing) storages = Storage.objects.filter(computing=self.computing)
initialize_bind_paths_command = ''
for storage in storages: for storage in storages:
if storage.type == 'generic_posix' and storage.bind_path: if storage.type == 'generic_posix' and storage.bind_path:
...@@ -335,6 +342,8 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -335,6 +342,8 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
if '$USER' in expanded_base_path: if '$USER' in expanded_base_path:
expanded_base_path = expanded_base_path.replace('$USER', task.user.username) expanded_base_path = expanded_base_path.replace('$USER', task.user.username)
initialize_bind_paths_command += 'mkdir -p {} && '.format(expanded_base_path)
# Expand the bind_path # Expand the bind_path
expanded_bind_path = storage.bind_path expanded_bind_path = storage.bind_path
if '$SSH_USER' in expanded_bind_path: if '$SSH_USER' in expanded_bind_path:
...@@ -345,17 +354,23 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -345,17 +354,23 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
if '$USER' in expanded_bind_path: if '$USER' in expanded_bind_path:
expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username) expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username)
# Read only?
if storage.read_only:
mode_string = ':ro'
else:
mode_string = ''
# Add the bind # Add the bind
if not binds: if not binds:
binds = '-v{}:{}'.format(expanded_base_path, expanded_bind_path) binds = '-v{}:{}{}'.format(expanded_base_path, expanded_bind_path, mode_string)
else: else:
binds += ' -v{}:{}'.format(expanded_base_path, expanded_bind_path) binds += ' -v{}:{}{}'.format(expanded_base_path, expanded_bind_path, mode_string)
# TODO: remove this hardcoding # TODO: remove this hardcoding
prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_engine=='docker') else '' prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_engine=='docker') else ''
run_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_port, computing_keys.private_key_file, computing_user, computing_host) run_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_port, computing_keys.private_key_file, computing_user, computing_host)
run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir /tmp/{}_data && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid) run_command += '/bin/bash -c \'"{}rm -rf /tmp/{}_data && mkdir /tmp/{}_data && chmod 700 /tmp/{}_data && '.format(initialize_bind_paths_command, task.uuid, task.uuid, task.uuid)
run_command += 'curl {} {}/api/v1/base/agent/?task_uuid={} -o /tmp/{}_data/agent.py &> /dev/null && export TASK_PORT=\$(python3 /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_CURL_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid) run_command += 'curl {} {}/api/v1/base/agent/?task_uuid={} -o /tmp/{}_data/agent.py &> /dev/null && export TASK_PORT=\$(python3 /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_CURL_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'exec nohup {} {} run -p \$TASK_PORT:{} {} {} {} '.format(prefix, container_engine, task.container.interface_port, authstring, varsstring, binds) run_command += 'exec nohup {} {} run -p \$TASK_PORT:{} {} {} {} '.format(prefix, container_engine, task.container.interface_port, authstring, varsstring, binds)
if container_engine == 'podman': if container_engine == 'podman':
......
...@@ -4,3 +4,6 @@ class ErrorMessage(Exception): ...@@ -4,3 +4,6 @@ class ErrorMessage(Exception):
class ConsistencyException(Exception): class ConsistencyException(Exception):
pass pass
class PermissionDenied(Exception):
pass
\ No newline at end of file
# Generated by Django 2.2.1 on 2025-03-05 09:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core_app', '0036_container_disable_http_basicauth_embedding'),
]
operations = [
migrations.AddField(
model_name='storage',
name='read_only',
field=models.BooleanField(default=False, verbose_name='Read only? (if supported)'),
),
]
...@@ -387,6 +387,9 @@ class Storage(models.Model): ...@@ -387,6 +387,9 @@ class Storage(models.Model):
base_path = models.CharField('Base path', max_length=4096, blank=False, null=False) base_path = models.CharField('Base path', max_length=4096, blank=False, null=False)
bind_path = models.CharField('Bind path', max_length=4096, blank=True, null=True) bind_path = models.CharField('Bind path', max_length=4096, blank=True, null=True)
# Read only?
read_only = models.BooleanField('Read only? (if supported)', default=False)
# Link with a computing resource # Link with a computing resource
computing = models.ForeignKey(Computing, related_name='storages', on_delete=models.CASCADE, blank=True, null=True) # Make optional? computing = models.ForeignKey(Computing, related_name='storages', on_delete=models.CASCADE, blank=True, null=True) # Make optional?
access_through_computing = models.BooleanField('Access through linked computing resource?', default=False) access_through_computing = models.BooleanField('Access through linked computing resource?', default=False)
......