Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • exact/Rosetta
1 result
Select Git revision
Show changes
Commits on Source (5)
Showing
with 630 additions and 528 deletions
...@@ -12,7 +12,7 @@ if [[ "x$1" == "xnocache" ]] ; then ...@@ -12,7 +12,7 @@ if [[ "x$1" == "xnocache" ]] ; then
NOCACHE=true NOCACHE=true
SERVICE="" SERVICE=""
elif [[ "x$2" == "xnocache" ]] ; then elif [[ "x$2" == "xnocache" ]] ; then
NOCACHE=true NOCACHE=true
SERVICE=$1 SERVICE=$1
else else
if [[ "x$NOCACHE" == "x" ]] ; then if [[ "x$NOCACHE" == "x" ]] ; then
...@@ -29,20 +29,20 @@ else ...@@ -29,20 +29,20 @@ else
fi fi
if [[ "x$SERVICE" == "x" ]] ; then if [[ "x$SERVICE" == "x" ]] ; then
# Build all services # Build all services
NOCACHE=$NOCACHE rosetta/build base_ubuntu18.04 NOCACHE=$NOCACHE rosetta/build base_ubuntu18.04
NOCACHE=$NOCACHE rosetta/build base_ubuntu22.04 NOCACHE=$NOCACHE rosetta/build base_ubuntu22.04
NOCACHE=$NOCACHE rosetta/build slurmbase NOCACHE=$NOCACHE rosetta/build slurmbase
NOCACHE=$NOCACHE rosetta/build slurmcluster NOCACHE=$NOCACHE rosetta/build slurmcluster
NOCACHE=$NOCACHE rosetta/build slurmclustermaster NOCACHE=$NOCACHE rosetta/build slurmclustermaster
NOCACHE=$NOCACHE rosetta/build slurmclusterworker NOCACHE=$NOCACHE rosetta/build slurmclusterworker
NOCACHE=$NOCACHE rosetta/build standaloneworker NOCACHE=$NOCACHE rosetta/build standaloneworker
NOCACHE=$NOCACHE rosetta/build dregistry NOCACHE=$NOCACHE rosetta/build dregistry
NOCACHE=$NOCACHE rosetta/build webapp NOCACHE=$NOCACHE rosetta/build webapp
NOCACHE=$NOCACHE rosetta/build postgres NOCACHE=$NOCACHE rosetta/build postgres
NOCACHE=$NOCACHE rosetta/build proxy NOCACHE=$NOCACHE rosetta/build proxy
else else
# Build a specific image # Build a specific image
......
...@@ -14,7 +14,7 @@ fi ...@@ -14,7 +14,7 @@ fi
if [[ "x$2" != "x" ]] ; then if [[ "x$2" != "x" ]] ; then
tail -f -n 1000 data/$1/log/$2.log tail -f -n 1000 data/$1/log/$2.log
else else
docker-compose logs -f $1 docker-compose logs -f $1
fi fi
...@@ -14,7 +14,7 @@ elif [[ $# -gt 2 ]] ; then ...@@ -14,7 +14,7 @@ elif [[ $# -gt 2 ]] ; then
echo "Use double quotes to wrap commands with spaces" echo "Use double quotes to wrap commands with spaces"
exit 1 exit 1
else else
COMMAND=$2 COMMAND=$2
if [[ "x$COMMAND" == "x" ]] ; then if [[ "x$COMMAND" == "x" ]] ; then
echo "" echo ""
......
...@@ -10,16 +10,16 @@ if [[ $# -eq 0 ]] ; then ...@@ -10,16 +10,16 @@ if [[ $# -eq 0 ]] ; then
declare -a container_names declare -a container_names
OUT=$(rosetta/ps) OUT=$(rosetta/ps)
while read -r line; do while read -r line; do
if [[ $line == *"Up"* ]]; then if [[ $line == *"Up"* ]]; then
container_name=$(echo $line | cut -d ' ' -f1) container_name=$(echo $line | cut -d ' ' -f1)
container_names+=($container_name); container_names+=($container_name);
fi fi
done <<< "$OUT" done <<< "$OUT"
for container_name in ${container_names[@]} for container_name in ${container_names[@]}
do do
echo "" echo ""
...@@ -27,7 +27,7 @@ if [[ $# -eq 0 ]] ; then ...@@ -27,7 +27,7 @@ if [[ $# -eq 0 ]] ; then
docker-compose exec $container_name /bin/bash -c "supervisorctl status" docker-compose exec $container_name /bin/bash -c "supervisorctl status"
done done
echo "" echo ""
else else
docker-compose exec $@ /bin/bash -c "supervisorctl status" docker-compose exec $@ /bin/bash -c "supervisorctl status"
fi fi
...@@ -9,9 +9,9 @@ logger = logging.getLogger(__name__) ...@@ -9,9 +9,9 @@ logger = logging.getLogger(__name__)
class RosettaOIDCAuthenticationBackend(OIDCAuthenticationBackend): class RosettaOIDCAuthenticationBackend(OIDCAuthenticationBackend):
def create_user(self, claims): def create_user(self, claims):
# Call parent user creation function # Call parent user creation function
user = super(RosettaOIDCAuthenticationBackend, self).create_user(claims) user = super(RosettaOIDCAuthenticationBackend, self).create_user(claims)
...@@ -27,12 +27,12 @@ class RosettaOIDCAuthenticationBackend(OIDCAuthenticationBackend): ...@@ -27,12 +27,12 @@ class RosettaOIDCAuthenticationBackend(OIDCAuthenticationBackend):
class RosettaOIDCAuthenticationCallbackView(OIDCAuthenticationCallbackView): class RosettaOIDCAuthenticationCallbackView(OIDCAuthenticationCallbackView):
def login_success(self): def login_success(self):
# Call parent login_success but do not return # Call parent login_success but do not return
super(RosettaOIDCAuthenticationCallbackView, self).login_success() super(RosettaOIDCAuthenticationCallbackView, self).login_success()
logger.debug('Trying to get cookie-based post login redirect') logger.debug('Trying to get cookie-based post login redirect')
post_login_page = self.request.COOKIES.get('post_login_redirect') post_login_page = self.request.COOKIES.get('post_login_redirect')
if post_login_page: if post_login_page:
...@@ -44,3 +44,4 @@ class RosettaOIDCAuthenticationCallbackView(OIDCAuthenticationCallbackView): ...@@ -44,3 +44,4 @@ class RosettaOIDCAuthenticationCallbackView(OIDCAuthenticationCallbackView):
logger.debug('No cookie-based post login redirect found, redirecting to "%s"', self.success_url) logger.debug('No cookie-based post login redirect found, redirecting to "%s"', self.success_url)
return HttpResponseRedirect(self.success_url) return HttpResponseRedirect(self.success_url)
...@@ -2,7 +2,7 @@ import os ...@@ -2,7 +2,7 @@ import os
from django.conf import settings from django.conf import settings
def export_vars(request): def export_vars(request):
data = {} data = {}
# Set open id connect enabled or not # Set open id connect enabled or not
if settings.OIDC_RP_CLIENT_ID: if settings.OIDC_RP_CLIENT_ID:
data['OPENID_ENABLED'] = True data['OPENID_ENABLED'] = True
...@@ -19,6 +19,6 @@ def export_vars(request): ...@@ -19,6 +19,6 @@ def export_vars(request):
if settings.INVITATION_CODE: if settings.INVITATION_CODE:
data['INVITATION_CODE_ENABLED'] = True data['INVITATION_CODE_ENABLED'] = True
else: else:
data['INVITATION_CODE_ENABLED'] = False data['INVITATION_CODE_ENABLED'] = False
return data return data
\ No newline at end of file
...@@ -15,7 +15,7 @@ from .utils import format_exception, send_email, os_shell, now_t, get_ssh_access ...@@ -15,7 +15,7 @@ from .utils import format_exception, send_email, os_shell, now_t, get_ssh_access
from .models import Profile, Task, TaskStatuses, Computing, Storage, KeyPair from .models import Profile, Task, TaskStatuses, Computing, Storage, KeyPair
from .exceptions import ConsistencyException from .exceptions import ConsistencyException
import json import json
# Setup logging # Setup logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -24,33 +24,33 @@ ROSETTA_AGENT_CHECK_SSL = booleanize(os.environ.get('ROSETTA_AGENT_CHECK_SSL', T ...@@ -24,33 +24,33 @@ ROSETTA_AGENT_CHECK_SSL = booleanize(os.environ.get('ROSETTA_AGENT_CHECK_SSL', T
#============================== #==============================
# Common returns # Common returns
#============================== #==============================
# Ok (with data) # Ok (with data)
def ok200(data=None): def ok200(data=None):
return Response({"results": data}, status=status.HTTP_200_OK) return Response({"results": data}, status=status.HTTP_200_OK)
# Error 400 # Error 400
def error400(data=None): def error400(data=None):
return Response({"detail": data}, status=status.HTTP_400_BAD_REQUEST) return Response({"detail": data}, status=status.HTTP_400_BAD_REQUEST)
# Error 401 # Error 401
def error401(data=None): def error401(data=None):
return Response({"detail": data}, status=status.HTTP_401_UNAUTHORIZED) return Response({"detail": data}, status=status.HTTP_401_UNAUTHORIZED)
# Error 404 # Error 404
def error404(data=None): def error404(data=None):
return Response({"detail": data}, status=status.HTTP_404_NOT_FOUND) return Response({"detail": data}, status=status.HTTP_404_NOT_FOUND)
# Error 500 # Error 500
def error500(data=None): def error500(data=None):
return Response({"detail": data}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) return Response({"detail": data}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
#============================== #==============================
# Authentication helper # Authentication helper
#============================== #==============================
def rosetta_authenticate(request): def rosetta_authenticate(request):
# Get data # Get data
...@@ -65,22 +65,22 @@ def rosetta_authenticate(request): ...@@ -65,22 +65,22 @@ def rosetta_authenticate(request):
# Try username/password authentication # Try username/password authentication
elif username or password: elif username or password:
# Check we got both # Check we got both
if not username: if not username:
return error400('Got empty username') return error400('Got empty username')
if not password: if not password:
return error400('Got empty password') return error400('Got empty password')
# Authenticate # Authenticate
user = authenticate(username=username, password=password) user = authenticate(username=username, password=password)
if not user: if not user:
return error401('Wrong username/password') return error401('Wrong username/password')
else: else:
login(request, user) login(request, user)
return user return user
# Try auth toekn authentication # Try auth toekn authentication
elif authtoken: elif authtoken:
try: try:
profile = Profile.objects.get(authtoken=authtoken) profile = Profile.objects.get(authtoken=authtoken)
...@@ -96,7 +96,7 @@ def rosetta_authenticate(request): ...@@ -96,7 +96,7 @@ def rosetta_authenticate(request):
# CSRF exempt auth class # CSRF exempt auth class
#============================== #==============================
from rest_framework.authentication import SessionAuthentication, BasicAuthentication from rest_framework.authentication import SessionAuthentication, BasicAuthentication
class CsrfExemptSessionAuthentication(SessionAuthentication): class CsrfExemptSessionAuthentication(SessionAuthentication):
...@@ -107,10 +107,10 @@ class CsrfExemptSessionAuthentication(SessionAuthentication): ...@@ -107,10 +107,10 @@ class CsrfExemptSessionAuthentication(SessionAuthentication):
#============================== #==============================
# Base public API class # Base public API class
#============================== #==============================
class PublicPOSTAPI(APIView): class PublicPOSTAPI(APIView):
'''Base public POST API class''' '''Base public POST API class'''
# POST # POST
def post(self, request): def post(self, request):
try: try:
...@@ -118,9 +118,9 @@ class PublicPOSTAPI(APIView): ...@@ -118,9 +118,9 @@ class PublicPOSTAPI(APIView):
except Exception as e: except Exception as e:
logger.error(format_exception(e)) logger.error(format_exception(e))
return error500('Got error in processing request: {}'.format(e)) return error500('Got error in processing request: {}'.format(e))
class PublicGETAPI(APIView): class PublicGETAPI(APIView):
'''Base public GET API class''' '''Base public GET API class'''
# GET # GET
def get(self, request): def get(self, request):
try: try:
...@@ -134,43 +134,43 @@ class PublicGETAPI(APIView): ...@@ -134,43 +134,43 @@ class PublicGETAPI(APIView):
#============================== #==============================
# Base private API class # Base private API class
#============================== #==============================
class PrivatePOSTAPI(APIView): class PrivatePOSTAPI(APIView):
'''Base private POST API class''' '''Base private POST API class'''
# POST # POST
def post(self, request): def post(self, request):
try: try:
# Authenticate using rosetta authentication # Authenticate using rosetta authentication
response = rosetta_authenticate(request) response = rosetta_authenticate(request)
# If we got a response return it, otherwise set it as the user. # If we got a response return it, otherwise set it as the user.
if isinstance(response, Response): if isinstance(response, Response):
return response return response
else: else:
self.user = response self.user = response
# Call API logic # Call API logic
return self._post(request) return self._post(request)
except Exception as e: except Exception as e:
logger.error(format_exception(e)) logger.error(format_exception(e))
return error500('Got error in processing request: {}'.format(e)) return error500('Got error in processing request: {}'.format(e))
class PrivateGETAPI(APIView): class PrivateGETAPI(APIView):
'''Base private GET API class''' '''Base private GET API class'''
# GET # GET
def get(self, request): def get(self, request):
try: try:
# Authenticate using rosetta authentication # Authenticate using rosetta authentication
response = rosetta_authenticate(request) response = rosetta_authenticate(request)
# If we got a response return it, otherwise set it as the user. # If we got a response return it, otherwise set it as the user.
if isinstance(response, Response): if isinstance(response, Response):
return response return response
else: else:
self.user = response self.user = response
# Call API logic # Call API logic
return self._get(request) return self._get(request)
except Exception as e: except Exception as e:
...@@ -191,29 +191,29 @@ class login_api(PrivateGETAPI, PrivatePOSTAPI): ...@@ -191,29 +191,29 @@ class login_api(PrivateGETAPI, PrivatePOSTAPI):
post: post:
Authorize and returns the auth token. Authorize and returns the auth token.
""" """
def _post(self, request): def _post(self, request):
return ok200({'authtoken': self.user.profile.authtoken}) return ok200({'authtoken': self.user.profile.authtoken})
def _get(self, request): def _get(self, request):
return ok200({'authtoken': self.user.profile.authtoken}) return ok200({'authtoken': self.user.profile.authtoken})
class logout_api(PrivateGETAPI): class logout_api(PrivateGETAPI):
""" """
get: get:
Logout the user Logout the user
""" """
def _get(self, request): def _get(self, request):
logout(request) logout(request)
return ok200() return ok200()
class agent_api(PublicGETAPI): class agent_api(PublicGETAPI):
def _get(self, request): def _get(self, request):
task_uuid = request.GET.get('task_uuid', None) task_uuid = request.GET.get('task_uuid', None)
if not task_uuid: if not task_uuid:
return HttpResponse('MISSING task_uuid') return HttpResponse('MISSING task_uuid')
...@@ -228,9 +228,9 @@ class agent_api(PublicGETAPI): ...@@ -228,9 +228,9 @@ class agent_api(PublicGETAPI):
from.utils import get_webapp_conn_string from.utils import get_webapp_conn_string
webapp_conn_string = get_webapp_conn_string() webapp_conn_string = get_webapp_conn_string()
action = request.GET.get('action', None) action = request.GET.get('action', None)
if not action: if not action:
# Return the agent code # Return the agent code
agent_code=''' agent_code='''
...@@ -290,7 +290,7 @@ if not check_ssl: ...@@ -290,7 +290,7 @@ if not check_ssl:
else: else:
response = urlopen("'''+webapp_conn_string+'''/api/v1/base/agent/?task_uuid={}&action=set_ip_port&ip={}&port={}".format(task_uuid, ip, port)) response = urlopen("'''+webapp_conn_string+'''/api/v1/base/agent/?task_uuid={}&action=set_ip_port&ip={}&port={}".format(task_uuid, ip, port))
response_content = response.read() response_content = response.read()
if response_content not in ['OK', b'OK']: if response_content not in ['OK', b'OK']:
logger.error(response_content) logger.error(response_content)
...@@ -301,37 +301,37 @@ else: ...@@ -301,37 +301,37 @@ else:
logger.info('Everything OK') logger.info('Everything OK')
print(port) print(port)
''' '''
return HttpResponse(agent_code) return HttpResponse(agent_code)
elif action=='set_ip_port': elif action=='set_ip_port':
task_interface_ip = request.GET.get('ip', None) task_interface_ip = request.GET.get('ip', None)
if not task_interface_ip: if not task_interface_ip:
return HttpResponse('IP not valid (got "{}")'.format(task_interface_ip)) return HttpResponse('IP not valid (got "{}")'.format(task_interface_ip))
task_interface_port = request.GET.get('port', None) task_interface_port = request.GET.get('port', None)
if not task_interface_port: if not task_interface_port:
return HttpResponse('Port not valid (got "{}")'.format(task_interface_port)) return HttpResponse('Port not valid (got "{}")'.format(task_interface_port))
try: try:
int(task_interface_port) int(task_interface_port)
except (TypeError, ValueError): except (TypeError, ValueError):
return HttpResponse('Port not valid (got "{}")'.format(task_interface_port)) return HttpResponse('Port not valid (got "{}")'.format(task_interface_port))
# Set fields # Set fields
logger.info('Agent API setting task "{}" to ip "{}" and port "{}"'.format(task.uuid, task_interface_ip, task_interface_port)) logger.info('Agent API setting task "{}" to ip "{}" and port "{}"'.format(task.uuid, task_interface_ip, task_interface_port))
task.status = TaskStatuses.running task.status = TaskStatuses.running
task.interface_ip = task_interface_ip task.interface_ip = task_interface_ip
# Get container engine # Get container engine
container_engine = None container_engine = None
if task.computing_options: if task.computing_options:
container_engine = task.computing_options.get('container_engine', None) container_engine = task.computing_options.get('container_engine', None)
if not container_engine: if not container_engine:
container_engine = task.computing.default_container_engine container_engine = task.computing.default_container_engine
if container_engine=='singularity': if container_engine=='singularity':
# For Singularity, set this only if the container supports custom # For Singularity, set this only if the container supports custom
# interface ports. Otherwise, use the task container interface port. # interface ports. Otherwise, use the task container interface port.
...@@ -342,10 +342,10 @@ print(port) ...@@ -342,10 +342,10 @@ print(port)
else: else:
# For all other container engines, set it in any case # For all other container engines, set it in any case
task.interface_port = int(task_interface_port) task.interface_port = int(task_interface_port)
# Save the task # Save the task
task.save() task.save()
# Notify the user that the task called back home if using a WMS # Notify the user that the task called back home if using a WMS
if task.computing.wms: if task.computing.wms:
if settings.DJANGO_EMAIL_APIKEY: if settings.DJANGO_EMAIL_APIKEY:
...@@ -357,7 +357,7 @@ print(port) ...@@ -357,7 +357,7 @@ print(port)
except Exception as e: except Exception as e:
logger.error('Cannot send task ready email: "{}"'.format(e)) logger.error('Cannot send task ready email: "{}"'.format(e))
return HttpResponse('OK') return HttpResponse('OK')
else: else:
return HttpResponse('Unknown action "{}"'.format(action)) return HttpResponse('Unknown action "{}"'.format(action))
...@@ -375,7 +375,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -375,7 +375,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
post: post:
Perform actions or upload files. Perform actions or upload files.
""" """
# The RichFilemanager has no CSRF support... # The RichFilemanager has no CSRF support...
authentication_classes = (CsrfExemptSessionAuthentication, BasicAuthentication) authentication_classes = (CsrfExemptSessionAuthentication, BasicAuthentication)
...@@ -384,12 +384,12 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -384,12 +384,12 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
# Prepare paths for scp. They have been already made shell-ready, but we need to triple-escape # Prepare paths for scp. They have been already made shell-ready, but we need to triple-escape
# spaces on remote source or destination: My\ Folder must become My\\\ Folder. # spaces on remote source or destination: My\ Folder must become My\\\ Folder.
if mode=='get': if mode=='get':
source = source.replace('\ ', '\\\\\\ ') source = source.replace('\ ', '\\\\\\ ')
else: else:
dest = dest.replace('\ ', '\\\\\\ ') dest = dest.replace('\ ', '\\\\\\ ')
# Get credentials # Get credentials
computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(computing, user) computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(computing, user)
...@@ -410,15 +410,15 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -410,15 +410,15 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
if storage.access_through_computing: if storage.access_through_computing:
# Get credentials # Get credentials
computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(storage.computing, user) computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(storage.computing, user)
# Command # Command
command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} "{}"'.format(computing_port, computing_keys.private_key_file, computing_user, computing_host, command) command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} "{}"'.format(computing_port, computing_keys.private_key_file, computing_user, computing_host, command)
else: else:
raise NotImplementedError('Not accessing through computing is not implemented for storage type "{}"'.format(storage.type)) raise NotImplementedError('Not accessing through computing is not implemented for storage type "{}"'.format(storage.type))
elif storage.access_mode == 'cli': elif storage.access_mode == 'cli':
try: try:
as_user = storage.conf['as_user'] as_user = storage.conf['as_user']
# Is "as_user" a UID? # Is "as_user" a UID?
try: try:
uid = int(as_user) uid = int(as_user)
...@@ -440,8 +440,8 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -440,8 +440,8 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
raise Exception(out.sterr) raise Exception(out.sterr)
as_user = 'user_' + str(uid) as_user = 'user_' + str(uid)
else: else:
as_user = out.stdout.strip() as_user = out.stdout.strip()
except (KeyError, TypeError): except (KeyError, TypeError):
as_user = None as_user = None
if as_user: if as_user:
...@@ -449,8 +449,8 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -449,8 +449,8 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
else: else:
command = '/bin/bash -c "{}"'.format(command) command = '/bin/bash -c "{}"'.format(command)
else: else:
raise NotImplementedError('Access mode "{}" not implemented for storage type "{}"'.format(storage.access_mode, storage.type)) raise NotImplementedError('Access mode "{}" not implemented for storage type "{}"'.format(storage.access_mode, storage.type))
return command return command
@staticmethod @staticmethod
...@@ -468,14 +468,14 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -468,14 +468,14 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
@staticmethod @staticmethod
def sanitize_and_prepare_shell_path(path, user, storage, escapes=True): def sanitize_and_prepare_shell_path(path, user, storage, escapes=True):
if escapes: if escapes:
path = path.replace(' ', '\ ') path = path.replace(' ', '\ ')
cleaner = re.compile('(?:\\\)+') cleaner = re.compile('(?:\\\)+')
path = re.sub(cleaner,r"\\",path) path = re.sub(cleaner,r"\\",path)
# Prepare the base path (expand it with variables substitution) # Prepare the base path (expand it with variables substitution)
base_path_expanded = storage.base_path base_path_expanded = storage.base_path
if '$SSH_USER' in base_path_expanded: if '$SSH_USER' in base_path_expanded:
if storage.access_through_computing: if storage.access_through_computing:
computing = storage.computing computing = storage.computing
...@@ -486,7 +486,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -486,7 +486,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
base_path_expanded = base_path_expanded.replace('$SSH_USER', computing_user) base_path_expanded = base_path_expanded.replace('$SSH_USER', computing_user)
else: else:
base_path_expanded = base_path_expanded.replace('$SSH_USER', computing.conf.get('user')) base_path_expanded = base_path_expanded.replace('$SSH_USER', computing.conf.get('user'))
else: else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented') raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in base_path_expanded: if '$USER' in base_path_expanded:
...@@ -495,7 +495,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -495,7 +495,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
# If the path is not starting with the base path, do it # If the path is not starting with the base path, do it
if not path.startswith(base_path_expanded): if not path.startswith(base_path_expanded):
path = base_path_expanded+'/'+path path = base_path_expanded+'/'+path
return path return path
def get_storage_from_path(self, path, request): def get_storage_from_path(self, path, request):
...@@ -508,10 +508,10 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -508,10 +508,10 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
except IndexError: except IndexError:
storage_name = storage_id storage_name = storage_id
computing_name = None computing_name = None
# Get all the storages this user has access to: # Get all the storages this user has access to:
storages = list(Storage.objects.filter(group=None, name=storage_name)) + list(Storage.objects.filter(group__user=request.user, name=storage_name)) storages = list(Storage.objects.filter(group=None, name=storage_name)) + list(Storage.objects.filter(group__user=request.user, name=storage_name))
# Filter by computing resource name (or None) # Filter by computing resource name (or None)
if computing_name: if computing_name:
unfiltered_storages = storages unfiltered_storages = storages
...@@ -524,8 +524,8 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -524,8 +524,8 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
storages = [] storages = []
for storage in unfiltered_storages: for storage in unfiltered_storages:
if storage.computing is None: if storage.computing is None:
storages.append(storage) storages.append(storage)
# Check that we had at least and no more than one storage in the end # Check that we had at least and no more than one storage in the end
if len(storages) == 0: if len(storages) == 0:
...@@ -537,21 +537,21 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -537,21 +537,21 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
storage = storages[0] storage = storages[0]
return storage return storage
def ls(self, path, user, storage): def ls(self, path, user, storage):
# Data container # Data container
data = [] data = []
if storage.type == 'generic_posix': if storage.type == 'generic_posix':
shell_path = self.sanitize_and_prepare_shell_path(path, user, storage) shell_path = self.sanitize_and_prepare_shell_path(path, user, storage)
# Prepare command # Prepare command
# https://askubuntu.com/questions/1116634/ls-command-show-time-only-in-iso-format # https://askubuntu.com/questions/1116634/ls-command-show-time-only-in-iso-format
# https://www.howtogeek.com/451022/how-to-use-the-stat-command-on-linux/ # https://www.howtogeek.com/451022/how-to-use-the-stat-command-on-linux/
command = self.prepare_sh_command('cd {} && stat --printf=\'%F/%s/%Y/%n\\n\' * .*'.format(shell_path), user, storage) command = self.prepare_sh_command('cd {} && stat --printf=\'%F/%s/%Y/%n\\n\' * .*'.format(shell_path), user, storage)
# Execute_command # Execute_command
...@@ -560,39 +560,39 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -560,39 +560,39 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
# Did we just get a "cannot stat - No such file or directory" (bash) or a "can't cd to" (sh) error? # Did we just get a "cannot stat - No such file or directory" (bash) or a "can't cd to" (sh) error?
if 'No such file or directory' in out.stderr or 'can\'t cd to' in out.stderr : if 'No such file or directory' in out.stderr or 'can\'t cd to' in out.stderr :
# Create the folder if this was the root for the user (storage base path) # Create the folder if this was the root for the user (storage base path)
# Note: if the folder is completely empty, this gets execute as well. # Note: if the folder is completely empty, this gets execute as well.
# TODO: fix me (e.g. check for "cannot stat" or similar) # TODO: fix me (e.g. check for "cannot stat" or similar)
if path == '/': if path == '/':
self.mkdir(self.sanitize_and_prepare_shell_path('/', user, storage), user, storage, force=True) self.mkdir(self.sanitize_and_prepare_shell_path('/', user, storage), user, storage, force=True)
# Return (empty) data # Return (empty) data
return data return data
else: else:
raise Exception(out.stderr) raise Exception(out.stderr)
# Log # Log
#logger.debug('Shell exec output: "{}"'.format(out)) #logger.debug('Shell exec output: "{}"'.format(out))
out_lines = out.stdout.split('\n') out_lines = out.stdout.split('\n')
for line in out_lines: for line in out_lines:
# Example line: directory/My folder/68/1617030350 # Example line: directory/My folder/68/1617030350
# Set name # Set name
line_pieces = line.split('/') line_pieces = line.split('/')
type = line_pieces[0] type = line_pieces[0]
size = line_pieces[1] size = line_pieces[1]
timestamp = line_pieces[2] timestamp = line_pieces[2]
name = line_pieces[3] name = line_pieces[3]
# Define and clean listing path: # Define and clean listing path:
listing_path = '/{}/{}/{}/'.format(storage.id, path, name) listing_path = '/{}/{}/{}/'.format(storage.id, path, name)
listing_path = self.clean_path(listing_path) listing_path = self.clean_path(listing_path)
# File or directory? # File or directory?
if type == 'directory': if type == 'directory':
if name not in ['.', '..']: if name not in ['.', '..']:
...@@ -604,12 +604,12 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -604,12 +604,12 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
'name': name, 'name': name,
'readable': 1, 'readable': 1,
'writable': 1, 'writable': 1,
'path': listing_path 'path': listing_path
} }
}) })
else: else:
data.append({ data.append({
'id': listing_path[:-1], # Remove trailing slash 'id': listing_path[:-1], # Remove trailing slash
'type': 'file', 'type': 'file',
'attributes':{ 'attributes':{
'modified': timestamp, 'modified': timestamp,
...@@ -617,68 +617,68 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -617,68 +617,68 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
'readable': 1, 'readable': 1,
'writable': 1, 'writable': 1,
"size": size, "size": size,
'path': listing_path[:-1] # Remove trailing slash 'path': listing_path[:-1] # Remove trailing slash
} }
}) })
else: else:
raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type)) raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type))
return data return data
def stat(self, path, user, storage): def stat(self, path, user, storage):
# Data container # Data container
data = [] data = []
if storage.type == 'generic_posix': if storage.type == 'generic_posix':
shell_path = self.sanitize_and_prepare_shell_path(path, user, storage) shell_path = self.sanitize_and_prepare_shell_path(path, user, storage)
# Prepare command. See the ls function above for some more info # Prepare command. See the ls function above for some more info
command = self.prepare_sh_command('stat --printf=\'%F/%s/%Y/%n\\n\' {}'.format(shell_path), user, storage) command = self.prepare_sh_command('stat --printf=\'%F/%s/%Y/%n\\n\' {}'.format(shell_path), user, storage)
# Execute_command # Execute_command
out = os_shell(command, capture=True) out = os_shell(command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
# Did we just get a "cannot stat - No such file or directory error? # Did we just get a "cannot stat - No such file or directory error?
if 'No such file or directory' in out.stderr: if 'No such file or directory' in out.stderr:
pass pass
else: else:
raise Exception(out.stderr) raise Exception(out.stderr)
# Log # Log
#logger.debug('Shell exec output: "{}"'.format(out)) #logger.debug('Shell exec output: "{}"'.format(out))
out_lines = out.stdout.split('\n') out_lines = out.stdout.split('\n')
if len(out_lines) > 1: if len(out_lines) > 1:
raise Exception('Internal error on stat: more than one ouput line') raise Exception('Internal error on stat: more than one ouput line')
out_line = out_lines[0] out_line = out_lines[0]
# Example output line: directory:My folder:68/1617030350 # Example output line: directory:My folder:68/1617030350
# In this context, we also might get the following output: # In this context, we also might get the following output:
# directory/68/1617030350//My folder/ # directory/68/1617030350//My folder/
# ..so, use the clean path to remove all extra slashes. # ..so, use the clean path to remove all extra slashes.
# The only uncovered case is to rename the root folder... # The only uncovered case is to rename the root folder...
out_line = self.clean_path(out_line) out_line = self.clean_path(out_line)
# Set names # Set names
line_pieces = out_line.split('/') line_pieces = out_line.split('/')
type = line_pieces[0] type = line_pieces[0]
size = line_pieces[1] size = line_pieces[1]
timestamp = line_pieces[2] timestamp = line_pieces[2]
name = '/'.join(line_pieces[3:]) name = '/'.join(line_pieces[3:])
data = {'type': type, 'name': name, 'size': size, 'timestamp': timestamp} data = {'type': type, 'name': name, 'size': size, 'timestamp': timestamp}
else: else:
raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type)) raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type))
return data return data
def delete(self, path, user, storage): def delete(self, path, user, storage):
...@@ -686,18 +686,18 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -686,18 +686,18 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
if storage.type == 'generic_posix': if storage.type == 'generic_posix':
shell_path = self.sanitize_and_prepare_shell_path(path, user, storage) shell_path = self.sanitize_and_prepare_shell_path(path, user, storage)
# Prepare command # Prepare command
command = self.prepare_sh_command('rm -rf {}'.format(shell_path), user, storage) command = self.prepare_sh_command('rm -rf {}'.format(shell_path), user, storage)
# Execute_command # Execute_command
out = os_shell(command, capture=True) out = os_shell(command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
raise Exception(out.stderr) raise Exception(out.stderr)
else: else:
raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type)) raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type))
def mkdir(self, path, user, storage, force=False): def mkdir(self, path, user, storage, force=False):
...@@ -709,28 +709,28 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -709,28 +709,28 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
command = self.prepare_sh_command('mkdir -p {}'.format(path), user, storage) command = self.prepare_sh_command('mkdir -p {}'.format(path), user, storage)
else: else:
command = self.prepare_sh_command('mkdir {}'.format(path), user, storage) command = self.prepare_sh_command('mkdir {}'.format(path), user, storage)
# Execute_command # Execute_command
out = os_shell(command, capture=True) out = os_shell(command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
raise Exception(out.stderr) raise Exception(out.stderr)
else: else:
raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type)) raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type))
def cat(self, path, user, storage): def cat(self, path, user, storage):
# Data container # Data container
data = [] data = []
if storage.type == 'generic_posix': if storage.type == 'generic_posix':
shell_path = self.sanitize_and_prepare_shell_path(path, user, storage) shell_path = self.sanitize_and_prepare_shell_path(path, user, storage)
# Prepare command # Prepare command
command = self.prepare_sh_command('cat {}'.format(shell_path), user, storage) command = self.prepare_sh_command('cat {}'.format(shell_path), user, storage)
# Execute_command # Execute_command
out = os_shell(command, capture=True) out = os_shell(command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
...@@ -738,7 +738,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -738,7 +738,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
data = out.stdout data = out.stdout
else: else:
raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type)) raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type))
return data return data
...@@ -746,22 +746,22 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -746,22 +746,22 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
def rename(self, old, new, user, storage): def rename(self, old, new, user, storage):
if storage.type == 'generic_posix': if storage.type == 'generic_posix':
old = self.sanitize_and_prepare_shell_path(old, user, storage) old = self.sanitize_and_prepare_shell_path(old, user, storage)
new = self.sanitize_and_prepare_shell_path(new, user, storage) new = self.sanitize_and_prepare_shell_path(new, user, storage)
# Prepare command # Prepare command
command = self.prepare_sh_command('mv {} {}'.format(old, new), user, storage) command = self.prepare_sh_command('mv {} {}'.format(old, new), user, storage)
logger.critical(command) logger.critical(command)
# Execute_command # Execute_command
out = os_shell(command, capture=True) out = os_shell(command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
raise Exception(out.stderr) raise Exception(out.stderr)
else: else:
raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type)) raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type))
def copy(self, source, target, user, storage): def copy(self, source, target, user, storage):
...@@ -770,17 +770,17 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -770,17 +770,17 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
source = self.sanitize_and_prepare_shell_path(source, user, storage) source = self.sanitize_and_prepare_shell_path(source, user, storage)
target = self.sanitize_and_prepare_shell_path(target, user, storage) target = self.sanitize_and_prepare_shell_path(target, user, storage)
# Prepare command # Prepare command
command = self.prepare_sh_command('cp -a {} {}'.format(source, target), user, storage) command = self.prepare_sh_command('cp -a {} {}'.format(source, target), user, storage)
# Execute_command # Execute_command
out = os_shell(command, capture=True) out = os_shell(command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
raise Exception(out.stderr) raise Exception(out.stderr)
else: else:
raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type)) raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type))
def scp_from(self, source, target, user, storage, mode='get'): def scp_from(self, source, target, user, storage, mode='get'):
...@@ -815,44 +815,44 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -815,44 +815,44 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
# API GET # API GET
#============================ #============================
def _get(self, request): def _get(self, request):
mode = request.GET.get('mode', None) mode = request.GET.get('mode', None)
time = request.GET.get('time', None) time = request.GET.get('time', None)
path = request.GET.get('path', None) path = request.GET.get('path', None)
_ = request.GET.get('_', None) _ = request.GET.get('_', None)
# Clean for some issues that happen sometimes # Clean for some issues that happen sometimes
if path: if path:
path = self.clean_path(path) path = self.clean_path(path)
# Init # Init
if mode == 'initiate': if mode == 'initiate':
data = json.loads('{"data":{"attributes":{"config":{"options":{"culture":"en"},"security":{"allowFolderDownload":true,"extensions":{"ignoreCase":true,"policy":"DISALLOW_LIST","restrictions":[]},"readOnly":false}}},"id":"/","type":"initiate"}}') data = json.loads('{"data":{"attributes":{"config":{"options":{"culture":"en"},"security":{"allowFolderDownload":true,"extensions":{"ignoreCase":true,"policy":"DISALLOW_LIST","restrictions":[]},"readOnly":false}}},"id":"/","type":"initiate"}}')
elif mode == 'readfolder': elif mode == 'readfolder':
# Base folder (computing resource-level) # Base folder (computing resource-level)
if path == '/': if path == '/':
# Data container # Data container
data = {'data':[]} data = {'data':[]}
# Get storages # Get storages
storages = list(Storage.objects.filter(group=None,browsable=True)) + list(Storage.objects.filter(group__user=request.user,browsable=True)) storages = list(Storage.objects.filter(group=None,browsable=True)) + list(Storage.objects.filter(group__user=request.user,browsable=True))
# Oder storages (re-orderded in the file manager anyway) # Oder storages (re-orderded in the file manager anyway)
storages.sort(key=lambda storage: storage.id) storages.sort(key=lambda storage: storage.id)
# Prepare the output # Prepare the output
for storage in storages: for storage in storages:
# For now, we only support generic posix, SSH-based storages # For now, we only support generic posix, SSH-based storages
if not storage.type=='generic_posix' and storage.access_mode=='ssh+cli': if not storage.type=='generic_posix' and storage.access_mode=='ssh+cli':
continue continue
if storage.access_through_computing and not storage.computing.manager.is_configured_for(user=request.user): if storage.access_through_computing and not storage.computing.manager.is_configured_for(user=request.user):
continue continue
data['data'].append({ data['data'].append({
'id': '/{}/'.format(storage.id), 'id': '/{}/'.format(storage.id),
'type': 'folder', 'type': 'folder',
...@@ -860,15 +860,15 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -860,15 +860,15 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
'name': storage.id, 'name': storage.id,
'readable': 1, 'readable': 1,
'writable': 1, 'writable': 1,
'path': '/{}/'.format(storage.id) 'path': '/{}/'.format(storage.id)
} }
}) })
else: else:
storage = self.get_storage_from_path(path, request) storage = self.get_storage_from_path(path, request)
# Get base directoris and files for this storage: # Get base directoris and files for this storage:
ls_path = '/'+'/'.join(path.split('/')[2:]) ls_path = '/'+'/'.join(path.split('/')[2:])
data = {'data': self.ls(ls_path, request.user, storage)} data = {'data': self.ls(ls_path, request.user, storage)}
...@@ -879,12 +879,12 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -879,12 +879,12 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
# Set support vars # Set support vars
storage = self.get_storage_from_path(path, request) storage = self.get_storage_from_path(path, request)
file_path = '/'+'/'.join(path.split('/')[2:]) file_path = '/'+'/'.join(path.split('/')[2:])
if path.endswith('/'): if path.endswith('/'):
return error400('Downloading a folder is not supported') return error400('Downloading a folder is not supported')
if storage.type != 'generic_posix': if storage.type != 'generic_posix':
raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type)) raise NotImplementedError('Storage type "{}" not implemented'.format(storage.type))
# TODO: here we are not handling the ajax request, Maybe they have been deperacted? # TODO: here we are not handling the ajax request, Maybe they have been deperacted?
# The download process consists of 2 requests: # The download process consists of 2 requests:
...@@ -894,23 +894,23 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -894,23 +894,23 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
if storage.access_mode == 'ssh+cli': if storage.access_mode == 'ssh+cli':
target_path = '/tmp/{}'.format(uuid.uuid4()) target_path = '/tmp/{}'.format(uuid.uuid4())
# Get the file # Get the file
self.scp_from(file_path, target_path, request.user, storage, mode='get') self.scp_from(file_path, target_path, request.user, storage, mode='get')
# Detect content type # Detect content type
try: try:
content_type = str(magic.from_file(target_path, mime=True)) content_type = str(magic.from_file(target_path, mime=True))
except: except:
content_type = None content_type = None
# Read file data # Read file data
with open(target_path, 'rb') as f: with open(target_path, 'rb') as f:
data = f.read() data = f.read()
# Remove temporary file # Remove temporary file
os.remove(target_path) os.remove(target_path)
elif storage.access_mode == 'cli': elif storage.access_mode == 'cli':
# Define storage internal source path # Define storage internal source path
...@@ -921,13 +921,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -921,13 +921,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
content_type = str(magic.from_file(storage_source_path, mime=True)) content_type = str(magic.from_file(storage_source_path, mime=True))
except: except:
content_type = None content_type = None
# Read file data # Read file data
with open(storage_source_path, 'rb') as f: with open(storage_source_path, 'rb') as f:
data = f.read() data = f.read()
else: else:
raise NotImplementedError('Storage access mode "{}" not implemented'.format(storage.access_mode)) raise NotImplementedError('Storage access mode "{}" not implemented'.format(storage.access_mode))
# Return file data # Return file data
response = HttpResponse(data, status=status.HTTP_200_OK, content_type=content_type) response = HttpResponse(data, status=status.HTTP_200_OK, content_type=content_type)
...@@ -936,21 +936,21 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -936,21 +936,21 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
elif mode == 'readfile': elif mode == 'readfile':
logger.debug('Reading "{}"'.format(path)) logger.debug('Reading "{}"'.format(path))
# Set support vars # Set support vars
storage = self.get_storage_from_path(path, request) storage = self.get_storage_from_path(path, request)
file_path = '/'+'/'.join(path.split('/')[2:]) file_path = '/'+'/'.join(path.split('/')[2:])
# Get file contents # Get file contents
data = self.cat(file_path, request.user, storage) data = self.cat(file_path, request.user, storage)
# Return file contents # Return file contents
return HttpResponse(data, status=status.HTTP_200_OK) return HttpResponse(data, status=status.HTTP_200_OK)
elif mode == 'delete': elif mode == 'delete':
logger.debug('Deleting "{}"'.format(path)) logger.debug('Deleting "{}"'.format(path))
# Set support vars # Set support vars
storage = self.get_storage_from_path(path, request) storage = self.get_storage_from_path(path, request)
path = '/'+'/'.join(path.split('/')[2:]) path = '/'+'/'.join(path.split('/')[2:])
...@@ -972,23 +972,23 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -972,23 +972,23 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
'name': path, 'name': path,
'readable': 1, 'readable': 1,
'writable': 1, 'writable': 1,
'path': '/{}{}'.format(storage.id, path) 'path': '/{}{}'.format(storage.id, path)
} }
} }
} }
# Return file contents # Return file contents
return Response(data, status=status.HTTP_200_OK) return Response(data, status=status.HTTP_200_OK)
elif mode == 'addfolder': elif mode == 'addfolder':
logger.debug('Deleting "{}"'.format(path)) logger.debug('Deleting "{}"'.format(path))
name = request.GET.get('name', None) name = request.GET.get('name', None)
if not name: if not name:
raise ValueError('No folder name set') raise ValueError('No folder name set')
# Set support vars # Set support vars
storage = self.get_storage_from_path(path, request) storage = self.get_storage_from_path(path, request)
path = '/'+'/'.join(path.split('/')[2:]) + name path = '/'+'/'.join(path.split('/')[2:]) + name
...@@ -1005,27 +1005,27 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1005,27 +1005,27 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
'name': name, 'name': name,
'readable': 1, 'readable': 1,
'writable': 1, 'writable': 1,
'path': '/{}{}'.format(storage.id, path) 'path': '/{}{}'.format(storage.id, path)
} }
} }
} }
# Return file contents # Return file contents
return Response(data, status=status.HTTP_200_OK) return Response(data, status=status.HTTP_200_OK)
elif mode == 'rename': elif mode == 'rename':
logger.debug('Renaming "{}"'.format(path)) logger.debug('Renaming "{}"'.format(path))
# Get old file name with path # Get old file name with path
old_name_with_path = request.GET.get('old', None) old_name_with_path = request.GET.get('old', None)
if not old_name_with_path: if not old_name_with_path:
raise Exception('Missing old name') raise Exception('Missing old name')
# Set support vars # Set support vars
storage = self.get_storage_from_path(old_name_with_path, request) storage = self.get_storage_from_path(old_name_with_path, request)
old_name_with_path = '/'+'/'.join(old_name_with_path.split('/')[2:]) old_name_with_path = '/'+'/'.join(old_name_with_path.split('/')[2:])
# Is it a folder? # Is it a folder?
if old_name_with_path.endswith('/'): if old_name_with_path.endswith('/'):
is_folder=True is_folder=True
...@@ -1036,7 +1036,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1036,7 +1036,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
new_name = request.GET.get('new', None) new_name = request.GET.get('new', None)
if not new_name: if not new_name:
raise Exception('Missing new name') raise Exception('Missing new name')
if is_folder: if is_folder:
new_name_with_path = '/'.join(old_name_with_path.split('/')[:-2]) + '/' + new_name new_name_with_path = '/'.join(old_name_with_path.split('/')[:-2]) + '/' + new_name
old_name_with_path = old_name_with_path[:-1] old_name_with_path = old_name_with_path[:-1]
...@@ -1045,11 +1045,11 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1045,11 +1045,11 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
# Rename # Rename
self.rename(old_name_with_path, new_name_with_path, request.user, storage) self.rename(old_name_with_path, new_name_with_path, request.user, storage)
# Add trailing slash for listing # Add trailing slash for listing
if is_folder: if is_folder:
new_name_with_path = new_name_with_path+'/' new_name_with_path = new_name_with_path+'/'
# Get new info # Get new info
stat = self.stat(new_name_with_path, request.user, storage) stat = self.stat(new_name_with_path, request.user, storage)
...@@ -1062,31 +1062,31 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1062,31 +1062,31 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
'name': new_name, 'name': new_name,
'readable': 1, 'readable': 1,
'writable': 1, 'writable': 1,
'path': '/{}{}'.format(storage.id, new_name_with_path) 'path': '/{}{}'.format(storage.id, new_name_with_path)
} }
} }
} }
# Add size if file # Add size if file
if not is_folder: data['data']['attributes']['size'] = stat['size'] if not is_folder: data['data']['attributes']['size'] = stat['size']
# Return file contents # Return file contents
return Response(data, status=status.HTTP_200_OK) return Response(data, status=status.HTTP_200_OK)
elif mode == 'copy': elif mode == 'copy':
logger.debug('Copying "{}"'.format(path)) logger.debug('Copying "{}"'.format(path))
# Get source for copy # Get source for copy
source_name_with_path = request.GET.get('source', None) source_name_with_path = request.GET.get('source', None)
if not source_name_with_path: if not source_name_with_path:
raise Exception('Missing source for copy') raise Exception('Missing source for copy')
# Get target for copy # Get target for copy
target_path = request.GET.get('target', None) target_path = request.GET.get('target', None)
if not target_path: if not target_path:
raise Exception('Missing target for copy') raise Exception('Missing target for copy')
if source_name_with_path.endswith('/'): if source_name_with_path.endswith('/'):
is_folder=True is_folder=True
...@@ -1096,10 +1096,10 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1096,10 +1096,10 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
# Set support vars # Set support vars
storage = self.get_storage_from_path(source_name_with_path, request) storage = self.get_storage_from_path(source_name_with_path, request)
if is_folder: if is_folder:
source_name_with_path = '/'+'/'.join(source_name_with_path.split('/')[2:])[:-1] source_name_with_path = '/'+'/'.join(source_name_with_path.split('/')[2:])[:-1]
target_name_with_path = '/'+'/'.join(target_path.split('/')[2:]) + source_name_with_path.split('/')[-1] target_name_with_path = '/'+'/'.join(target_path.split('/')[2:]) + source_name_with_path.split('/')[-1]
else: else:
source_name_with_path = '/'+'/'.join(source_name_with_path.split('/')[2:]) source_name_with_path = '/'+'/'.join(source_name_with_path.split('/')[2:])
target_name_with_path = '/'+'/'.join(target_path.split('/')[2:]) + source_name_with_path.split('/')[-1] target_name_with_path = '/'+'/'.join(target_path.split('/')[2:]) + source_name_with_path.split('/')[-1]
...@@ -1121,7 +1121,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1121,7 +1121,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
# Get new info # Get new info
stat = self.stat(target_name_with_path, request.user, storage) stat = self.stat(target_name_with_path, request.user, storage)
# Response data # Response data
data = { 'data': { data = { 'data': {
'id': '/{}{}'.format(storage.id, target_name_with_path), 'id': '/{}{}'.format(storage.id, target_name_with_path),
...@@ -1131,25 +1131,25 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1131,25 +1131,25 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
'name': target_name_with_path.split('/')[-2] if is_folder else target_name_with_path.split('/')[-1], 'name': target_name_with_path.split('/')[-2] if is_folder else target_name_with_path.split('/')[-1],
'readable': 1, 'readable': 1,
'writable': 1, 'writable': 1,
'path': '/{}{}'.format(storage.id, target_name_with_path) 'path': '/{}{}'.format(storage.id, target_name_with_path)
} }
} }
} }
# Add size if file # Add size if file
if not is_folder: data['data']['attributes']['size'] = stat['size'] if not is_folder: data['data']['attributes']['size'] = stat['size']
# Return file contents # Return file contents
return Response(data, status=status.HTTP_200_OK) return Response(data, status=status.HTTP_200_OK)
else: else:
return error400('Operation "{}" not supported'.format(mode)) return error400('Operation "{}" not supported'.format(mode))
return Response(data, status=status.HTTP_200_OK) return Response(data, status=status.HTTP_200_OK)
#============================ #============================
# API POST # API POST
#============================ #============================
def _post(self, request): def _post(self, request):
...@@ -1160,33 +1160,33 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1160,33 +1160,33 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
if mode == 'savefile': if mode == 'savefile':
return error400('Operation "{}" not supported'.format(mode)) return error400('Operation "{}" not supported'.format(mode))
elif mode == 'upload': elif mode == 'upload':
# Set support vars # Set support vars
storage = self.get_storage_from_path(path, request) storage = self.get_storage_from_path(path, request)
path = '/'+'/'.join(path.split('/')[2:]) path = '/'+'/'.join(path.split('/')[2:])
# Bug workaround? # Bug workaround?
if not path.endswith('/'): if not path.endswith('/'):
path += '/' path += '/'
# Get the file upload # Get the file upload
file_upload = request.FILES['files'] file_upload = request.FILES['files']
if storage.access_mode == 'ssh+cli': if storage.access_mode == 'ssh+cli':
# Generate temporary UUID # Generate temporary UUID
file_uuid = uuid.uuid4() file_uuid = uuid.uuid4()
with open('/tmp/{}'.format(file_uuid), 'wb') as temp_file: with open('/tmp/{}'.format(file_uuid), 'wb') as temp_file:
temp_file.write(file_upload.read()) temp_file.write(file_upload.read())
logger.debug('Wrote "/tmp/{}" for "{}"'.format(file_uuid, file_upload.name)) logger.debug('Wrote "/tmp/{}" for "{}"'.format(file_uuid, file_upload.name))
# Now copy with scp # Now copy with scp
self.scp_to('/tmp/{}'.format(file_uuid), path + file_upload.name , request.user, storage, mode='put') self.scp_to('/tmp/{}'.format(file_uuid), path + file_upload.name , request.user, storage, mode='put')
# Response data # Response data
data = { 'data': [{ data = { 'data': [{
'id': '/{}{}{}'.format(storage.id, path, file_upload.name), 'id': '/{}{}{}'.format(storage.id, path, file_upload.name),
...@@ -1197,19 +1197,19 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1197,19 +1197,19 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
'readable': 1, 'readable': 1,
'size': os.path.getsize('/tmp/{}'.format(file_uuid)), # This is kind of an approximation! 'size': os.path.getsize('/tmp/{}'.format(file_uuid)), # This is kind of an approximation!
'writable': 1, 'writable': 1,
'path': '/{}{}{}'.format(storage.id, path, file_upload.name) 'path': '/{}{}{}'.format(storage.id, path, file_upload.name)
} }
}] }]
} }
# Remove file # Remove file
os.remove('/tmp/{}'.format(file_uuid)) os.remove('/tmp/{}'.format(file_uuid))
if storage.access_mode == 'cli': if storage.access_mode == 'cli':
try: try:
as_user = storage.conf['as_user'] as_user = storage.conf['as_user']
# Is "as_user" a UID? # Is "as_user" a UID?
try: try:
uid = int(as_user) uid = int(as_user)
...@@ -1231,12 +1231,12 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1231,12 +1231,12 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
raise Exception(out.sterr) raise Exception(out.sterr)
as_user = 'user_' + str(uid) as_user = 'user_' + str(uid)
else: else:
as_user = out.stdout.strip() as_user = out.stdout.strip()
except (KeyError, TypeError): except (KeyError, TypeError):
as_user = None as_user = None
# Define storage internal dest path # Define storage internal dest path
storage_dest_path = self.sanitize_and_prepare_shell_path(path + file_upload.name, request.user, storage, escapes=False) storage_dest_path = self.sanitize_and_prepare_shell_path(path + file_upload.name, request.user, storage, escapes=False)
storage_dest_path_escaped = self.sanitize_and_prepare_shell_path(path + file_upload.name, request.user, storage, escapes=True) storage_dest_path_escaped = self.sanitize_and_prepare_shell_path(path + file_upload.name, request.user, storage, escapes=True)
...@@ -1246,26 +1246,26 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1246,26 +1246,26 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
if as_user: if as_user:
# Generate temporary UUID # Generate temporary UUID
file_uuid = uuid.uuid4() file_uuid = uuid.uuid4()
with open('/tmp/{}'.format(file_uuid), 'wb') as temp_file: with open('/tmp/{}'.format(file_uuid), 'wb') as temp_file:
temp_file.write(file_upload.read()) temp_file.write(file_upload.read())
file_size = os.path.getsize('/tmp/{}'.format(file_uuid)) file_size = os.path.getsize('/tmp/{}'.format(file_uuid))
# Change ownership and move # Change ownership and move
os_shell('sudo -i -u {0} chown {0}:{0} /tmp/{1}'.format(as_user, file_uuid), capture=True) os_shell('sudo -i -u {0} chown {0}:{0} /tmp/{1}'.format(as_user, file_uuid), capture=True)
if out.exit_code != 0: if out.exit_code != 0:
raise Exception(out.sterr) raise Exception(out.sterr)
os_shell('sudo -i -u {0} mv /tmp/{1} {2}'.format(as_user, file_uuid, storage_dest_path_escaped), capture=True) os_shell('sudo -i -u {0} mv /tmp/{1} {2}'.format(as_user, file_uuid, storage_dest_path_escaped), capture=True)
if out.exit_code != 0: if out.exit_code != 0:
raise Exception(out.sterr) raise Exception(out.sterr)
else: else:
with open(storage_dest_path, 'wb') as upload_file: with open(storage_dest_path, 'wb') as upload_file:
upload_file.write(file_upload.read()) upload_file.write(file_upload.read())
file_size = os.path.getsize(storage_dest_path) file_size = os.path.getsize(storage_dest_path)
logger.debug('Wrote "{}" for "{}"'.format(storage_dest_path, file_upload.name)) logger.debug('Wrote "{}" for "{}"'.format(storage_dest_path, file_upload.name))
# Response data # Response data
data = { 'data': [{ data = { 'data': [{
'id': '/{}{}{}'.format(storage.id, path, file_upload.name), 'id': '/{}{}{}'.format(storage.id, path, file_upload.name),
...@@ -1276,17 +1276,17 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): ...@@ -1276,17 +1276,17 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
'readable': 1, 'readable': 1,
'size': file_size, 'size': file_size,
'writable': 1, 'writable': 1,
'path': '/{}{}{}'.format(storage.id, path, file_upload.name) 'path': '/{}{}{}'.format(storage.id, path, file_upload.name)
} }
}] }]
} }
else: else:
raise NotImplementedError('Storage access mode "{}" not implemented'.format(storage.access_mode)) raise NotImplementedError('Storage access mode "{}" not implemented'.format(storage.access_mode))
# Return # Return
return Response(data, status=status.HTTP_200_OK) return Response(data, status=status.HTTP_200_OK)
else: else:
return error400('Operation "{}" not supported'.format(mode)) return error400('Operation "{}" not supported'.format(mode))
...@@ -1303,7 +1303,7 @@ class ImportRepositoryAPI(PrivateGETAPI): ...@@ -1303,7 +1303,7 @@ class ImportRepositoryAPI(PrivateGETAPI):
""" """
def _get(self, request): def _get(self, request):
repository_url = request.GET.get('repository_url', None) repository_url = request.GET.get('repository_url', None)
repository_tag = request.GET.get('repository_tag', None) repository_tag = request.GET.get('repository_tag', None)
container_name = request.GET.get('container_name', None) container_name = request.GET.get('container_name', None)
...@@ -1314,11 +1314,11 @@ class ImportRepositoryAPI(PrivateGETAPI): ...@@ -1314,11 +1314,11 @@ class ImportRepositoryAPI(PrivateGETAPI):
if not repository_tag: if not repository_tag:
return error400('Missing "repository_tag"') return error400('Missing "repository_tag"')
logger.debug('Importing repository "%s" with tag "%s"', repository_url, repository_tag) logger.debug('Importing repository "%s" with tag "%s"', repository_url, repository_tag)
results = {} results = {}
try: try:
container = get_or_create_container_from_repository(request.user, repository_url, repository_tag, container_name, container_description) container = get_or_create_container_from_repository(request.user, repository_url, repository_tag, container_name, container_description)
except Exception as e: except Exception as e:
results['import_succeded'] = False results['import_succeded'] = False
...@@ -1326,8 +1326,9 @@ class ImportRepositoryAPI(PrivateGETAPI): ...@@ -1326,8 +1326,9 @@ class ImportRepositoryAPI(PrivateGETAPI):
else: else:
results['import_succeded'] = True results['import_succeded'] = True
results['container_uuid'] = str(container.uuid) results['container_uuid'] = str(container.uuid)
return ok200(results) return ok200(results)
......
import os import os
from .models import TaskStatuses, KeyPair, Task, Storage from .models import TaskStatuses, KeyPair, Task, Storage
from .utils import os_shell, get_ssh_access_mode_credentials, sanitize_container_env_vars, booleanize from .utils import os_shell, get_ssh_access_mode_credentials, sanitize_container_env_vars, booleanize, setup_tunnel_and_proxy
from .exceptions import ErrorMessage, ConsistencyException from .exceptions import ErrorMessage, ConsistencyException
from django.conf import settings from django.conf import settings
...@@ -9,40 +9,40 @@ import logging ...@@ -9,40 +9,40 @@ import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
ROSETTA_AGENT_CHECK_SSL = booleanize(os.environ.get('ROSETTA_AGENT_CHECK_SSL', True)) ROSETTA_AGENT_CHECK_SSL = booleanize(os.environ.get('ROSETTA_AGENT_CHECK_SSL', True))
CHECK_WGET_CERT_STR = '--no-check-certificate' if not ROSETTA_AGENT_CHECK_SSL else '' CHECK_CURL_CERT_STR = '--insecure' if not ROSETTA_AGENT_CHECK_SSL else ''
class ComputingManager(object): class ComputingManager(object):
def __init__(self, computing): def __init__(self, computing):
self.computing = computing self.computing = computing
def start_task(self, task, **kwargs): def start_task(self, task, **kwargs):
# Check for run task logic implementation # Check for run task logic implementation
try: try:
self._start_task self._start_task
except AttributeError: except AttributeError:
raise NotImplementedError('Not implemented') raise NotImplementedError('Not implemented')
# Call actual run task logic # Call actual run task logic
self._start_task(task, **kwargs) self._start_task(task, **kwargs)
def stop_task(self, task, **kwargs): def stop_task(self, task, **kwargs):
# Check for stop task logic implementation # Check for stop task logic implementation
try: try:
self._stop_task self._stop_task
except AttributeError: except AttributeError:
raise NotImplementedError('Not implemented') raise NotImplementedError('Not implemented')
# Call actual stop task logic # Call actual stop task logic
self._stop_task(task, **kwargs) self._stop_task(task, **kwargs)
# Ok, save status as deleted # Ok, save status as deleted
task.status = 'stopped' task.status = 'stopped'
task.save() task.save()
# Check if the tunnel is active and if so kill it # Check if the tunnel is active and if so kill it
logger.debug('Checking if task "{}" has a running tunnel'.format(task.uuid)) logger.debug('Checking if task "{}" has a running tunnel'.format(task.uuid))
check_command = 'ps -ef | grep ":'+str(task.tcp_tunnel_port)+':'+str(task.interface_ip)+':'+str(task.interface_port)+'" | grep -v grep | awk \'{print $2}\'' check_command = 'ps -ef | grep ":'+str(task.tcp_tunnel_port)+':'+str(task.interface_ip)+':'+str(task.interface_port)+'" | grep -v grep | awk \'{print $2}\''
...@@ -54,10 +54,10 @@ class ComputingManager(object): ...@@ -54,10 +54,10 @@ class ComputingManager(object):
tunnel_pid = out.stdout tunnel_pid = out.stdout
# Kill Tunnel command # Kill Tunnel command
kill_tunnel_command= 'kill -9 {}'.format(tunnel_pid) kill_tunnel_command= 'kill -9 {}'.format(tunnel_pid)
# Log # Log
logger.debug('Killing tunnel with command: {}'.format(kill_tunnel_command)) logger.debug('Killing tunnel with command: {}'.format(kill_tunnel_command))
# Execute # Execute
os_shell(kill_tunnel_command, capture=True) os_shell(kill_tunnel_command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
...@@ -65,16 +65,16 @@ class ComputingManager(object): ...@@ -65,16 +65,16 @@ class ComputingManager(object):
def get_task_log(self, task, **kwargs): def get_task_log(self, task, **kwargs):
# Check for get task log logic implementation # Check for get task log logic implementation
try: try:
self._get_task_log self._get_task_log
except AttributeError: except AttributeError:
raise NotImplementedError('Not implemented') raise NotImplementedError('Not implemented')
# Call actual get task log logic # Call actual get task log logic
return self._get_task_log(task, **kwargs) return self._get_task_log(task, **kwargs)
def is_configured_for(self, user): def is_configured_for(self, user):
return True return True
...@@ -88,7 +88,7 @@ class ClusterComputingManager(ComputingManager): ...@@ -88,7 +88,7 @@ class ClusterComputingManager(ComputingManager):
class SSHComputingManager(ComputingManager): class SSHComputingManager(ComputingManager):
def is_configured_for(self, user): def is_configured_for(self, user):
try: try:
get_ssh_access_mode_credentials(self.computing, user) get_ssh_access_mode_credentials(self.computing, user)
...@@ -100,7 +100,7 @@ class SSHComputingManager(ComputingManager): ...@@ -100,7 +100,7 @@ class SSHComputingManager(ComputingManager):
class InternalStandaloneComputingManager(StandaloneComputingManager): class InternalStandaloneComputingManager(StandaloneComputingManager):
def _start_task(self, task): def _start_task(self, task):
# Init run command #--cap-add=NET_ADMIN --cap-add=NET_RAW # Init run command #--cap-add=NET_ADMIN --cap-add=NET_RAW
...@@ -121,29 +121,51 @@ class InternalStandaloneComputingManager(StandaloneComputingManager): ...@@ -121,29 +121,51 @@ class InternalStandaloneComputingManager(StandaloneComputingManager):
# Env vars if any # Env vars if any
if task.container.env_vars: if task.container.env_vars:
# Sanitize again just in case the DB got somehow compromised: # Sanitize again just in case the DB got somehow compromised:
env_vars = sanitize_container_env_vars(task.container.env_vars) env_vars = sanitize_container_env_vars(task.container.env_vars)
for env_var in env_vars: for env_var in env_vars:
run_command += ' -e{}={} '.format(env_var, env_vars[env_var]) run_command += ' -e{}={} '.format(env_var, env_vars[env_var])
# User data volume # User data volume
#run_command += ' -v {}/user-{}:/data'.format(settings.LOCAL_USER_DATA_DIR, task.user.id) #run_command += ' -v {}/user-{}:/data'.format(settings.LOCAL_USER_DATA_DIR, task.user.id)
# Handle storages (binds)
binds = ''
storages = Storage.objects.filter(computing=self.computing)
for storage in storages:
if storage.type == 'generic_posix' and storage.bind_path:
# Expand the base path
expanded_base_path = storage.base_path
if '$USER' in expanded_base_path:
expanded_base_path = expanded_base_path.replace('$USER', task.user.username)
# Expand the bind_path
expanded_bind_path = storage.bind_path
if '$USER' in expanded_bind_path:
expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username)
# Add the bind
if not binds:
binds = '-v{}:{}'.format(expanded_base_path, expanded_bind_path)
else:
binds += ' -v{}:{}'.format(expanded_base_path, expanded_bind_path)
# Host name, image entry command # Host name, image entry command
run_command += ' -h task-{} --name task-{} -d -t {}/{}:{}'.format(task.short_uuid, task.short_uuid, task.container.registry, task.container.image_name, task.container.image_tag) run_command += ' {} -h task-{} --name task-{} -d -t {}/{}:{}'.format(binds, task.short_uuid, task.short_uuid, task.container.registry, task.container.image_name, task.container.image_tag)
# Debug # Debug
logger.debug('Running new task with command="{}"'.format(run_command)) logger.debug('Running new task with command="{}"'.format(run_command))
# Run the task # Run the task
out = os_shell(run_command, capture=True) out = os_shell(run_command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
logger.error('Got error in starting task: {}'.format(out)) logger.error('Got error in starting task: {}'.format(out))
raise Exception(out.stderr) raise Exception(out.stderr)
else: else:
# Get task IP address # Get task IP address
out = os_shell('export CONTAINER_ID=$(sudo docker ps -a --filter name=task-'+task.short_uuid+' --format {{.ID}}) && sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' $CONTAINER_ID | tail -n1', capture=True) out = os_shell('export CONTAINER_ID=$(sudo docker ps -a --filter name=task-'+task.short_uuid+' --format {{.ID}}) && sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' $CONTAINER_ID | tail -n1', capture=True)
if out.exit_code != 0: if out.exit_code != 0:
...@@ -158,23 +180,31 @@ class InternalStandaloneComputingManager(StandaloneComputingManager): ...@@ -158,23 +180,31 @@ class InternalStandaloneComputingManager(StandaloneComputingManager):
# Save # Save
task.save() task.save()
# Setup the tunnel if using a custom protocol (otherwise it will get set up via the "connect" button)
if task.container.interface_protocol not in ['http', 'https']:
setup_tunnel_and_proxy(task)
def _stop_task(self, task): def _stop_task(self, task):
# Delete the Docker container # Delete the Docker container
stop_command = 'export CONTAINER_ID=$(sudo docker ps -a --filter name=task-'+task.short_uuid+' --format {{.ID}}) && sudo docker stop $CONTAINER_ID && sudo docker rm $CONTAINER_ID' stop_command = 'export CONTAINER_ID=$(sudo docker ps -a --filter name=task-'+task.short_uuid+' --format {{.ID}}) && sudo docker stop $CONTAINER_ID && sudo docker rm $CONTAINER_ID'
out = os_shell(stop_command, capture=True) out = os_shell(stop_command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
if 'No such container' in out.stderr: if 'No such container' in out.stderr:
# No container was found
pass
elif 'requires at least 1 argument' in out.stderr:
# No container was found
pass pass
else: else:
raise Exception(out.stderr) raise Exception(out.stderr)
# Set task as stopped # Set task as stopped
task.status = TaskStatuses.stopped task.status = TaskStatuses.stopped
task.save() task.save()
def _get_task_log(self, task, **kwargs): def _get_task_log(self, task, **kwargs):
# View the Docker container log (attach) # View the Docker container log (attach)
...@@ -189,7 +219,7 @@ class InternalStandaloneComputingManager(StandaloneComputingManager): ...@@ -189,7 +219,7 @@ class InternalStandaloneComputingManager(StandaloneComputingManager):
class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingManager): class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingManager):
def _start_task(self, task, **kwargs): def _start_task(self, task, **kwargs):
logger.debug('Starting a remote task "{}"'.format(self.computing)) logger.debug('Starting a remote task "{}"'.format(self.computing))
...@@ -199,7 +229,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -199,7 +229,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
# Get webapp conn string # Get webapp conn string
from.utils import get_webapp_conn_string from.utils import get_webapp_conn_string
webapp_conn_string = get_webapp_conn_string() webapp_conn_string = get_webapp_conn_string()
# Handle container engine # Handle container engine
container_engine = None container_engine = None
if task.computing_options: if task.computing_options:
...@@ -207,7 +237,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -207,7 +237,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
if not container_engine: if not container_engine:
container_engine = task.computing.default_container_engine container_engine = task.computing.default_container_engine
# engine-specific part # engine-specific part
if container_engine == 'singularity': if container_engine == 'singularity':
#if not task.container.supports_custom_interface_port: #if not task.container.supports_custom_interface_port:
...@@ -223,20 +253,20 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -223,20 +253,20 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
varsstring = '' varsstring = ''
# Sanitize again just in case the DB got somehow compromised: # Sanitize again just in case the DB got somehow compromised:
env_vars = sanitize_container_env_vars(task.container.env_vars) env_vars = sanitize_container_env_vars(task.container.env_vars)
for env_var in env_vars: for env_var in env_vars:
varsstring += ' && export SINGULARITYENV_{}={} '.format(env_var, env_vars[env_var]) varsstring += ' && export SINGULARITYENV_{}={} '.format(env_var, env_vars[env_var])
else: else:
varsstring = '' varsstring = ''
# Handle storages (binds) # Handle storages (binds)
binds = '' binds = ''
storages = Storage.objects.filter(computing=self.computing) storages = Storage.objects.filter(computing=self.computing)
for storage in storages: for storage in storages:
if storage.type == 'generic_posix' and storage.bind_path: if storage.type == 'generic_posix' and storage.bind_path:
# Expand the base path # Expand the base path
expanded_base_path = storage.base_path expanded_base_path = storage.base_path
if '$SSH_USER' in expanded_base_path: if '$SSH_USER' in expanded_base_path:
if storage.access_through_computing: if storage.access_through_computing:
expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user) expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user)
...@@ -244,9 +274,9 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -244,9 +274,9 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented') raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_base_path: if '$USER' in expanded_base_path:
expanded_base_path = expanded_base_path.replace('$USER', task.user.username) expanded_base_path = expanded_base_path.replace('$USER', task.user.username)
# Expand the bind_path # Expand the bind_path
expanded_bind_path = storage.bind_path expanded_bind_path = storage.bind_path
if '$SSH_USER' in expanded_bind_path: if '$SSH_USER' in expanded_bind_path:
if storage.access_through_computing: if storage.access_through_computing:
expanded_bind_path = expanded_bind_path.replace('$SSH_USER', computing_user) expanded_bind_path = expanded_bind_path.replace('$SSH_USER', computing_user)
...@@ -254,19 +284,19 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -254,19 +284,19 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented') raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_bind_path: if '$USER' in expanded_bind_path:
expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username) expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username)
# Add the bind # Add the bind
if not binds: if not binds:
binds = '-B {}:{}'.format(expanded_base_path, expanded_bind_path) binds = '-B {}:{}'.format(expanded_base_path, expanded_bind_path)
else: else:
binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path) binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)
run_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_port, computing_keys.private_key_file, computing_user, computing_host) run_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_port, computing_keys.private_key_file, computing_user, computing_host)
run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp && mkdir -p /tmp/{}_data/home && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid) run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp && mkdir -p /tmp/{}_data/home && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'wget {} {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export BASE_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid) run_command += 'curl {} {}/api/v1/base/agent/?task_uuid={} -o /tmp/{}_data/agent.py &> /dev/null && export BASE_PORT=\$(python3 /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_CURL_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT {} {} &&'.format(authstring, varsstring) run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT {} {} &&'.format(authstring, varsstring)
run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid) run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)
# Container part # Container part
run_command+='docker://{}/{}:{} &>> /tmp/{}_data/task.log & echo \$!"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid) run_command+='docker://{}/{}:{} &>> /tmp/{}_data/task.log & echo \$!"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid)
...@@ -283,20 +313,20 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -283,20 +313,20 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
varsstring = '' varsstring = ''
# Sanitize again just in case the DB got somehow compromised: # Sanitize again just in case the DB got somehow compromised:
env_vars = sanitize_container_env_vars(task.container.env_vars) env_vars = sanitize_container_env_vars(task.container.env_vars)
for env_var in env_vars: for env_var in env_vars:
varsstring += ' -e {}={} '.format(env_var, env_vars[env_var]) varsstring += ' -e {}={} '.format(env_var, env_vars[env_var])
else: else:
varsstring = '' varsstring = ''
# Handle storages (binds) # Handle storages (binds)
binds = '' binds = ''
storages = Storage.objects.filter(computing=self.computing) storages = Storage.objects.filter(computing=self.computing)
for storage in storages: for storage in storages:
if storage.type == 'generic_posix' and storage.bind_path: if storage.type == 'generic_posix' and storage.bind_path:
# Expand the base path # Expand the base path
expanded_base_path = storage.base_path expanded_base_path = storage.base_path
if '$SSH_USER' in expanded_base_path: if '$SSH_USER' in expanded_base_path:
if storage.access_through_computing: if storage.access_through_computing:
expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user) expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user)
...@@ -304,9 +334,9 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -304,9 +334,9 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented') raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_base_path: if '$USER' in expanded_base_path:
expanded_base_path = expanded_base_path.replace('$USER', task.user.username) expanded_base_path = expanded_base_path.replace('$USER', task.user.username)
# Expand the bind_path # Expand the bind_path
expanded_bind_path = storage.bind_path expanded_bind_path = storage.bind_path
if '$SSH_USER' in expanded_bind_path: if '$SSH_USER' in expanded_bind_path:
if storage.access_through_computing: if storage.access_through_computing:
expanded_bind_path = expanded_bind_path.replace('$SSH_USER', computing_user) expanded_bind_path = expanded_bind_path.replace('$SSH_USER', computing_user)
...@@ -314,26 +344,26 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -314,26 +344,26 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented') raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_bind_path: if '$USER' in expanded_bind_path:
expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username) expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username)
# Add the bind # Add the bind
if not binds: if not binds:
binds = '-v{}:{}'.format(expanded_base_path, expanded_bind_path) binds = '-v{}:{}'.format(expanded_base_path, expanded_bind_path)
else: else:
binds += ' -v{}:{}'.format(expanded_base_path, expanded_bind_path) binds += ' -v{}:{}'.format(expanded_base_path, expanded_bind_path)
# TODO: remove this hardcoding # TODO: remove this hardcoding
prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_engine=='docker') else '' prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_engine=='docker') else ''
run_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_port, computing_keys.private_key_file, computing_user, computing_host) run_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_port, computing_keys.private_key_file, computing_user, computing_host)
run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir /tmp/{}_data && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid) run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir /tmp/{}_data && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid)
run_command += 'wget {} {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export TASK_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid) run_command += 'curl {} {}/api/v1/base/agent/?task_uuid={} -o /tmp/{}_data/agent.py &> /dev/null && export TASK_PORT=\$(python3 /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_CURL_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'exec nohup {} {} run -p \$TASK_PORT:{} {} {} {} '.format(prefix, container_engine, task.container.interface_port, authstring, varsstring, binds) run_command += 'exec nohup {} {} run -p \$TASK_PORT:{} {} {} {} '.format(prefix, container_engine, task.container.interface_port, authstring, varsstring, binds)
if container_engine == 'podman': if container_engine == 'podman':
run_command += '--network=private --uts=private --userns=keep-id ' run_command += '--network=private --uts=private --userns=keep-id '
#run_command += '-d -t {}/{}:{}'.format(task.container.registry, task.container.image_name, task.container.image_tag) #run_command += '-d -t {}/{}:{}'.format(task.container.registry, task.container.image_name, task.container.image_tag)
run_command += '-h task-{} --name task-{} -t {}/{}:{}'.format(task.short_uuid, task.short_uuid, task.container.registry, task.container.image_name, task.container.image_tag) run_command += '-h task-{} --name task-{} -t {}/{}:{}'.format(task.short_uuid, task.short_uuid, task.container.registry, task.container.image_name, task.container.image_tag)
run_command += '&>> /tmp/{}_data/task.log &"\''.format(task.uuid) run_command += '&>> /tmp/{}_data/task.log &"\''.format(task.uuid)
else: else:
raise NotImplementedError('Container engine {} not supported'.format(container_engine)) raise NotImplementedError('Container engine {} not supported'.format(container_engine))
...@@ -341,8 +371,8 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -341,8 +371,8 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
if out.exit_code != 0: if out.exit_code != 0:
logger.error('Got error in starting task: {}'.format(out)) logger.error('Got error in starting task: {}'.format(out))
raise Exception(out.stderr) raise Exception(out.stderr)
# Log # Log
logger.debug('Shell exec output: "{}"'.format(out)) logger.debug('Shell exec output: "{}"'.format(out))
# Load back the task to avoid concurrency problems in the agent call # Load back the task to avoid concurrency problems in the agent call
...@@ -356,9 +386,13 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -356,9 +386,13 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
# Save # Save
task.save() task.save()
# Setup the tunnel if using a custom protocol (otherwise it will get set up via the "connect" button)
if task.container.interface_protocol not in ['http', 'https']:
setup_tunnel_and_proxy(task)
def _stop_task(self, task, **kwargs): def _stop_task(self, task, **kwargs):
# Get credentials # Get credentials
computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user) computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)
...@@ -370,7 +404,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -370,7 +404,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
container_engine = task.computing.default_container_engine container_engine = task.computing.default_container_engine
if container_engine=='singularity': if container_engine=='singularity':
internal_stop_command = 'kill -9 {}'.format(task.process_id) internal_stop_command = 'kill -9 {}'.format(task.process_id)
elif container_engine in ['docker', 'podman']: elif container_engine in ['docker', 'podman']:
# TODO: remove this hardcoding # TODO: remove this hardcoding
prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_engine=='docker') else '' prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_engine=='docker') else ''
...@@ -381,7 +415,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -381,7 +415,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
stop_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "{}"\''.format(computing_port, computing_keys.private_key_file, computing_user, computing_host, internal_stop_command) stop_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "{}"\''.format(computing_port, computing_keys.private_key_file, computing_user, computing_host, internal_stop_command)
out = os_shell(stop_command, capture=True) out = os_shell(stop_command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
if ('No such process' in out.stderr) or ('No such container' in out.stderr) or ('no container' in out.stderr) or ('missing' in out.stderr): if ('No such process' in out.stderr) or ('No such container' in out.stderr) or ('no container' in out.stderr) or ('missing' in out.stderr):
pass pass
...@@ -398,7 +432,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -398,7 +432,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
# Get credentials # Get credentials
computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user) computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)
# Handle container engine # Handle container engine
container_engine = None container_engine = None
if task.computing_options: if task.computing_options:
...@@ -407,13 +441,13 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -407,13 +441,13 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
container_engine = task.computing.default_container_engine container_engine = task.computing.default_container_engine
if container_engine=='singularity': if container_engine=='singularity':
internal_view_log_command = 'cat /tmp/{}_data/task.log'.format(task.uuid) internal_view_log_command = 'cat /tmp/{}_data/task.log'.format(task.uuid)
elif container_engine in ['docker','podman']: elif container_engine in ['docker','podman']:
# TODO: consider podman/docker logs? # TODO: consider podman/docker logs?
internal_view_log_command = 'cat /tmp/{}_data/task.log'.format(task.uuid) internal_view_log_command = 'cat /tmp/{}_data/task.log'.format(task.uuid)
else: else:
raise NotImplementedError('Container engine {} not supported'.format(container_engine)) raise NotImplementedError('Container engine {} not supported'.format(container_engine))
# Prepare full comand # Prepare full comand
view_log_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "{}"\''.format(computing_port, computing_keys.private_key_file, computing_user, computing_host, internal_view_log_command) view_log_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "{}"\''.format(computing_port, computing_keys.private_key_file, computing_user, computing_host, internal_view_log_command)
...@@ -426,7 +460,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana ...@@ -426,7 +460,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManager): class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManager):
def _start_task(self, task, **kwargs): def _start_task(self, task, **kwargs):
logger.debug('Starting a remote task "{}"'.format(self.computing)) logger.debug('Starting a remote task "{}"'.format(self.computing))
...@@ -439,7 +473,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag ...@@ -439,7 +473,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
# Initialize sbatch args (force 1 task for now) # Initialize sbatch args (force 1 task for now)
sbatch_args = '-N1 ' sbatch_args = '-N1 '
# Get task computing parameters and set sbatch args # Get task computing parameters and set sbatch args
if task.computing_options: if task.computing_options:
task_partition = task.computing_options.get('partition', None) task_partition = task.computing_options.get('partition', None)
...@@ -464,7 +498,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag ...@@ -464,7 +498,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
if not container_engine: if not container_engine:
container_engine = task.computing.default_container_engine container_engine = task.computing.default_container_engine
# engine-specific part # engine-specific part
if container_engine == 'singularity': if container_engine == 'singularity':
#if not task.container.supports_custom_interface_port: #if not task.container.supports_custom_interface_port:
...@@ -480,20 +514,20 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag ...@@ -480,20 +514,20 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
varsstring = '' varsstring = ''
# Sanitize again just in case the DB got somehow compromised: # Sanitize again just in case the DB got somehow compromised:
env_vars = sanitize_container_env_vars(task.container.env_vars) env_vars = sanitize_container_env_vars(task.container.env_vars)
for env_var in env_vars: for env_var in env_vars:
varsstring += ' && export SINGULARITYENV_{}={} '.format(env_var, env_vars[env_var]) varsstring += ' && export SINGULARITYENV_{}={} '.format(env_var, env_vars[env_var])
else: else:
varsstring = '' varsstring = ''
# Handle storages (binds) # Handle storages (binds)
binds = '' binds = ''
storages = Storage.objects.filter(computing=self.computing) storages = Storage.objects.filter(computing=self.computing)
for storage in storages: for storage in storages:
if storage.type == 'generic_posix' and storage.bind_path: if storage.type == 'generic_posix' and storage.bind_path:
# Expand the base path # Expand the base path
expanded_base_path = storage.base_path expanded_base_path = storage.base_path
if '$SSH_USER' in expanded_base_path: if '$SSH_USER' in expanded_base_path:
if storage.access_through_computing: if storage.access_through_computing:
expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user) expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user)
...@@ -501,9 +535,9 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag ...@@ -501,9 +535,9 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented') raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_base_path: if '$USER' in expanded_base_path:
expanded_base_path = expanded_base_path.replace('$USER', task.user.username) expanded_base_path = expanded_base_path.replace('$USER', task.user.username)
# Expand the bind_path # Expand the bind_path
expanded_bind_path = storage.bind_path expanded_bind_path = storage.bind_path
if '$SSH_USER' in expanded_bind_path: if '$SSH_USER' in expanded_bind_path:
if storage.access_through_computing: if storage.access_through_computing:
expanded_bind_path = expanded_bind_path.replace('$SSH_USER', computing_user) expanded_bind_path = expanded_bind_path.replace('$SSH_USER', computing_user)
...@@ -511,7 +545,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag ...@@ -511,7 +545,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented') raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_bind_path: if '$USER' in expanded_bind_path:
expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username) expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username)
# Add the bind # Add the bind
if not binds: if not binds:
binds = '-B {}:{}'.format(expanded_base_path, expanded_bind_path) binds = '-B {}:{}'.format(expanded_base_path, expanded_bind_path)
...@@ -519,11 +553,11 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag ...@@ -519,11 +553,11 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path) binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)
run_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_port, computing_keys.private_key_file, computing_user, computing_host) run_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_port, computing_keys.private_key_file, computing_user, computing_host)
run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {} {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid) run_command += '\'bash -c "echo \\"#!/bin/bash\ncurl {} {}/api/v1/base/agent/?task_uuid={} -o \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python3 \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(CHECK_CURL_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT {} {} && '.format(authstring, varsstring) run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT {} {} && '.format(authstring, varsstring)
run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid) run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid) run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)
# Double to escape for Python, six for shell (double times three as \\\ escapes a single slash in shell) # Double to escape for Python, six for shell (double times three as \\\ escapes a single slash in shell)
run_command+='docker://{}/{}:{} &> \$HOME/{}.log\\" > \$HOME/{}.sh && sbatch {} \$HOME/{}.sh"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid, task.uuid, sbatch_args, task.uuid) run_command+='docker://{}/{}:{} &> \$HOME/{}.log\\" > \$HOME/{}.sh && sbatch {} \$HOME/{}.sh"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid, task.uuid, sbatch_args, task.uuid)
...@@ -535,7 +569,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag ...@@ -535,7 +569,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
logger.error('Got error in starting task: {}'.format(out)) logger.error('Got error in starting task: {}'.format(out))
raise Exception(out.stderr) raise Exception(out.stderr)
# Log # Log
logger.debug('Shell exec output: "{}"'.format(out)) logger.debug('Shell exec output: "{}"'.format(out))
# Parse sbatch output. Example: Output(stdout='Submitted batch job 3', stderr='', exit_code=0) # Parse sbatch output. Example: Output(stdout='Submitted batch job 3', stderr='', exit_code=0)
...@@ -544,24 +578,27 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag ...@@ -544,24 +578,27 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
int(job_id) int(job_id)
except: except:
raise Exception('Cannot find int job id from output string "{}"'.format(out.stdout)) raise Exception('Cannot find int job id from output string "{}"'.format(out.stdout))
# Load back the task to avoid concurrency problems in the agent call # Load back the task to avoid concurrency problems in the agent call
task_uuid = task.uuid task_uuid = task.uuid
task = Task.objects.get(uuid=task_uuid) task = Task.objects.get(uuid=task_uuid)
# Save job id # Save job id
task.job_id = job_id task.job_id = job_id
# Set status (only fi we get here before the agent which sets the status as running via the API) # Set status (only fi we get here before the agent which sets the status as running via the API)
if task.status != TaskStatuses.running: if task.status != TaskStatuses.running:
task.status = TaskStatuses.sumbitted task.status = TaskStatuses.sumbitted
# Save # Save
task.save() task.save()
# Setup the tunnel if using a custom protocol (otherwise it will get set up via the "connect" button)
if task.container.interface_protocol not in ['http', 'https']:
setup_tunnel_and_proxy(task)
def _stop_task(self, task, **kwargs): def _stop_task(self, task, **kwargs):
# Get credentials # Get credentials
computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user) computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)
...@@ -570,14 +607,14 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag ...@@ -570,14 +607,14 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
out = os_shell(stop_command, capture=True) out = os_shell(stop_command, capture=True)
if out.exit_code != 0: if out.exit_code != 0:
raise Exception(out.stderr) raise Exception(out.stderr)
# Set task as topped # Set task as topped
task.status = TaskStatuses.stopped task.status = TaskStatuses.stopped
task.save() task.save()
def _get_task_log(self, task, **kwargs): def _get_task_log(self, task, **kwargs):
# Get credentials # Get credentials
computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user) computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)
...@@ -590,4 +627,3 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag ...@@ -590,4 +627,3 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
else: else:
return out.stdout return out.stdout
...@@ -51,7 +51,7 @@ def public_view(wrapped_view): ...@@ -51,7 +51,7 @@ def public_view(wrapped_view):
error_text = str(e) error_text = str(e)
else: else:
# Log the exception # Log the exception
logger.error(format_exception(e)) logger.error(format_exception(e))
# Raise the exception if we are in debug mode # Raise the exception if we are in debug mode
...@@ -109,13 +109,13 @@ def private_view(wrapped_view): ...@@ -109,13 +109,13 @@ def private_view(wrapped_view):
error_text = str(e) error_text = str(e)
else: else:
# Log the exception # Log the exception
logger.error(format_exception(e)) logger.error(format_exception(e))
# Raise the exception if we are in debug mode # Raise the exception if we are in debug mode
if settings.DEBUG: if settings.DEBUG:
raise raise
# Otherwise, mask it # Otherwise, mask it
else: else:
error_text = 'something went wrong ({})'.format(e) error_text = 'something went wrong ({})'.format(e)
...@@ -137,3 +137,4 @@ def private_view(wrapped_view): ...@@ -137,3 +137,4 @@ def private_view(wrapped_view):
response.set_cookie('post_login_redirect', request.build_absolute_uri()) response.set_cookie('post_login_redirect', request.build_absolute_uri())
return response return response
return private_view_wrapper return private_view_wrapper
...@@ -3,4 +3,4 @@ class ErrorMessage(Exception): ...@@ -3,4 +3,4 @@ class ErrorMessage(Exception):
pass pass
class ConsistencyException(Exception): class ConsistencyException(Exception):
pass pass
\ No newline at end of file
...@@ -26,4 +26,4 @@ class JSONField(Field): ...@@ -26,4 +26,4 @@ class JSONField(Field):
def value_to_string(self, obj): def value_to_string(self, obj):
return self.value_from_object(obj) return self.value_from_object(obj)
# Credits: https://medium.com/@philamersune/using-postgresql-jsonfield-in-sqlite-95ad4ad2e5f1 # Credits: https://medium.com/@philamersune/using-postgresql-jsonfield-in-sqlite-95ad4ad2e5f1
\ No newline at end of file
...@@ -8,13 +8,13 @@ class Command(BaseCommand): ...@@ -8,13 +8,13 @@ class Command(BaseCommand):
def handle(self, *args, **options): def handle(self, *args, **options):
#===================== #=====================
# Testuser # Testuser
#===================== #=====================
try: try:
testuser = User.objects.get(username='testuser') testuser = User.objects.get(username='testuser')
print('Not creating test user as it already exists') print('Not creating test user as it already exists')
except User.DoesNotExist: except User.DoesNotExist:
print('Creating test user with default password') print('Creating test user with default password')
testuser = User.objects.create_user('testuser', 'testuser@rosetta.platform', 'testpass') testuser = User.objects.create_user('testuser', 'testuser@rosetta.platform', 'testpass')
...@@ -22,7 +22,7 @@ class Command(BaseCommand): ...@@ -22,7 +22,7 @@ class Command(BaseCommand):
testuser.is_staff = True testuser.is_staff = True
testuser.is_admin=True testuser.is_admin=True
testuser.is_superuser=True testuser.is_superuser=True
testuser.save() testuser.save()
print('Creating testuser profile') print('Creating testuser profile')
Profile.objects.create(user=testuser, auth='local', authtoken='129aac94-284a-4476-953c-ffa4349b4a50') Profile.objects.create(user=testuser, auth='local', authtoken='129aac94-284a-4476-953c-ffa4349b4a50')
...@@ -32,23 +32,23 @@ class Command(BaseCommand): ...@@ -32,23 +32,23 @@ class Command(BaseCommand):
default = True, default = True,
private_key_file = '/rosetta/.ssh/id_rsa', private_key_file = '/rosetta/.ssh/id_rsa',
public_key_file = '/rosetta/.ssh/id_rsa.pub') public_key_file = '/rosetta/.ssh/id_rsa.pub')
#=====================
#=====================
# Platform keys # Platform keys
#===================== #=====================
# TODO: create a different pair # TODO: create a different pair
try: try:
KeyPair.objects.get(user=None, default=True) KeyPair.objects.get(user=None, default=True)
print('Not creating default platform keys as they already exist') print('Not creating default platform keys as they already exist')
except KeyPair.DoesNotExist: except KeyPair.DoesNotExist:
print('Creating platform default keys') print('Creating platform default keys')
KeyPair.objects.create(user = None, KeyPair.objects.create(user = None,
default = True, default = True,
private_key_file = '/rosetta/.ssh/id_rsa', private_key_file = '/rosetta/.ssh/id_rsa',
public_key_file = '/rosetta/.ssh/id_rsa.pub') public_key_file = '/rosetta/.ssh/id_rsa.pub')
#===================== #=====================
# Default home page # Default home page
...@@ -60,7 +60,7 @@ class Command(BaseCommand): ...@@ -60,7 +60,7 @@ class Command(BaseCommand):
<h1>&nbsp;&nbsp;Rosetta <img src="/static/img/emoji_u1f6f0.png" style="height:84px; width:64px; padding-bottom:20px"></h1> <h1>&nbsp;&nbsp;Rosetta <img src="/static/img/emoji_u1f6f0.png" style="height:84px; width:64px; padding-bottom:20px"></h1>
<h2 style="margin-top:10px; margin-left:25px; margin-right:25px; font-weight:100; line-height: 30px;"><i>A container-centric Science Platform<br></i></h2> <h2 style="margin-top:10px; margin-left:25px; margin-right:25px; font-weight:100; line-height: 30px;"><i>A container-centric Science Platform<br></i></h2>
</div> </div>
</div> </div>
<div class="container"> <div class="container">
<div class="dashboard"> <div class="dashboard">
<div class="span8 offset2" style="margin: 30px auto; max-width:800px"> <div class="span8 offset2" style="margin: 30px auto; max-width:800px">
...@@ -69,7 +69,7 @@ class Command(BaseCommand): ...@@ -69,7 +69,7 @@ class Command(BaseCommand):
This is the default main page content loaded after populating the platform with the default/demo data. This is the default main page content loaded after populating the platform with the default/demo data.
To change it, head to the <a href="/admin">admin</a> section and edit the <code>Page</code> model with id "main". To change it, head to the <a href="/admin">admin</a> section and edit the <code>Page</code> model with id "main".
<br/><br/> <br/><br/>
A test user with admin rights registered with email <code>testuser@rosetta.platform</code> and password A test user with admin rights registered with email <code>testuser@rosetta.platform</code> and password
<code>testpass</code> has been created as well, which you can use to login on the menu on the right and give Rosetta <code>testpass</code> has been created as well, which you can use to login on the menu on the right and give Rosetta
immediately a try. If you are using the default docker-compose file (i.e. you just ran <code>rosetta/setup</code>), immediately a try. If you are using the default docker-compose file (i.e. you just ran <code>rosetta/setup</code>),
then you will also have a few demo computing and storage resources (beside the internal one) already available then you will also have a few demo computing and storage resources (beside the internal one) already available
...@@ -79,10 +79,10 @@ class Command(BaseCommand): ...@@ -79,10 +79,10 @@ class Command(BaseCommand):
<br /> <br />
You can also create custom pages and access them under <code>/pages/page_id</code> should you need to provide You can also create custom pages and access them under <code>/pages/page_id</code> should you need to provide
your users informations about the platform and its storage and computing resources. For example, see this your users informations about the platform and its storage and computing resources. For example, see this
demo extra <a href="/pages/help">help page</a>. demo extra <a href="/pages/help">help page</a>.
</div> </div>
</div> </div>
</div> </div>
</header> </header>
''' '''
home_page = Page.objects.filter(id='main') home_page = Page.objects.filter(id='main')
...@@ -96,15 +96,15 @@ class Command(BaseCommand): ...@@ -96,15 +96,15 @@ class Command(BaseCommand):
<h1>Help!</h1> <h1>Help!</h1>
<hr> <hr>
<p> <p>
This is a demo extra page (a help page, in this case). Here you could for example provide the instructions on how to set up SSH-based This is a demo extra page (a help page, in this case). Here you could for example provide the instructions on how to set up SSH-based
computing resources using user keys, or who to contact to join a specific group to access its software and computing resources. computing resources using user keys, or who to contact to join a specific group to access its software and computing resources.
</p> </p>
<p> <p>
In general, the part of the URL following the <code>/pages/</code> path is parsed as the page id, In general, the part of the URL following the <code>/pages/</code> path is parsed as the page id,
so that if a page with that id exists in the database, its content will show up here. so that if a page with that id exists in the database, its content will show up here.
You can use this system for creating a mini-website inside the platform You can use this system for creating a mini-website inside the platform
to provide help, news and informations on your deployment. Or you can just ignore the whole thing and leave a plain logo in the main page. to provide help, news and informations on your deployment. Or you can just ignore the whole thing and leave a plain logo in the main page.
</p> </p>
''' '''
...@@ -117,16 +117,16 @@ to provide help, news and informations on your deployment. Or you can just ignor ...@@ -117,16 +117,16 @@ to provide help, news and informations on your deployment. Or you can just ignor
#===================== #=====================
# Platform containers # Platform containers
#===================== #=====================
platform_containers = Container.objects.filter(user=None) platform_containers = Container.objects.filter(user=None)
if platform_containers: if platform_containers:
print('Not creating public containers as they already exist') print('Not creating public containers as they already exist')
else: else:
print('Creating platform containers...') print('Creating platform containers...')
# Minimal Desktop # Minimal Desktop
Container.objects.create(user = None, Container.objects.create(user = None,
name = 'Minimal Desktop', name = 'Minimal Desktop',
...@@ -157,8 +157,8 @@ to provide help, news and informations on your deployment. Or you can just ignor ...@@ -157,8 +157,8 @@ to provide help, news and informations on your deployment. Or you can just ignor
supports_custom_interface_port = True, supports_custom_interface_port = True,
supports_interface_auth = True, supports_interface_auth = True,
interface_auth_user = None) interface_auth_user = None)
# Jupyter Notebook # Jupyter Notebook
Container.objects.create(user = None, Container.objects.create(user = None,
name = 'Jupyter Notebook', name = 'Jupyter Notebook',
...@@ -178,7 +178,7 @@ to provide help, news and informations on your deployment. Or you can just ignor ...@@ -178,7 +178,7 @@ to provide help, news and informations on your deployment. Or you can just ignor
# Official Jupyter containers # Official Jupyter containers
for tag in ['lab-3.2.2', 'lab-3.1.17']: for tag in ['lab-3.2.2', 'lab-3.1.17']:
Container.objects.create(user = None, Container.objects.create(user = None,
name = 'Jupyter Data Science Lab', name = 'Jupyter Data Science Lab',
description = 'The official Jupyter Lab. The Data Science variant, which includes libraries for data analysis from the Julia, Python, and R communities.', description = 'The official Jupyter Lab. The Data Science variant, which includes libraries for data analysis from the Julia, Python, and R communities.',
...@@ -192,7 +192,7 @@ to provide help, news and informations on your deployment. Or you can just ignor ...@@ -192,7 +192,7 @@ to provide help, news and informations on your deployment. Or you can just ignor
interface_transport = 'tcp/ip', interface_transport = 'tcp/ip',
supports_custom_interface_port = True, supports_custom_interface_port = True,
supports_interface_auth = True) supports_interface_auth = True)
for arch in ['amd64', 'arm64']: for arch in ['amd64', 'arm64']:
Container.objects.create(user = None, Container.objects.create(user = None,
name = 'Jupyter Lab', name = 'Jupyter Lab',
...@@ -225,9 +225,9 @@ to provide help, news and informations on your deployment. Or you can just ignor ...@@ -225,9 +225,9 @@ to provide help, news and informations on your deployment. Or you can just ignor
supports_interface_auth = True, supports_interface_auth = True,
interface_auth_user = 'metauser') interface_auth_user = 'metauser')
#===================== #=====================
# Testuser containers # Testuser containers
#===================== #=====================
#testuser_containers = Container.objects.filter(user=testuser) #testuser_containers = Container.objects.filter(user=testuser)
#if testuser_containers: #if testuser_containers:
# print('Not creating testuser private containers as they already exist') # print('Not creating testuser private containers as they already exist')
...@@ -250,9 +250,9 @@ to provide help, news and informations on your deployment. Or you can just ignor ...@@ -250,9 +250,9 @@ to provide help, news and informations on your deployment. Or you can just ignor
# supports_interface_auth = False) # supports_interface_auth = False)
#===================== #=====================
# Computing resources # Computing resources
#===================== #=====================
computing_resources = Computing.objects.all() computing_resources = Computing.objects.all()
if computing_resources: if computing_resources:
print('Not creating demo computing resources as they already exist') print('Not creating demo computing resources as they already exist')
...@@ -270,7 +270,7 @@ to provide help, news and informations on your deployment. Or you can just ignor ...@@ -270,7 +270,7 @@ to provide help, news and informations on your deployment. Or you can just ignor
wms = None, wms = None,
container_engines = ['docker']) container_engines = ['docker'])
# Demo standalone computing plus conf # Demo standalone computing plus conf
demo_standalone_computing = Computing.objects.create(name = 'Demo Standalone', demo_standalone_computing = Computing.objects.create(name = 'Demo Standalone',
description = 'A demo standalone computing resource.', description = 'A demo standalone computing resource.',
...@@ -311,22 +311,22 @@ to provide help, news and informations on your deployment. Or you can just ignor ...@@ -311,22 +311,22 @@ to provide help, news and informations on your deployment. Or you can just ignor
wms = 'slurm', wms = 'slurm',
conf = {'host': 'slurmclustermaster', 'default_partition': 'partition1'}, conf = {'host': 'slurmclustermaster', 'default_partition': 'partition1'},
container_engines = ['singularity']) container_engines = ['singularity'])
# Add testuser extra conf for this computing resource # Add testuser extra conf for this computing resource
testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_slurm_computing, value= 'testuser') testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_slurm_computing, value= 'testuser')
#===================== #=====================
# Storages # Storages
#===================== #=====================
storages = Storage.objects.all() storages = Storage.objects.all()
if storages: if storages:
print('Not creating demo storage resources as they already exist') print('Not creating demo storage resources as they already exist')
else: else:
print('Creating demo storage resources...') print('Creating demo storage resources...')
# Get demo computing resources # Get demo computing resources
demo_computing_resources = [] demo_computing_resources = []
try: try:
demo_slurm_computing = Computing.objects.get(name='Demo Cluster') demo_slurm_computing = Computing.objects.get(name='Demo Cluster')
demo_computing_resources.append(demo_slurm_computing) demo_computing_resources.append(demo_slurm_computing)
except: except:
...@@ -336,8 +336,8 @@ to provide help, news and informations on your deployment. Or you can just ignor ...@@ -336,8 +336,8 @@ to provide help, news and informations on your deployment. Or you can just ignor
demo_computing_resources.append(demo_standalone_computing) demo_computing_resources.append(demo_standalone_computing)
except: except:
pass pass
for computing in demo_computing_resources: for computing in demo_computing_resources:
# Demo shared storage # Demo shared storage
Storage.objects.create(computing = computing, Storage.objects.create(computing = computing,
...@@ -348,7 +348,7 @@ to provide help, news and informations on your deployment. Or you can just ignor ...@@ -348,7 +348,7 @@ to provide help, news and informations on your deployment. Or you can just ignor
auth_mode = 'user_keys', auth_mode = 'user_keys',
base_path = '/shared/data/shared', base_path = '/shared/data/shared',
bind_path = '/storages/shared') bind_path = '/storages/shared')
# Demo personal storage # Demo personal storage
Storage.objects.create(computing = computing, Storage.objects.create(computing = computing,
access_through_computing = True, access_through_computing = True,
...@@ -359,3 +359,4 @@ to provide help, news and informations on your deployment. Or you can just ignor ...@@ -359,3 +359,4 @@ to provide help, news and informations on your deployment. Or you can just ignor
base_path = '/shared/data/users/$SSH_USER', base_path = '/shared/data/users/$SSH_USER',
bind_path = '/storages/personal') bind_path = '/storages/personal')
...@@ -37,7 +37,7 @@ class TaskStatuses(object): ...@@ -37,7 +37,7 @@ class TaskStatuses(object):
# using an UUID pointing to some other model instead of the value in future, should this be necessary. # using an UUID pointing to some other model instead of the value in future, should this be necessary.
#========================= #=========================
# Profile # Profile
#========================= #=========================
class Profile(models.Model): class Profile(models.Model):
...@@ -68,25 +68,25 @@ class Profile(models.Model): ...@@ -68,25 +68,25 @@ class Profile(models.Model):
self.extra_confs = {} self.extra_confs = {}
self.extra_confs[str(uuid.uuid4())] = {'type': conf_type, 'object_uuid': str(object.uuid), 'value': value} self.extra_confs[str(uuid.uuid4())] = {'type': conf_type, 'object_uuid': str(object.uuid), 'value': value}
self.save() self.save()
def get_extra_conf(self, conf_type, object=None): def get_extra_conf(self, conf_type, object=None):
if self.extra_confs: if self.extra_confs:
for extra_conf in self.extra_confs: for extra_conf in self.extra_confs:
if conf_type == self.extra_confs[extra_conf]['type']: if conf_type == self.extra_confs[extra_conf]['type']:
if object: if object:
#logger.debug("{} vs {}".format(self.extra_confs[extra_conf]['object_uuid'], str(object.uuid))) #logger.debug("{} vs {}".format(self.extra_confs[extra_conf]['object_uuid'], str(object.uuid)))
if self.extra_confs[extra_conf]['object_uuid'] == str(object.uuid): if self.extra_confs[extra_conf]['object_uuid'] == str(object.uuid):
return self.extra_confs[extra_conf]['value'] return self.extra_confs[extra_conf]['value']
else: else:
return self.extra_confs[extra_conf]['value'] return self.extra_confs[extra_conf]['value']
return None return None
#========================= #=========================
# Login Token # Login Token
#========================= #=========================
class LoginToken(models.Model): class LoginToken(models.Model):
...@@ -106,7 +106,7 @@ class LoginToken(models.Model): ...@@ -106,7 +106,7 @@ class LoginToken(models.Model):
class Container(models.Model): class Container(models.Model):
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
user = models.ForeignKey(User, related_name='containers', on_delete=models.CASCADE, blank=True, null=True) user = models.ForeignKey(User, related_name='containers', on_delete=models.CASCADE, blank=True, null=True)
# If a container has no user, it will be available to anyone. Can be created, edited and deleted only by admins. # If a container has no user, it will be available to anyone. Can be created, edited and deleted only by admins.
group = models.ForeignKey(Group, related_name='containers', on_delete=models.CASCADE, blank=True, null=True) group = models.ForeignKey(Group, related_name='containers', on_delete=models.CASCADE, blank=True, null=True)
# If a container has no group, it will be available to anyone. Can be created, edited and deleted only by admins. # If a container has no group, it will be available to anyone. Can be created, edited and deleted only by admins.
...@@ -114,25 +114,25 @@ class Container(models.Model): ...@@ -114,25 +114,25 @@ class Container(models.Model):
# Generic attributes # Generic attributes
name = models.CharField('Name', max_length=255, blank=False, null=False) name = models.CharField('Name', max_length=255, blank=False, null=False)
description = models.TextField('Description', blank=True, null=True) description = models.TextField('Description', blank=True, null=True)
# Registry # Registry
registry = models.CharField('Registry', max_length=255, blank=False, null=False) registry = models.CharField('Registry', max_length=255, blank=False, null=False)
# Image name # Image name
image_name = models.CharField('Image', max_length=255, blank=False, null=False) image_name = models.CharField('Image', max_length=255, blank=False, null=False)
# Image identifiers # Image identifiers
image_tag = models.CharField('Tag', max_length=255, blank=False, null=False, default='latest') image_tag = models.CharField('Tag', max_length=255, blank=False, null=False, default='latest')
image_arch = models.CharField('Architecture', max_length=36, blank=True, null=True) image_arch = models.CharField('Architecture', max_length=36, blank=True, null=True)
image_os = models.CharField('Operating system', max_length=36, blank=True, null=True) image_os = models.CharField('Operating system', max_length=36, blank=True, null=True)
# -- OR -- # -- OR --
image_digest = models.CharField('SHA 256 digest', max_length=96, blank=True, null=True) image_digest = models.CharField('SHA 256 digest', max_length=96, blank=True, null=True)
# TODO: do we want more control with respect to kernel, CPUs, instruction sets? # TODO: do we want more control with respect to kernel, CPUs, instruction sets?
# requires = i.e. kernel > 3, intel, AVX2 # requires = i.e. kernel > 3, intel, AVX2
# Port, protocol and transport for the container interface # Port, protocol and transport for the container interface
interface_port = models.IntegerField('Interface port', blank=True, null=True) interface_port = models.IntegerField('Interface port', blank=True, null=True)
interface_protocol = models.CharField('Interface protocol', max_length=36, blank=True, null=True) interface_protocol = models.CharField('Interface protocol', max_length=36, blank=True, null=True)
interface_transport = models.CharField('Interface transport', max_length=36, blank=True, null=True) interface_transport = models.CharField('Interface transport', max_length=36, blank=True, null=True)
...@@ -156,7 +156,7 @@ class Container(models.Model): ...@@ -156,7 +156,7 @@ class Container(models.Model):
# Check that digest starts with sha256: # Check that digest starts with sha256:
if self.image_digest and not self.image_digest.startswith('sha256:'): if self.image_digest and not self.image_digest.startswith('sha256:'):
raise ValueError('The digest field must start with "sha256:"') raise ValueError('The digest field must start with "sha256:"')
super(Container, self).save(*args, **kwargs) super(Container, self).save(*args, **kwargs)
@property @property
...@@ -183,7 +183,7 @@ class Computing(models.Model): ...@@ -183,7 +183,7 @@ class Computing(models.Model):
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
group = models.ForeignKey(Group, related_name='computings', on_delete=models.CASCADE, blank=True, null=True) group = models.ForeignKey(Group, related_name='computings', on_delete=models.CASCADE, blank=True, null=True)
# If a compute resource has no group, it will be available to anyone. Can be created, edited and deleted only by admins. # If a compute resource has no group, it will be available to anyone. Can be created, edited and deleted only by admins.
name = models.CharField('Name', max_length=255, blank=False, null=False) name = models.CharField('Name', max_length=255, blank=False, null=False)
description = models.TextField('Description', blank=True, null=True) description = models.TextField('Description', blank=True, null=True)
...@@ -195,16 +195,16 @@ class Computing(models.Model): ...@@ -195,16 +195,16 @@ class Computing(models.Model):
access_mode = models.CharField('Access (control) mode', max_length=36, blank=False, null=False) access_mode = models.CharField('Access (control) mode', max_length=36, blank=False, null=False)
auth_mode = models.CharField('Auth mode', max_length=36, blank=False, null=False) auth_mode = models.CharField('Auth mode', max_length=36, blank=False, null=False)
wms = models.CharField('Workload management system', max_length=36, blank=True, null=True) wms = models.CharField('Workload management system', max_length=36, blank=True, null=True)
# Supported container engines (e.g. ['docker', 'singularity', 'podman']) # Supported container engines (e.g. ['docker', 'singularity', 'podman'])
container_engines = JSONField('Container engines/runtimes', blank=False, null=False) container_engines = JSONField('Container engines/runtimes', blank=False, null=False)
#container_runtime = models.CharField('Container runtimes', max_length=256, blank=False, null=False) #container_runtime = models.CharField('Container runtimes', max_length=256, blank=False, null=False)
# Supported architectures (i.e. 386 for amd64), as list: ['386'] # Supported architectures (i.e. 386 for amd64), as list: ['386']
supported_archs = JSONField('Supported architectures', blank=True, null=True) supported_archs = JSONField('Supported architectures', blank=True, null=True)
# Emulated architectures, by container runtime: {'docker': ['arm64/v7', 'arm64/v8'] # Emulated architectures, by container runtime: {'docker': ['arm64/v7', 'arm64/v8']
emulated_archs = JSONField('Emulated architectures', blank=True, null=True) emulated_archs = JSONField('Emulated architectures', blank=True, null=True)
# Conf # Conf
conf = JSONField(blank=True, null=True) conf = JSONField(blank=True, null=True)
...@@ -233,23 +233,23 @@ class Computing(models.Model): ...@@ -233,23 +233,23 @@ class Computing(models.Model):
@property @property
def default_container_engine(self): def default_container_engine(self):
return self.container_engines[0] return self.container_engines[0]
#======================= #=======================
# Computing manager # Computing manager
#======================= #=======================
@property @property
def manager(self): def manager(self):
from . import computing_managers from . import computing_managers
# Hash table mapping # Hash table mapping
managers_mapping = {} managers_mapping = {}
managers_mapping['cluster'+'ssh+cli'+'user_keys'+'slurm'] = computing_managers.SlurmSSHClusterComputingManager managers_mapping['cluster'+'ssh+cli'+'user_keys'+'slurm'] = computing_managers.SlurmSSHClusterComputingManager
managers_mapping['standalone'+'ssh+cli'+'user_keys'+'None'] = computing_managers.SSHStandaloneComputingManager managers_mapping['standalone'+'ssh+cli'+'user_keys'+'None'] = computing_managers.SSHStandaloneComputingManager
managers_mapping['standalone'+'ssh+cli'+'platform_keys'+'None'] = computing_managers.SSHStandaloneComputingManager managers_mapping['standalone'+'ssh+cli'+'platform_keys'+'None'] = computing_managers.SSHStandaloneComputingManager
managers_mapping['standalone'+'internal'+'internal'+'None'] = computing_managers.InternalStandaloneComputingManager managers_mapping['standalone'+'internal'+'internal'+'None'] = computing_managers.InternalStandaloneComputingManager
# Instantiate the computing manager and return (if not already done) # Instantiate the computing manager and return (if not already done)
try: try:
return self._manager return self._manager
...@@ -262,10 +262,10 @@ class Computing(models.Model): ...@@ -262,10 +262,10 @@ class Computing(models.Model):
else: else:
return self._manager return self._manager
#========================= #=========================
# Tasks # Tasks
#========================= #=========================
class Task(models.Model): class Task(models.Model):
...@@ -277,16 +277,16 @@ class Task(models.Model): ...@@ -277,16 +277,16 @@ class Task(models.Model):
# Task management # Task management
status = models.CharField('Status', max_length=36, blank=True, null=True) status = models.CharField('Status', max_length=36, blank=True, null=True)
created = models.DateTimeField('Created on', default=timezone.now) created = models.DateTimeField('Created on', default=timezone.now)
process_id = models.CharField('Process ID', max_length=64, blank=True, null=True) # i.e. Singularity PID process_id = models.CharField('Process ID', max_length=64, blank=True, null=True) # i.e. Singularity PID
job_id = models.CharField('Job ID', max_length=64, blank=True, null=True) # i.e. Slurm job id job_id = models.CharField('Job ID', max_length=64, blank=True, null=True) # i.e. Slurm job id
# How to reach the task interface. The IP has to be intended either as the container IP if this is directly # How to reach the task interface. The IP has to be intended either as the container IP if this is directly
# reachable (i.e. using a Docker or Kubernetes network) or as the host IP address, depending on the # reachable (i.e. using a Docker or Kubernetes network) or as the host IP address, depending on the
# computing resource and its computing manager/WMS/container runtime. The port is to be intended # computing resource and its computing manager/WMS/container runtime. The port is to be intended
# as the port where the task interface is exposed on its IP address. # as the port where the task interface is exposed on its IP address.
interface_ip = models.CharField('Interface IP address', max_length=36, blank=True, null=True) interface_ip = models.CharField('Interface IP address', max_length=36, blank=True, null=True)
interface_port = models.IntegerField('Interface port', blank=True, null=True) interface_port = models.IntegerField('Interface port', blank=True, null=True)
# Task access # Task access
requires_tcp_tunnel = models.BooleanField('Requires TCP tunnel') requires_tcp_tunnel = models.BooleanField('Requires TCP tunnel')
tcp_tunnel_port = models.IntegerField('TCP tunnel port', blank=True, null=True) tcp_tunnel_port = models.IntegerField('TCP tunnel port', blank=True, null=True)
...@@ -301,13 +301,13 @@ class Task(models.Model): ...@@ -301,13 +301,13 @@ class Task(models.Model):
# Computing options # Computing options
# TODO: add the option for selecting the runtime as advanced option when creating the task? # TODO: add the option for selecting the runtime as advanced option when creating the task?
computing_options = JSONField('Computing options', blank=True, null=True) # i.e. CPUs, RAM, cluster partition etc. TODO: why here? computing_options = JSONField('Computing options', blank=True, null=True) # i.e. CPUs, RAM, cluster partition etc. TODO: why here?
class Meta: class Meta:
ordering = ['-created'] ordering = ['-created']
def save(self, *args, **kwargs): def save(self, *args, **kwargs):
try: try:
getattr(TaskStatuses, str(self.status)) getattr(TaskStatuses, str(self.status))
except AttributeError: except AttributeError:
...@@ -318,25 +318,25 @@ class Task(models.Model): ...@@ -318,25 +318,25 @@ class Task(models.Model):
def update_status(self): def update_status(self):
if self.computing == 'local': if self.computing == 'local':
check_command = 'sudo docker inspect --format \'{{.State.Status}}\' ' + self.tid # or, .State.Running check_command = 'sudo docker inspect --format \'{{.State.Status}}\' ' + self.tid # or, .State.Running
out = os_shell(check_command, capture=True) out = os_shell(check_command, capture=True)
logger.debug('Status: "{}"'.format(out.stdout)) logger.debug('Status: "{}"'.format(out.stdout))
if out.exit_code != 0: if out.exit_code != 0:
if (('No such' in out.stderr) and (self.tid in out.stderr)): if (('No such' in out.stderr) and (self.tid in out.stderr)):
logger.debug('Task "{}" is not running in reality'.format(self.tid)) logger.debug('Task "{}" is not running in reality'.format(self.tid))
self.status = TaskStatuses.exited self.status = TaskStatuses.exited
else: else:
if out.stdout == 'running': if out.stdout == 'running':
self.status = TaskStatuses.running self.status = TaskStatuses.running
elif out.stdout == 'exited': elif out.stdout == 'exited':
self.status = TaskStatuses.exited self.status = TaskStatuses.exited
else: else:
raise Exception('Unknown task status: "{}"'.format(out.stdout)) raise Exception('Unknown task status: "{}"'.format(out.stdout))
self.save() self.save()
def __str__(self): def __str__(self):
...@@ -351,11 +351,11 @@ class Task(models.Model): ...@@ -351,11 +351,11 @@ class Task(models.Model):
string_int_hash = hash_string_to_int(self.name) string_int_hash = hash_string_to_int(self.name)
color_map_index = string_int_hash % len(color_map) color_map_index = string_int_hash % len(color_map)
return color_map[color_map_index] return color_map[color_map_index]
@property @property
def sharable_link(self): def sharable_link(self):
return 'https://{}/t/{}'.format(settings.ROSETTA_HOST, self.short_uuid) return 'https://{}/t/{}'.format(settings.ROSETTA_HOST, self.short_uuid)
@property @property
def tcp_tunnel_host(self): def tcp_tunnel_host(self):
return get_rosetta_tasks_tunnel_host() return get_rosetta_tasks_tunnel_host()
...@@ -369,61 +369,65 @@ class Task(models.Model): ...@@ -369,61 +369,65 @@ class Task(models.Model):
#========================= #=========================
class Storage(models.Model): class Storage(models.Model):
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
group = models.ForeignKey(Group, related_name='storages', on_delete=models.CASCADE, blank=True, null=True) group = models.ForeignKey(Group, related_name='storages', on_delete=models.CASCADE, blank=True, null=True)
name = models.CharField('Name', max_length=255, blank=False, null=False) name = models.CharField('Name', max_length=255, blank=False, null=False)
#description = models.TextField('Description', blank=True, null=True) #description = models.TextField('Description', blank=True, null=True)
# Storage type # Storage type
type = models.CharField('Type', max_length=255, blank=False, null=False) type = models.CharField('Type', max_length=255, blank=False, null=False)
# Access and auth mode # Access and auth mode
access_mode = models.CharField('Access (control) mode', max_length=36, blank=False, null=False) access_mode = models.CharField('Access (control) mode', max_length=36, blank=False, null=False)
auth_mode = models.CharField('Auth mode', max_length=36, blank=False, null=False) auth_mode = models.CharField('Auth mode', max_length=36, blank=False, null=False)
# Paths # Paths
base_path = models.CharField('Base path', max_length=4096, blank=False, null=False) base_path = models.CharField('Base path', max_length=4096, blank=False, null=False)
bind_path = models.CharField('Bind path', max_length=4096, blank=True, null=True) bind_path = models.CharField('Bind path', max_length=4096, blank=True, null=True)
# Link with a computing resource # Link with a computing resource
computing = models.ForeignKey(Computing, related_name='storages', on_delete=models.CASCADE, blank=True, null=True) # Make optional? computing = models.ForeignKey(Computing, related_name='storages', on_delete=models.CASCADE, blank=True, null=True) # Make optional?
access_through_computing = models.BooleanField('Access through linked computing resource?', default=False) access_through_computing = models.BooleanField('Access through linked computing resource?', default=False)
# If the above is linked, some configuration can be taken from the linked computing resource (i.e. the hostname) # If the above is linked, some configuration can be taken from the linked computing resource (i.e. the hostname)
# Configuration # Configuration
conf = JSONField(blank=True, null=True) conf = JSONField(blank=True, null=True)
# Include as browsable in the file manager? # Include as browsable in the file manager?
browsable = models.BooleanField('Browsable in the file manager?', default=True) browsable = models.BooleanField('Browsable in the file manager?', default=True)
def save(self, *args, **kwargs):
if self.access_mode == 'internal' and self.browsable:
raise ValueError('A storage with "internal" access mode cannot be marked as browsable since it is not yet supported by the file manager')
super(Storage, self).save(*args, **kwargs)
class Meta: class Meta:
ordering = ['name'] ordering = ['name']
def __str__(self): def __str__(self):
if self.group: if self.group:
return str('Storage "{}" of group "{}"'.format(self.id, self.group)) return str('Storage "{}" of group "{}"'.format(self.id, self.group))
else: else:
return str('Storage "{}"'.format(self.id)) return str('Storage "{}"'.format(self.id))
@property @property
def id(self): def id(self):
return (self.name if not self.computing else '{}:{}'.format(self.computing.name,self.name)) return (self.name if not self.computing else '{}:{}'.format(self.computing.name,self.name))
#========================= #=========================
# KeyPair # KeyPair
#========================= #=========================
class KeyPair(models.Model): class KeyPair(models.Model):
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
user = models.ForeignKey(User, related_name='key_pairs', on_delete=models.CASCADE, blank=True, null=True) user = models.ForeignKey(User, related_name='key_pairs', on_delete=models.CASCADE, blank=True, null=True)
private_key_file = models.CharField('Private key file', max_length=4096, blank=False, null=False) private_key_file = models.CharField('Private key file', max_length=4096, blank=False, null=False)
public_key_file = models.CharField('Public key file', max_length=4096, blank=False, null=False) public_key_file = models.CharField('Public key file', max_length=4096, blank=False, null=False)
...@@ -436,12 +440,12 @@ class KeyPair(models.Model): ...@@ -436,12 +440,12 @@ class KeyPair(models.Model):
return str('KeyPair of user "{}" (default={})'.format( self.user.email, self.default)) return str('KeyPair of user "{}" (default={})'.format( self.user.email, self.default))
else: else:
return str('KeyPair of user Platform (default={})'.format(self.default)) return str('KeyPair of user Platform (default={})'.format(self.default))
#========================= #=========================
# Page # Page
#========================= #=========================
class Page(models.Model): class Page(models.Model):
...@@ -457,3 +461,4 @@ class Page(models.Model): ...@@ -457,3 +461,4 @@ class Page(models.Model):
@keyframes spin {
to {
transform: rotate(360deg);
}
}
#navigate-away-loader {
position: fixed;
top: 0;
right: 0;
bottom: 0;
left: 0;
background-color: rgba(0, 0, 0, 0.5);
z-index: 2000;
align-items: center;
justify-content: center;
color: white;
transition: opacity 0.4s ease-in-out;
visibility: hidden;
pointer-events: none;
display: flex;
& > svg {
animation: spin 1.5s infinite linear;
opacity: 0.5;
}
}
function showNavigationLoader() {
var navigationLoader = document.querySelector("#navigate-away-loader")
navigationLoader.style.visibility = "visible"
navigationLoader.style.opacity = 1
navigationLoader.style.pointerEvents = "all"
document.querySelector("#navigate-away-loader > svg").style.animation =
"spin 1.5s infinite linear"
}
function hideNavigationLoader() {
var navigationLoader = document.querySelector("#navigate-away-loader")
navigationLoader.style.visibility = "hidden"
navigationLoader.style.opacity = 0
navigationLoader.style.pointerEvents = "none"
}
window.addEventListener("beforeunload", function (e) {
showNavigationLoader()
return true
})
window.addEventListener("pageshow", function (e) {
hideNavigationLoader()
})
{% load static %} {% load static %}
{% include "header.html" %} {% include "header.html" %}
{% include "navigation.html" %} {% include "navigation.html" %}
{% include "logo.html" %} {% include "logo.html" %}
...@@ -23,24 +23,24 @@ ...@@ -23,24 +23,24 @@
{{data.user.username}} {{data.user.username}}
</td> </td>
</tr> </tr>
<tr> <tr>
<td> <td>
<b>Email</b> <b>Email</b>
</td> </td>
<td> <td>
{% if data.edit == 'email' %} {% if data.edit == 'email' %}
<table class="frametab"><tr><td> <table class="frametab"><tr><td>
<input type='hidden' name='edit' value='email' /> <input type='hidden' name='edit' value='email' />
<input type="email" value="{{data.user.email}}" name='value' style='width:95%' required autofocus /> <input type="email" value="{{data.user.email}}" name='value' style='width:95%' required autofocus />
</td><td> </td><td>
<input type="submit" value="Go"> <input type="submit" value="Go">
</td></tr></table> </td></tr></table>
{% else %} {% else %}
{{data.user.email}}{% if data.user.profile.auth == 'local' %} | <a href="/account/?edit=email">Change</a>{% endif %} {{data.user.email}}{% if data.user.profile.auth == 'local' %} | <a href="/account/?edit=email">Change</a>{% endif %}
{% endif %} {% endif %}
</td> </td>
</tr> </tr>
{% if data.user.profile.auth == 'local' %} {% if data.user.profile.auth == 'local' %}
<tr> <tr>
...@@ -48,19 +48,19 @@ ...@@ -48,19 +48,19 @@
<b>Password</b> <b>Password</b>
</td> </td>
<td> <td>
{% if data.edit == 'password' %} {% if data.edit == 'password' %}
<table class="frametab"><tr><td> <table class="frametab"><tr><td>
<input type='hidden' name='edit' value='password' /> <input type='hidden' name='edit' value='password' />
<input type="password" name='value' style='width:95%' required autofocus /> <input type="password" name='value' style='width:95%' required autofocus />
</td><td> </td><td>
<input type="submit" value="Go"> <input type="submit" value="Go">
</td></tr></table> </td></tr></table>
{% else %} {% else %}
******* | <a href="/account/?edit=password">Change</a> ******* | <a href="/account/?edit=password">Change</a>
{% endif %} {% endif %}
</td> </td>
</tr> </tr>
{% endif %} {% endif %}
<tr> <tr>
...@@ -75,24 +75,24 @@ ...@@ -75,24 +75,24 @@
{% else %} {% else %}
{{ data.user.profile.auth }} {{ data.user.profile.auth }}
{% endif %} {% endif %}
</td> </td>
</tr> </tr>
</table> </table>
<br /> <br />
<h3>Profile</h3> <h3>Profile</h3>
<table class="dashboard"> <table class="dashboard">
<tr> <tr>
<td> <td>
<b>Time Zone</b> <b>Time Zone</b>
</td> </td>
<td> <td>
{% if data.edit == 'timezone' %} {% if data.edit == 'timezone' %}
{% include "components/tzselect.html" %} {% include "components/tzselect.html" %}
<input type='hidden' name='edit' value='timezone' /> <input type='hidden' name='edit' value='timezone' />
<input type="submit" value="Go"> <input type="submit" value="Go">
...@@ -101,7 +101,7 @@ ...@@ -101,7 +101,7 @@
{% endif %} {% endif %}
</td> </td>
</tr> </tr>
<tr> <tr>
<td valign=top> <td valign=top>
...@@ -117,23 +117,23 @@ ...@@ -117,23 +117,23 @@
{% endif %} {% endif %}
{% endfor %} {% endfor %}
{% endif %} {% endif %}
: <code>{{conf_data.value}}</code> | <a href='?delete_extra_conf_uuid={{conf_uuid}}'>delete</a> : <code>{{conf_data.value}}</code> | <a href='?delete_extra_conf_uuid={{conf_uuid}}'>delete</a>
<br/> <br/>
{% endfor %} {% endfor %}
<span style="margin:3px"><a href="/add_profile_conf" style="line-height:2em">Add new...</a></span> <span style="margin:3px"><a href="/add_profile_conf" style="line-height:2em">Add new...</a></span>
</td> </td>
</tr> </tr>
</table> </table>
<br /> <br />
<h3>KeyPair</h3> <h3>KeyPair</h3>
<table class="dashboard"> <table class="dashboard">
<tr> <tr>
<td valign="top"> <td valign="top">
<b>Default public key</b> <b>Default public key</b>
...@@ -145,7 +145,7 @@ ...@@ -145,7 +145,7 @@
</table> </table>
</form> </form>
<div style="margin-left:10px; margin-top:40px"> <div style="margin-left:10px; margin-top:40px">
{% if data.user.profile.auth == 'oidc' %} {% if data.user.profile.auth == 'oidc' %}
<form action="{% url 'oidc_logout' %}" method="post"> <form action="{% url 'oidc_logout' %}" method="post">
...@@ -155,18 +155,18 @@ ...@@ -155,18 +155,18 @@
{% else %} {% else %}
<form action="/logout/" method="get"> <form action="/logout/" method="get">
<input type="submit" value="logout"> <input type="submit" value="logout">
</form> </form>
{% endif %} {% endif %}
</div> </div>
<br/> <br/>
<br/> <br/>
<br/> <br/>
<br/> <br/>
<br/> <br/>
<br/> <br/>
</div> </div>
</div> </div>
</div> </div>
...@@ -178,3 +178,4 @@ ...@@ -178,3 +178,4 @@
{% load static %} {% load static %}
{% include "header.html" %} {% include "header.html" %}
{% include "navigation.html" %} {% include "navigation.html" %}
{% include "logo.html" %} {% include "logo.html" %}
...@@ -6,19 +6,19 @@ ...@@ -6,19 +6,19 @@
<div class="container"> <div class="container">
<div class="dashboard"> <div class="dashboard">
<div class="span8 offset2"> <div class="span8 offset2">
<h1>Add extra profile configuration</h1> <h1>Add extra profile configuration</h1>
<hr> <hr>
<h4>Chose the configuration type and add the values</h4> <h4>Chose the configuration type and add the values</h4>
<br/> <br/>
<form action="/add_profile_conf/" method="POST"> <form action="/add_profile_conf/" method="POST">
{% csrf_token %} {% csrf_token %}
<table class="dashboard" style="max-width:430px"> <table class="dashboard" style="max-width:430px">
<tr> <tr>
<td> <td>
{%if data.conf_type %} {%if data.conf_type %}
<select name="conf_type"> <select name="conf_type">
...@@ -29,7 +29,7 @@ ...@@ -29,7 +29,7 @@
{% for conf_type in data.conf_types %} {% for conf_type in data.conf_types %}
<option value="{{conf_type}}">{{conf_type}}</option> <option value="{{conf_type}}">{{conf_type}}</option>
{% endfor %} {% endfor %}
</select> </select>
{% endif %} {% endif %}
</td> </td>
...@@ -44,11 +44,11 @@ ...@@ -44,11 +44,11 @@
{% for computing in data.computings %} {% for computing in data.computings %}
<option value="{{computing.uuid}}">{{computing.name}}</option> <option value="{{computing.uuid}}">{{computing.name}}</option>
{% endfor %} {% endfor %}
</select> </select>
{% endif %} {% endif %}
</td> </td>
{% endif %} {% endif %}
{% if data.last_step %} {% if data.last_step %}
<td colspan=2 align=center style="padding:20px"> <td colspan=2 align=center style="padding:20px">
...@@ -62,11 +62,10 @@ ...@@ -62,11 +62,10 @@
<input type="submit" value="Next"> <input type="submit" value="Next">
</td> </td>
{% endif %} {% endif %}
</table> </table>
</form> </form>
<br/>
<br/> <br/>
<br/> <br/>
<br/> <br/>
...@@ -75,7 +74,8 @@ ...@@ -75,7 +74,8 @@
<br/> <br/>
<br/> <br/>
<br/> <br/>
<br/>
</div> </div>
</div> </div>
</div> </div>
...@@ -87,3 +87,4 @@ ...@@ -87,3 +87,4 @@
{% load static %} {% load static %}
{% include "header.html" %} {% include "header.html" %}
{% include "navigation.html" %} {% include "navigation.html" %}
{% include "logo.html" %} {% include "logo.html" %}
...@@ -6,24 +6,24 @@ ...@@ -6,24 +6,24 @@
<div class="container"> <div class="container">
<div class="dashboard"> <div class="dashboard">
<div class="span8 offset2"> <div class="span8 offset2">
<h1>Add software container</h1> <h1>Add software container</h1>
<hr> <hr>
{% if not data.added %} {% if not data.added %}
Here you can add a new software container on the platform. You can add containers from image registries Here you can add a new software container on the platform. You can add containers from image registries
as <a href="https://hub.docker.com/">Docker Hub </a>or by importing Git repositories, provided that they as <a href="https://hub.docker.com/">Docker Hub </a>or by importing Git repositories, provided that they
are compatible with <a href="https://mybinder.readthedocs.io/en/latest/introduction.html">Binder</a> specifications. are compatible with <a href="https://mybinder.readthedocs.io/en/latest/introduction.html">Binder</a> specifications.
<br/> <br/>
<br/> <br/>
{% if data.new_container_from == 'registry' %} {% if data.new_container_from == 'registry' %}
<div style="font-size:1.1em; background:whitesmoke; display:inline-block; padding:2px 15px 2px 15px">New container from registry</div> <div style="font-size:1.1em; background:whitesmoke; display:inline-block; padding:2px 15px 2px 15px">New container from registry</div>
<div style="font-size:1.1em; background:white; display:inline-block; padding:2px 15px 2px 15px"><a href="?new_container_from=repository">New container from Git repository</a></div> <div style="font-size:1.1em; background:white; display:inline-block; padding:2px 15px 2px 15px"><a href="?new_container_from=repository">New container from Git repository</a></div>
<hr style="margin-top:0;"> <hr style="margin-top:0;">
<h4>Basics</h4> <h4>Basics</h4>
<form action="#" method="POST"> <form action="#" method="POST">
{% csrf_token %} {% csrf_token %}
...@@ -43,7 +43,7 @@ ...@@ -43,7 +43,7 @@
<textarea name="container_description" rows="3" cols="22"></textarea> <textarea name="container_description" rows="3" cols="22"></textarea>
</td> </td>
</tr> </tr>
<tr> <tr>
<td><b>Registry</b></td><td> <td><b>Registry</b></td><td>
<input type="text" name="container_registry" value="docker.io" size="23" required /> <input type="text" name="container_registry" value="docker.io" size="23" required />
...@@ -68,7 +68,7 @@ ...@@ -68,7 +68,7 @@
<h4>Interface </h4> <h4>Interface </h4>
<table class="dashboard" style="width:400px; margin-bottom:25px"> <table class="dashboard" style="width:400px; margin-bottom:25px">
<tr> <tr>
...@@ -80,7 +80,7 @@ ...@@ -80,7 +80,7 @@
<td><b>Interface protocol</b></td> <td><b>Interface protocol</b></td>
<td> <td>
{% if request.user.profile.is_power_user %} {% if request.user.profile.is_power_user %}
<input type="text" value="http" name="container_interface_protocol" size="5" /> <input type="text" value="http" name="container_interface_protocol" size="5" />
{% else %} {% else %}
<select name="container_interface_protocol" > <select name="container_interface_protocol" >
<option value="http" selected>http</option> <option value="http" selected>http</option>
...@@ -91,13 +91,13 @@ ...@@ -91,13 +91,13 @@
</tr> </tr>
</table> </table>
<a href="javascript:void(0);" id="show_button" onclick="toggle_visibility('advanced_div')">Advanced...</a> <a href="javascript:void(0);" id="show_button" onclick="toggle_visibility('advanced_div')">Advanced...</a>
<div id="advanced_div" style="display:none; width:400px;"> <div id="advanced_div" style="display:none; width:400px;">
<h4>Advanced <font size=-1>| <a href="javascript:void(0);" id="hide_button" onclick="toggle_visibility('advanced_div')" style="display:none">hide</a></font></h4> <h4>Advanced <font size=-1>| <a href="javascript:void(0);" id="hide_button" onclick="toggle_visibility('advanced_div')" style="display:none">hide</a></font></h4>
<table class="dashboard" style="width:400px; margin-bottom:25px"> <table class="dashboard" style="width:400px; margin-bottom:25px">
<tr> <tr>
...@@ -130,21 +130,21 @@ ...@@ -130,21 +130,21 @@
<tr> <tr>
<td colspan=2> <td colspan=2>
<b>Supports custom interface port</b> &nbsp; <b>Supports custom interface port</b> &nbsp;
<input type="checkbox" name="container_supports_custom_interface_port" value="True" /> <input type="checkbox" name="container_supports_custom_interface_port" value="True" />
</td> </td>
</tr> </tr>
<tr> <tr>
<td colspan=2> <td colspan=2>
<b>Supports interface auth</b> &nbsp; <b>Supports interface auth</b> &nbsp;
<input type="checkbox" name="container_supports_interface_auth" value="True" /> <input type="checkbox" name="container_supports_interface_auth" value="True" />
</td> </td>
</tr> </tr>
<tr> <tr>
<td colspan=2> <td colspan=2>
<b>Disable HTTP auth embedding</b> &nbsp; <b>Disable HTTP auth embedding</b> &nbsp;
<input type="checkbox" name="container_disable_http_basicauth_embedding" value="True" /> <input type="checkbox" name="container_disable_http_basicauth_embedding" value="True" />
</td> </td>
</tr> </tr>
...@@ -158,7 +158,7 @@ ...@@ -158,7 +158,7 @@
</table> </table>
</div> </div>
<table style="width:400px; border:0; background:#ffffff; margin-top:20px"> <table style="width:400px; border:0; background:#ffffff; margin-top:20px">
<tr><td align="center"> <tr><td align="center">
<input type="submit" value="Add"> <input type="submit" value="Add">
...@@ -166,17 +166,17 @@ ...@@ -166,17 +166,17 @@
</table> </table>
<input type="hidden" name="new_container_from" value="registry"> <input type="hidden" name="new_container_from" value="registry">
</form> </form>
{% else %} {% else %}
<div style="font-size:1.1em; background:white; display:inline-block; padding:2px 15px 2px 15px"><a href="?new_container_from=registry">New container from registry</a></div> <div style="font-size:1.1em; background:white; display:inline-block; padding:2px 15px 2px 15px"><a href="?new_container_from=registry">New container from registry</a></div>
<div style="font-size:1.1em; background:whitesmoke; display:inline-block; padding:2px 15px 2px 15px">New container from Git repository</div> <div style="font-size:1.1em; background:whitesmoke; display:inline-block; padding:2px 15px 2px 15px">New container from Git repository</div>
<hr style="margin-top:0;"> <hr style="margin-top:0;">
<h4>Basics</h4> <h4>Basics</h4>
<form action="#" method="POST"> <form action="#" method="POST">
{% csrf_token %} {% csrf_token %}
...@@ -196,7 +196,7 @@ ...@@ -196,7 +196,7 @@
<textarea name="container_description" rows="3" cols="22"></textarea> <textarea name="container_description" rows="3" cols="22"></textarea>
</td> </td>
</tr> </tr>
<tr> <tr>
<td><b>Repository URL</b></td><td> <td><b>Repository URL</b></td><td>
<input type="text" name="repository_url" size="23" required /> <input type="text" name="repository_url" size="23" required />
...@@ -214,24 +214,23 @@ ...@@ -214,24 +214,23 @@
</table> </table>
<table style="width:400px; border:0; background:#ffffff; margin-top:20px"> <table style="width:400px; border:0; background:#ffffff; margin-top:20px">
<tr><td align="center"> <tr><td align="center">
<input type="submit" value="Add"> <input type="submit" value="Add">
</td></tr> </td></tr>
</table> </table>
<input type="hidden" name="new_container_from" value="repository"> <input type="hidden" name="new_container_from" value="repository">
</form> </form>
{% endif %} {% endif %}
<br/>
<br/> <br/>
<br/> <br/>
<br/> <br/>
...@@ -240,21 +239,22 @@ ...@@ -240,21 +239,22 @@
<br/> <br/>
<br/> <br/>
<br/> <br/>
<br/>
{% else %} {% else %}
Ok, software container added. Go to <a href="/software">software</a>. Ok, software container added. Go to <a href="/software">software</a>.
{% endif %}
{% endif %}
<br/> <br/>
<br/> <br/>
<br/> <br/>
<br/> <br/>
<br/> <br/>
<br/> <br/>
</div> </div>
</div> </div>
</div> </div>
...@@ -270,7 +270,7 @@ function toggle_visibility(id) { ...@@ -270,7 +270,7 @@ function toggle_visibility(id) {
x.style.display = "block"; x.style.display = "block";
y.style.display = "none" y.style.display = "none"
z.style.display= "inline" z.style.display= "inline"
} else { } else {
x.style.display = "none"; x.style.display = "none";
y.style.display= "inline" y.style.display= "inline"
...@@ -283,3 +283,4 @@ function toggle_visibility(id) { ...@@ -283,3 +283,4 @@ function toggle_visibility(id) {
...@@ -13,10 +13,10 @@ ...@@ -13,10 +13,10 @@
<div style="color:green"> <div style="color:green">
{{data.success}} {{data.success}}
</div> </div>
{% else %} {% else %}
<br/> <br/>
{% endif %} {% endif %}
{% if request.user.is_authenticated %} {% if request.user.is_authenticated %}
Logged in as <b>{{ request.user.username }}</b> | <a href="/logout">Logout</a> Logged in as <b>{{ request.user.username }}</b> | <a href="/logout">Logout</a>
{% else %} {% else %}
...@@ -41,11 +41,12 @@ ...@@ -41,11 +41,12 @@
<li><a href="/api/v1/base/login/">/api/v1/base/login/</a></li> <li><a href="/api/v1/base/login/">/api/v1/base/login/</a></li>
<li><a href="/api/v1/base/logout/">/api/v1/base/logout/</a></li> <li><a href="/api/v1/base/logout/">/api/v1/base/logout/</a></li>
</ul> </ul>
</div> </div>
</body> </body>
</html> </html>
{% if data.computing %} {% if data.computing %}
<div style="float:left; width:500px; margin:10px; margin-bottom:20px"> <div style="float:left; width:500px; margin:10px; margin-bottom:20px">
<table class="dashboard" style="width:100%"> <table class="dashboard" style="width:100%">
...@@ -23,7 +23,7 @@ ...@@ -23,7 +23,7 @@
<td><b>Type</b></td> <td><b>Type</b></td>
<td>{{ data.computing.type }}</td> <td>{{ data.computing.type }}</td>
</tr> </tr>
<tr> <tr>
<td><b>Access mode</b></td> <td><b>Access mode</b></td>
<td>{{ data.computing.access_mode }}</td> <td>{{ data.computing.access_mode }}</td>
...@@ -73,7 +73,7 @@ ...@@ -73,7 +73,7 @@
</table> </table>
</div> </div>
{% else %} {% else %}
...@@ -95,10 +95,10 @@ ...@@ -95,10 +95,10 @@
{% if not computing_configured %} {% if not computing_configured %}
<div style="width:298px; float:left; margin:0; margin-bottom:5px; padding:2px; background: gary; color:black; text-align:center; border-bottom: #e0e0e0 solid 1px;"> <div style="width:298px; float:left; margin:0; margin-bottom:5px; padding:2px; background: gary; color:black; text-align:center; border-bottom: #e0e0e0 solid 1px;">
<i class="fa fa-info-circle"></i> Not configured <!-- style="color:#337ab7" --> <i class="fa fa-info-circle"></i> Not configured <!-- style="color:#337ab7" -->
</div> </div>
{% endif %} {% endif %}
{% if not computing_configured %} {% if not computing_configured %}
<!-- <div style="padding:10px; -webkit-filter: blur(5px); -moz-filter: blur(5px); -o-filter: blur(5px); -ms-filter: blur(5px); filter: blur(5px);"> --> <!-- <div style="padding:10px; -webkit-filter: blur(5px); -moz-filter: blur(5px); -o-filter: blur(5px); -ms-filter: blur(5px); filter: blur(5px);"> -->
...@@ -107,12 +107,12 @@ ...@@ -107,12 +107,12 @@
<div style="padding:10px;"> <div style="padding:10px;">
{% endif %} {% endif %}
{% if computing.description %} {% if computing.description %}
<div class="description-box" title="{{ computing.description }}"> <div class="description-box" title="{{ computing.description }}">
{{ computing.description }} {{ computing.description }}
</div> </div>
{% else %} {% else %}
<br/> <br/>
{% endif %} {% endif %}
<div class="image-version-box"> <div class="image-version-box">
<b>Type:</b> {{ computing.type }} <font style="font-size:0.9em">({{ computing.arch }})</font> <b>Type:</b> {{ computing.type }} <font style="font-size:0.9em">({{ computing.arch }})</font>
...@@ -133,7 +133,7 @@ ...@@ -133,7 +133,7 @@
<!-- ><b>Access:</b> {{ computing.access_mode }}<br/> -- <!-- ><b>Access:</b> {{ computing.access_mode }}<br/> --
<!-- <b>Owner:</b> {% if computing.user %}{{ computing.user }}{% else %}Platform{% endif %}<br/> --> <!-- <b>Owner:</b> {% if computing.user %}{{ computing.user }}{% else %}Platform{% endif %}<br/> -->
<!-- <b>Engines:</b> {{ computing.container_engines }} --> <!-- <b>Engines:</b> {{ computing.container_engines }} -->
<!-- {% if 'docker' in computing.container_engines %}<img src="/static/img/docker-logo.svg" style="height:18px; width:18px; margin-bottom:2px" />{% endif %} <!-- {% if 'docker' in computing.container_engines %}<img src="/static/img/docker-logo.svg" style="height:18px; width:18px; margin-bottom:2px" />{% endif %}
{% if 'singularity' in computing.container_engines %}<img src="/static/img/singularity-logo.svg" style="height:18px; width:18px; margin-bottom:2px" />{% endif %}--> {% if 'singularity' in computing.container_engines %}<img src="/static/img/singularity-logo.svg" style="height:18px; width:18px; margin-bottom:2px" />{% endif %}-->
{% if container %} {% if container %}
...@@ -152,15 +152,15 @@ ...@@ -152,15 +152,15 @@
{% endif %} {% endif %}
</div> </div>
{% endif %} {% endif %}
</div> </div>
</div> </div>
{% endif %} {% endif %}
{% endif %} {% endif %}
\ No newline at end of file