diff --git a/README.md b/README.md
index da97ed2fb6c99c4e4a319005674ec012c34f5e34..3ccd9f0f83aed8f57777fa3e616e19d09709287e 100755
--- a/README.md
+++ b/README.md
@@ -43,30 +43,40 @@ Clean
### Configuration
-Example Webapp configuraion:
+Webapp service configuraion parameters and their defaults:
- - SAFEMODE=False
- - DJANGO_DEV_SERVER=True
- - DJANGO_DEBUG=True
+ - SAFEMODE=false
+ - DJANGO_DEV_SERVER=true
+ - DJANGO_DEBUG=true
- DJANGO_LOG_LEVEL=ERROR
- ROSETTA_LOG_LEVEL=ERROR
- - ROSETTA_TUNNEL_HOST=localhost # Not http or https
- - ROSETTA_WEBAPP_HOST=
+ - ROSETTA_HOST=localhost
+ - ROSETTA_TUNNEL_HOST=localhost
+ - ROSETTA_WEBAPP_HOST=""
- ROSETTA_WEBAPP_PORT=8080
- - LOCAL_DOCKER_REGISTRY_HOST=
- - LOCAL_DOCKER_REGISTRY_PORT=5000
+ - ROSETTA_REGISTRY_HOST=proxy
+ - ROSETTA_REGISTRY_PORT=5000
- DJANGO_EMAIL_SERVICE=Sendgrid
- - DJANGO_EMAIL_APIKEY=
+ - DJANGO_EMAIL_APIKEY=""
- DJANGO_EMAIL_FROM="Rosetta <notifications@rosetta.local>"
- - DJANGO_PUBLIC_HTTP_HOST=http://localhost # Public facing, with http or https
+ - INVITATION_CODE=""
- OIDC_RP_CLIENT_ID=""
- OIDC_RP_CLIENT_SECRET=""
- OIDC_OP_AUTHORIZATION_ENDPOINT=""
- OIDC_OP_TOKEN_ENDPOINT=""
- OIDC_OP_JWKS_ENDPOINT=""
- - DISABLE_LOCAL_AUTH=False
-
-In Rosetta, only power users can:
+ - DISABLE_LOCAL_AUTH=false
+
+Notes:
+
+ - `ROSETTA_TUNNEL_HOST` must not include http:// or https://
+ - `ROSETTA_REGISTRY_HOST` should be set to the same value as `ROSETTA_HOST` for production scenarios, in order to be secured unders SSL. The `standaloneworker` is configured to treat the following hosts (and ports) as unsecure registies, where it can connect without a valid certificate: `proxy:5000`,`dregistry:5000` and `rosetta.platform:5000`.
+ - `ROSETTA_WEBAPP_HOST` is used for let the agent know where to connect, and it is differentiated from `ROSETTA_HOST` as it can be on an internal Docker network. It is indeed defaulted to the `webapp` container IP address.
+
+
+
+### User types
+In Rosetta there are two user types: standard users and power users. Their type is set in their user profile, and only power users can:
- set custom task passwords
- choose task access methods other than the default one (bypassing HTTP proxy + auth)
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index a72375381ef17c579800d994dd882651afe87567..5d7f06730fbb637066127789ee514f8275fd93c6 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -60,8 +60,8 @@ services:
- ROSETTA_LOG_LEVEL=DEBUG
#- ROSETTA_WEBAPP_HOST=localhost # Internal, for the agent
#- ROSETTA_WEBAPP_PORT=8080 # Internal, for the agent
- #- LOCAL_DOCKER_REGISTRY_HOST=
- #- LOCAL_DOCKER_REGISTRY_PORT=5000
+ #- ROSETTA_REGISTRY_HOST=
+ #- ROSETTA_REGISTRY_PORT=5000
#- DJANGO_EMAIL_APIKEY=""
#- DJANGO_EMAIL_FROM="Rosetta Platform <notifications@rosetta.platform>"
#- DJANGO_SECRET_KEY=""
diff --git a/services/standaloneworker/Dockerfile b/services/standaloneworker/Dockerfile
index dbd134af9d18c9c5c8089e14cce6b516de0dd9c2..11ceaceaf1455b0baca49b4dc7c2c4c41558da3f 100755
--- a/services/standaloneworker/Dockerfile
+++ b/services/standaloneworker/Dockerfile
@@ -21,8 +21,10 @@ RUN mkdir /home/testuser/.ssh
COPY keys/id_rsa.pub /home/testuser/.ssh/authorized_keys
RUN dnf install -y python wget
-# Install iputils (ping)
-RUN dnf install -y iputils
+# Install iputils (fpr ping) and openssh-clients (for scp)
+RUN dnf install -y iputils openssh-clients
+
+# TODO: Add the rosettta user
# Copy registries.conf to allow insecure access to dregistry
COPY registries.conf /etc/containers/registries.conf
diff --git a/services/standaloneworker/registries.conf b/services/standaloneworker/registries.conf
index e32f8d0e317231e5404308840507100e6a7a6375..fb97bbba383dfdd08172e801781b858adc8b69af 100644
--- a/services/standaloneworker/registries.conf
+++ b/services/standaloneworker/registries.conf
@@ -85,6 +85,10 @@ short-name-mode="enforcing"
location = "dregistry:5000"
insecure = true
+[[registry]]
+location = "proxy:5000"
+insecure = true
+
[[registry]]
location = "rosetta.platform:5000"
insecure = true
diff --git a/services/webapp/code/rosetta/core_app/api.py b/services/webapp/code/rosetta/core_app/api.py
index 4953ac0e074c4212d465927e708ffbb4af0d7048..c52dd77f46fd4fb3bd526f3bddfc69019b7ff9be 100644
--- a/services/webapp/code/rosetta/core_app/api.py
+++ b/services/webapp/code/rosetta/core_app/api.py
@@ -321,7 +321,7 @@ print(port)
return HttpResponse('Port not valid (got "{}")'.format(task_interface_port))
# Set fields
- logger.info('Setting task "{}" to ip "{}" and port "{}"'.format(task.uuid, task_interface_ip, task_interface_port))
+ logger.info('Agent API setting task "{}" to ip "{}" and port "{}"'.format(task.uuid, task_interface_ip, task_interface_port))
task.status = TaskStatuses.running
task.interface_ip = task_interface_ip
@@ -349,7 +349,7 @@ print(port)
# Notify the user that the task called back home if using a WMS
if task.computing.wms:
if settings.DJANGO_EMAIL_APIKEY:
- logger.info('Sending task ready mail notification to "{}"'.format(task.user.email))
+ logger.info('Agent API sending task ready mail notification to "{}"'.format(task.user.email))
mail_subject = 'Your Task "{}" is now starting up'.format(task.container.name)
mail_text = 'Hello,\n\nyour Task "{}" on {} is now starting up. Check logs or connect here: https://{}/tasks/?uuid={}\n\nThe Rosetta notifications bot.'.format(task.container.name, task.computing, settings.ROSETTA_HOST, task.uuid)
try:
@@ -441,7 +441,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
if computing.auth_mode == 'user_keys':
computing_user = user.profile.get_extra_conf('computing_user', storage.computing)
if not computing_user:
- raise Exception('Computing resource \'{}\' user is not configured'.format(storage.computing.name))
+ raise ValueError('No \'computing_user\' parameter found for computing resource \'{}\' in user profile'.format(storage.computing.name))
base_path_expanded = base_path_expanded.replace('$SSH_USER', computing_user)
else:
base_path_expanded = base_path_expanded.replace('$SSH_USER', computing.conf.get('user'))
@@ -449,7 +449,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in base_path_expanded:
- base_path_expanded = base_path_expanded.replace('$USER', user.name)
+ base_path_expanded = base_path_expanded.replace('$USER', user.username)
# If the path is not starting with the base path, do it
if not path.startswith(base_path_expanded):
@@ -757,6 +757,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
if not storage.type=='generic_posix' and storage.access_mode=='ssh+cli':
continue
+ if storage.access_through_computing and not storage.computing.manager.is_configured_for(user=request.user):
+ continue
+
data['data'].append({
'id': '/{}/'.format(storage.id),
'type': 'folder',
diff --git a/services/webapp/code/rosetta/core_app/computing_managers.py b/services/webapp/code/rosetta/core_app/computing_managers.py
index 4889817483fe89d874ae35c0c683a5bc132a4d71..9a92361787b2767aad42dd574afa896305608883 100644
--- a/services/webapp/code/rosetta/core_app/computing_managers.py
+++ b/services/webapp/code/rosetta/core_app/computing_managers.py
@@ -71,6 +71,9 @@ class ComputingManager(object):
# Call actual get task log logic
return self._get_task_log(task, **kwargs)
+
+ def is_configured_for(self, user):
+ return True
class StandaloneComputingManager(ComputingManager):
@@ -82,8 +85,14 @@ class ClusterComputingManager(ComputingManager):
class SSHComputingManager(ComputingManager):
- # SSH-f + keys utils here
- pass
+
+ def is_configured_for(self, user):
+ try:
+ get_ssh_access_mode_credentials(self.computing, user)
+ except:
+ return False
+ else:
+ return True
@@ -240,7 +249,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_base_path:
- expanded_base_path = expanded_base_path.replace('$USER', self.task.user.name)
+ expanded_base_path = expanded_base_path.replace('$USER', task.user.username)
# Expand the bind_path
expanded_bind_path = storage.bind_path
@@ -250,7 +259,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_bind_path:
- expanded_bind_path = expanded_bind_path.replace('$USER', self.task.user.name)
+ expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username)
# Add the bind
if not binds:
@@ -300,7 +309,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_base_path:
- expanded_base_path = expanded_base_path.replace('$USER', self.task.user.name)
+ expanded_base_path = expanded_base_path.replace('$USER', task.user.username)
# Expand the bind_path
expanded_bind_path = storage.bind_path
@@ -310,7 +319,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_bind_path:
- expanded_bind_path = expanded_bind_path.replace('$USER', self.task.user.name)
+ expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username)
# Add the bind
if not binds:
@@ -376,7 +385,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana
stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "{}"\''.format(computing_keys.private_key_file, computing_user, computing_host, internal_stop_command)
out = os_shell(stop_command, capture=True)
if out.exit_code != 0:
- if ('No such process' in out.stderr) or ('No such container' in out.stderr):
+ if ('No such process' in out.stderr) or ('No such container' in out.stderr) or ('no container' in out.stderr):
pass
else:
raise Exception(out.stderr)
@@ -493,7 +502,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_base_path:
- expanded_base_path = expanded_base_path.replace('$USER', self.task.user.name)
+ expanded_base_path = expanded_base_path.replace('$USER', task.user.username)
# Expand the bind_path
expanded_bind_path = storage.bind_path
@@ -503,7 +512,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
else:
raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
if '$USER' in expanded_bind_path:
- expanded_bind_path = expanded_bind_path.replace('$USER', self.task.user.name)
+ expanded_bind_path = expanded_bind_path.replace('$USER', task.user.username)
# Add the bind
if not binds:
diff --git a/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py b/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py
index 0f6b6b56c9f77d8b16aa85a3f73cb829102acdde..eea9f9764e626aac81422a6e87512d1f4ee80b23 100644
--- a/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py
+++ b/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py
@@ -280,8 +280,20 @@ to provide help, news and informations on your deployment. Or you can just ignor
auth_mode = 'user_keys',
wms = None,
conf = {'host': 'standaloneworker'},
- container_engines = ['singularity','podman'])
-
+ container_engines = ['podman','singularity'])
+
+ # Demo standalone platform computing plus conf
+ demo_singlenode_computing = Computing.objects.create(name = 'Demo Standalone Platform',
+ description = 'A demo standalone computing resource access as platform.',
+ type = 'standalone',
+ arch = 'amd64',
+ supported_archs = ['386'],
+ access_mode = 'ssh+cli',
+ auth_mode = 'platform_keys',
+ wms = None,
+ conf = {'host': 'standaloneworker', 'user': 'testuser'}, # TODO: use a dedicated user?
+ container_engines = ['podman','singularity'])
+
# Add testuser extra conf for this computing resource
testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_singlenode_computing, value= 'testuser')
diff --git a/services/webapp/code/rosetta/core_app/models.py b/services/webapp/code/rosetta/core_app/models.py
index 291e663c9b24e4260aec518559da700fed8d4d7b..4ea37b76d9833477babca5cb4dc43ce39551721f 100644
--- a/services/webapp/code/rosetta/core_app/models.py
+++ b/services/webapp/code/rosetta/core_app/models.py
@@ -246,14 +246,20 @@ class Computing(models.Model):
managers_mapping = {}
managers_mapping['cluster'+'ssh+cli'+'user_keys'+'slurm'] = computing_managers.SlurmSSHClusterComputingManager
managers_mapping['standalone'+'ssh+cli'+'user_keys'+'None'] = computing_managers.SSHStandaloneComputingManager
+ managers_mapping['standalone'+'ssh+cli'+'platform_keys'+'None'] = computing_managers.SSHStandaloneComputingManager
managers_mapping['standalone'+'internal'+'internal'+'None'] = computing_managers.InternalStandaloneComputingManager
# Instantiate the computing manager and return (if not already done)
try:
return self._manager
except AttributeError:
- self._manager = managers_mapping[self.type+self.access_mode+self.auth_mode+str(self.wms)](self)
- return self._manager
+ try:
+ self._manager = managers_mapping[self.type+self.access_mode+self.auth_mode+str(self.wms)](self)
+ except KeyError:
+ raise ValueError('No computing resource manager for type="{}", access_mode="{}", auth_mode="{}", wms="{}"'
+ .format(self.type, self.access_mode, self.auth_mode, self.wms)) from None
+ else:
+ return self._manager
diff --git a/services/webapp/code/rosetta/core_app/templates/components/computing.html b/services/webapp/code/rosetta/core_app/templates/components/computing.html
index 6c55b09e3c091d824a8d30f8cab75ded93c76545..8e709da9fc38ab48d1041ea8760087b85263e8ee 100644
--- a/services/webapp/code/rosetta/core_app/templates/components/computing.html
+++ b/services/webapp/code/rosetta/core_app/templates/components/computing.html
@@ -76,14 +76,36 @@
{% else %}
- <div style="width:300px; float:left; border: #e0e0e0 solid 1px; margin:10px; background:#f8f8f8; margin-bottom:15px">
+
+ {% load computing_helpers %}
+ {% is_computing_configured computing request.user as computing_configured %}
+
+ <!-- Insert an "if false" here to let the computign show up even when creating the task but to not allow to be chosen (greyed out button) -->
+ {% if container and not computing_configured %}
+ {% else%}
+
+ <div style="width:300px; float:left; border: #e0e0e0 solid 1px; margin:10px; background:#f8f8f8; margin-bottom:15px;">
<!-- <div style="background:{{computing.color}}; height:24px"></div> -->
-
+
<div style="padding:10px; margin-top:5px; text-align:center; border-bottom: #e0e0e0 solid 1px; ">
<a href="/computing/?uuid={{ computing.uuid }}&details=True">{{ computing.name }}</a>
</div>
+
+
+ {% if not computing_configured %}
+ <div style="width:298px; float:left; margin:0; margin-bottom:5px; padding:2px; background: gary; color:black; text-align:center; border-bottom: #e0e0e0 solid 1px;">
+ <i class="fa fa-info-circle"></i> Not configured <!-- style="color:#337ab7" -->
+ </div>
+ {% endif %}
+
+
+ {% if not computing_configured %}
+ <!-- <div style="padding:10px; -webkit-filter: blur(5px); -moz-filter: blur(5px); -o-filter: blur(5px); -ms-filter: blur(5px); filter: blur(5px);"> -->
+ <div style="padding:10px;">
+ {% else %}
<div style="padding:10px;">
+ {% endif %}
{% if computing.description %}
<div class="description-box" title="{{ computing.description }}">
@@ -116,6 +138,7 @@
{% if 'singularity' in computing.container_engines %}<img src="/static/img/singularity-logo.svg" style="height:18px; width:18px; margin-bottom:2px" />{% endif %}-->
{% if container %}
<div style="margin:0px; margin-top:2px; text-align:center; padding:5px">
+ {% if computing_configured %}
<form action="/new_task/" method="POST">
{% csrf_token %}
<input type="hidden" name="step" value="{{ data.next_step }}" />
@@ -124,12 +147,16 @@
<input type="hidden" name="task_computing_uuid" value="{{computing.uuid}}">
<input type="submit" value="Choose" class="btn btn-connect">
</form>
+ {% else %}
+ <input type="submit" value="Choose" class="btn btn-connect" disabled>
+ {% endif %}
</div>
{% endif %}
</div>
</div>
+ {% endif %}
diff --git a/services/webapp/code/rosetta/core_app/templates/new_task.html b/services/webapp/code/rosetta/core_app/templates/new_task.html
index 16aed722728a7ceb239121d41a38566a7445950d..78d8cffb5ffb500db7a06ab4b19b7aed9e9f3b87 100644
--- a/services/webapp/code/rosetta/core_app/templates/new_task.html
+++ b/services/webapp/code/rosetta/core_app/templates/new_task.html
@@ -132,7 +132,7 @@
<tr>
<td><b>Container engine</b></td><td>
<select name="container_engine" >
- <option value="" selected>Default</option>
+ <option value="" selected>Default ({{data.task_computing.container_engines.0}})</option>
{% for container_engine in data.task_computing.container_engines %}
<option value="{{container_engine}}">{{container_engine}}</option>
{% endfor %}
diff --git a/services/webapp/code/rosetta/core_app/templatetags/__init__.py b/services/webapp/code/rosetta/core_app/templatetags/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/services/webapp/code/rosetta/core_app/templatetags/computing_helpers.py b/services/webapp/code/rosetta/core_app/templatetags/computing_helpers.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e85d6915b034eaf899ad35c954b818a01b40390
--- /dev/null
+++ b/services/webapp/code/rosetta/core_app/templatetags/computing_helpers.py
@@ -0,0 +1,10 @@
+from django import template
+
+register = template.Library()
+
+@register.simple_tag
+def is_computing_configured(computing, user):
+ return computing.manager.is_configured_for(user)
+
+
+# {% is_computing_configured computing user %}
diff --git a/services/webapp/code/rosetta/core_app/utils.py b/services/webapp/code/rosetta/core_app/utils.py
index 35021f1af46c8d85a00d91500639064cc4e2d486..e296c8b495d03a08afb6ac79ae240ec56fd63aff 100644
--- a/services/webapp/code/rosetta/core_app/utils.py
+++ b/services/webapp/code/rosetta/core_app/utils.py
@@ -509,11 +509,11 @@ def get_webapp_conn_string():
webapp_conn_string = 'http://{}:{}'.format(webapp_host, webapp_port)
return webapp_conn_string
-def get_local_docker_registry_conn_string():
- local_docker_registry_host = os.environ.get('LOCAL_DOCKER_REGISTRY_HOST', 'dregistry')
- local_docker_registry_port = os.environ.get('LOCAL_DOCKER_REGISTRY_PORT', '5000')
- local_docker_registry_conn_string = '{}:{}'.format(local_docker_registry_host, local_docker_registry_port)
- return local_docker_registry_conn_string
+def get_platform_registry():
+ platform_registry_host = os.environ.get('PLATFORM_REGISTRY_HOST', 'proxy')
+ platform_registry_port = os.environ.get('PLATFORM_REGISTRY_PORT', '5000')
+ platform_registry_conn_string = '{}:{}'.format(platform_registry_host, platform_registry_port)
+ return platform_registry_conn_string
def get_task_tunnel_host():
tunnel_host = os.environ.get('TASK_TUNNEL_HOST', 'localhost')
@@ -715,13 +715,13 @@ def get_ssh_access_mode_credentials(computing, user):
except AttributeError:
computing_host = None
if not computing_host:
- raise Exception('No computing host?!')
+ raise ValueError('No computing host?!')
# Get computing user and keys
if computing.auth_mode == 'user_keys':
computing_user = user.profile.get_extra_conf('computing_user', computing)
if not computing_user:
- raise Exception('Computing resource \'{}\' user is not configured'.format(computing.name))
+ raise ValueError('No \'computing_user\' parameter found for computing resource \'{}\' in user profile'.format(computing.name))
# Get user key
computing_keys = KeyPair.objects.get(user=user, default=True)
elif computing.auth_mode == 'platform_keys':
@@ -730,7 +730,7 @@ def get_ssh_access_mode_credentials(computing, user):
else:
raise NotImplementedError('Auth modes other than user_keys and platform_keys not supported.')
if not computing_user:
- raise Exception('No computing user?!')
+ raise ValueError('No \'user\' parameter found for computing resource \'{}\' in its configuration'.format(computing.name))
return (computing_user, computing_host, computing_keys)
@@ -784,7 +784,7 @@ def get_or_create_container_from_repository(user, repository_url, repository_tag
raise Exception('Cannot build')
# Set image registry, name and tag. Use "strip()" as sometimes the newline chars might jump in.
- registry = os.environ.get('REGISTRY_HOST','proxy:5000').strip()
+ registry = get_platform_registry()
image_name = repository_name.lower().strip()
image_tag = repo2docker_image_name[-7:] # The last part of the image name generated by repo2docker is the git short hash