From 3efa2d9762d8bb6f819c51d9a2c325a3ed7c2226 Mon Sep 17 00:00:00 2001 From: Stefano Alberto Russo <stefano.russo@gmail.com> Date: Sun, 21 Nov 2021 13:24:07 +0100 Subject: [PATCH] Finalized architecutre support including in the computing resources and new tasks. --- .../management/commands/core_app_populate.py | 14 +++++------ .../migrations/0022_auto_20211121_1341.py | 18 ++++++++++++++ .../migrations/0023_auto_20211121_1342.py | 24 +++++++++++++++++++ .../0024_computing_emulated_archs.py | 19 +++++++++++++++ .../webapp/code/rosetta/core_app/models.py | 16 ++++++++++--- .../templates/components/computing.html | 17 ++++++++----- .../rosetta/core_app/templates/new_task.html | 6 +++++ .../webapp/code/rosetta/core_app/views.py | 15 ++++++++++++ 8 files changed, 113 insertions(+), 16 deletions(-) create mode 100644 services/webapp/code/rosetta/core_app/migrations/0022_auto_20211121_1341.py create mode 100644 services/webapp/code/rosetta/core_app/migrations/0023_auto_20211121_1342.py create mode 100644 services/webapp/code/rosetta/core_app/migrations/0024_computing_emulated_archs.py diff --git a/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py b/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py index ce496da..3084c8e 100644 --- a/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py +++ b/services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py @@ -134,7 +134,7 @@ to provide help, news and informations on your deployment. Or you can just ignor registry = 'docker.io', image = 'sarusso/minimaldesktop', tag = 'v0.2.0', - arch = 'x86_64', + arch = 'amd64', os = 'linux', interface_port = '8590', interface_protocol = 'http', @@ -149,7 +149,7 @@ to provide help, news and informations on your deployment. Or you can just ignor registry = 'docker.io', image = 'sarusso/basicdesktop', tag = 'v0.2.0', - arch = 'x86_64', + arch = 'amd64', os = 'linux', interface_port = '8590', interface_protocol = 'http', @@ -166,7 +166,7 @@ to provide help, news and informations on your deployment. Or you can just ignor registry = 'docker.io', image = 'sarusso/jupyternotebook', tag = 'v0.2.0', - arch = 'x86_64', + arch = 'amd64', os = 'linux', interface_port = '8888', interface_protocol = 'http', @@ -213,7 +213,7 @@ to provide help, news and informations on your deployment. Or you can just ignor registry = 'docker.io', image = 'sarusso/ssh', tag = 'v0.2.0', - arch = 'x86_64', + arch = 'amd64', os = 'linux', interface_port = '22', interface_protocol = 'ssh', @@ -263,7 +263,7 @@ to provide help, news and informations on your deployment. Or you can just ignor access_mode = 'internal', auth_mode = 'internal', wms = None, - container_runtimes = 'docker') + container_runtimes = ['docker']) # Demo standalone computing plus conf @@ -274,7 +274,7 @@ to provide help, news and informations on your deployment. Or you can just ignor auth_mode = 'user_keys', wms = None, conf = {'host': 'slurmclusterworker-one'}, - container_runtimes = 'singularity') + container_runtimes = ['singularity']) # Add testuser extra conf for this computing resource testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_singlenode_computing, value= 'slurmtestuser') @@ -287,7 +287,7 @@ to provide help, news and informations on your deployment. Or you can just ignor auth_mode = 'user_keys', wms = 'slurm', conf = {'host': 'slurmclustermaster-main', 'default_partition': 'partition1'}, - container_runtimes = 'singularity') + container_runtimes = ['singularity']) # Add testuser extra conf for this computing resource testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_slurm_computing, value= 'slurmtestuser') diff --git a/services/webapp/code/rosetta/core_app/migrations/0022_auto_20211121_1341.py b/services/webapp/code/rosetta/core_app/migrations/0022_auto_20211121_1341.py new file mode 100644 index 0000000..a7de657 --- /dev/null +++ b/services/webapp/code/rosetta/core_app/migrations/0022_auto_20211121_1341.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.1 on 2021-11-21 13:41 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core_app', '0021_container_interface_auth_user'), + ] + + operations = [ + migrations.RenameField( + model_name='computing', + old_name='container_runtimes', + new_name='container_runtime', + ), + ] diff --git a/services/webapp/code/rosetta/core_app/migrations/0023_auto_20211121_1342.py b/services/webapp/code/rosetta/core_app/migrations/0023_auto_20211121_1342.py new file mode 100644 index 0000000..dcde2e0 --- /dev/null +++ b/services/webapp/code/rosetta/core_app/migrations/0023_auto_20211121_1342.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.1 on 2021-11-21 13:42 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core_app', '0022_auto_20211121_1341'), + ] + + operations = [ + migrations.RemoveField( + model_name='computing', + name='container_runtime', + ), + migrations.AddField( + model_name='computing', + name='container_runtimes', + field=django.contrib.postgres.fields.jsonb.JSONField(default=['docker'], verbose_name='Container runtimes'), + preserve_default=False, + ), + ] diff --git a/services/webapp/code/rosetta/core_app/migrations/0024_computing_emulated_archs.py b/services/webapp/code/rosetta/core_app/migrations/0024_computing_emulated_archs.py new file mode 100644 index 0000000..0add854 --- /dev/null +++ b/services/webapp/code/rosetta/core_app/migrations/0024_computing_emulated_archs.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.1 on 2021-11-21 13:42 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core_app', '0023_auto_20211121_1342'), + ] + + operations = [ + migrations.AddField( + model_name='computing', + name='emulated_archs', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True, verbose_name='Emulated architectures'), + ), + ] diff --git a/services/webapp/code/rosetta/core_app/models.py b/services/webapp/code/rosetta/core_app/models.py index 60159fa..587377f 100644 --- a/services/webapp/code/rosetta/core_app/models.py +++ b/services/webapp/code/rosetta/core_app/models.py @@ -164,16 +164,21 @@ class Computing(models.Model): name = models.CharField('Name', max_length=255, blank=False, null=False) description = models.TextField('Description', blank=True, null=True) - # Tye (standalone / cluster) + # Type (standalone / cluster) and arch type = models.CharField('Type', max_length=255, blank=False, null=False) + arch = models.CharField('Architecture', max_length=255, blank=False, null=False) # Interfce and interaction definition access_mode = models.CharField('Access (control) mode', max_length=36, blank=False, null=False) auth_mode = models.CharField('Auth mode', max_length=36, blank=False, null=False) wms = models.CharField('Workload management system', max_length=36, blank=True, null=True) - # Supported container runtimes - container_runtimes = models.CharField('Container runtimes', max_length=256, blank=False, null=False) + # Supported container runtimes ['docker', 'singularity'] + container_runtimes = JSONField('Container runtimes', blank=False, null=False) + #container_runtime = models.CharField('Container runtimes', max_length=256, blank=False, null=False) + + # Emulated architectures, by container runtime {'docker': ['arm64', 'amd'] + emulated_archs = JSONField('Emulated architectures', blank=True, null=True) # Conf conf = JSONField(blank=True, null=True) @@ -202,6 +207,11 @@ class Computing(models.Model): @property def default_container_runtime(self): return str(self.container_runtimes).split(',')[0] + + @property + def arch(self): + return 'amd64' + #======================= diff --git a/services/webapp/code/rosetta/core_app/templates/components/computing.html b/services/webapp/code/rosetta/core_app/templates/components/computing.html index 7616957..0ad13b5 100644 --- a/services/webapp/code/rosetta/core_app/templates/components/computing.html +++ b/services/webapp/code/rosetta/core_app/templates/components/computing.html @@ -23,7 +23,7 @@ <td><b>Type</b></td> <td>{{ data.computing.type }}</td> </tr> - + <tr> <td><b>Access mode</b></td> <td>{{ data.computing.access_mode }}</td> @@ -39,11 +39,20 @@ <td>{{ data.computing.wms }}</td> </tr> + <tr> + <td><b>Arch</b></td> + <td>{{ data.computing.arch }}</td> + </tr> + <tr> <td><b>Container runtimes</b></td> <td>{{ data.computing.container_runtimes }}</td> </tr> + <tr> + <td><b>Emulated archs</b></td> + <td>{{ data.computing.emulated_archs }}</td> + </tr> <tr> <td><b>Storages</b></td> @@ -85,11 +94,7 @@ {% endif %} <div class="image-version-box"> <b>Type:</b> {{ computing.type }}<br/> - <!-- -->{% if computing.auth_mode == 'user_keys' %} - <b>Access:</b> user keys - {% else %} - <b>Access:</b> open - {% endif %} + <b>Arch:</b> {{ computing.arch }} <br/> <b>Storages:</b> {% if not computing.storages.all %} diff --git a/services/webapp/code/rosetta/core_app/templates/new_task.html b/services/webapp/code/rosetta/core_app/templates/new_task.html index eef352b..c8d9b21 100644 --- a/services/webapp/code/rosetta/core_app/templates/new_task.html +++ b/services/webapp/code/rosetta/core_app/templates/new_task.html @@ -166,6 +166,12 @@ <i class="fa fa-exclamation-triangle" style="color:orange"></i> This container does not support custom interface ports and the computing resource you selected might use a container runtime which does not support port mapping (Singularity). In this case, if the container interface port is already allocated, the task will fail to start. </p></div> {% endif %} + + {% if data.arch_emulation %} + <div> <p style="font-size:15px; max-width:700px; margin-bottom:20px; margin-left:5px"> + <i class="fa fa-exclamation-triangle" style="color:orange"></i> The selected software container architecture ({{ data.task_container_arch}}) can only be emulated on this computing resource, meaning the it will probably run quite slow. + </p></div> + {% endif %} <!-- {% if data.task_container.interface_port and not data.task_container.supports_interface_auth %} <div> <p style="font-size:15px; max-width:700px; margin-bottom:20px; margin-left:5px"> diff --git a/services/webapp/code/rosetta/core_app/views.py b/services/webapp/code/rosetta/core_app/views.py index 4b56e04..1f6c35c 100644 --- a/services/webapp/code/rosetta/core_app/views.py +++ b/services/webapp/code/rosetta/core_app/views.py @@ -524,6 +524,21 @@ def new_task(request): # Get computing resource data['task_computing'] = get_task_computing(request) + # Check that container required architecture is compatible with the computing resource + if data['task_container_arch'] != 'auto': + if data['task_container_arch'] != data['task_computing'].arch: + # TODO: support setting the container runtime when creating the task + # TODO: refactor and unroll this code + container_runtime = data['task_computing'].container_runtimes[0] + if container_runtime in data['task_computing'].emulated_archs and data['task_container_arch'] in data['task_computing'].emulated_archs[container_runtime]: + data['arch_emulation'] = True + else: + raise ErrorMessage('This computing resource does not support architecture "{}" nor as native or emulated'.format(data['task_container_arch'])) + + else: + raise ErrorMessage('Auto architectures are not supported yet') + + # Generate random auth token data['task_auth_token'] = str(uuid.uuid4()) -- GitLab