From 4b335dfd94ea00e38b49f9e9f433c1852217c0c7 Mon Sep 17 00:00:00 2001
From: Stefano Alberto Russo <stefano.russo@gmail.com>
Date: Tue, 5 Apr 2022 00:00:08 +0200
Subject: [PATCH] Added compatibility with Binder.
---
docker-compose-dev.yml | 2 +-
.../webapp/code/rosetta/core_app/views.py | 120 ++++++++++++++++++
services/webapp/code/rosetta/urls.py | 4 +
services/webapp/requirements.txt | 1 +
4 files changed, 126 insertions(+), 1 deletion(-)
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 0134366..a723753 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -68,7 +68,7 @@ services:
- TASK_PROXY_HOST=localhost
- TASK_TUNNEL_HOST=localhost
- ROSETTA_HOST=localhost
- - REGISTRY_HOST=proxy # Use same value as ROSETTA_HOST for production or to use "real" computing resurces
+ - REGISTRY_HOST=proxy:5000 # Use same value as ROSETTA_HOST for production or to use "real" computing resurces
ports:
- "8080:8080"
- "7000-7020:7000-7020"
diff --git a/services/webapp/code/rosetta/core_app/views.py b/services/webapp/code/rosetta/core_app/views.py
index 0f53b84..ea90213 100644
--- a/services/webapp/code/rosetta/core_app/views.py
+++ b/services/webapp/code/rosetta/core_app/views.py
@@ -1,6 +1,7 @@
import os
import uuid
import json
+import requests
import subprocess
import base64
from django.conf import settings
@@ -1166,13 +1167,132 @@ def sharable_link_handler(request, short_uuid):
return redirect(redirect_string)
+def get_or_create_container_from_repository(repository_url, repository_tag=None):
+ repository_name = '{}/{}'.format(repository_url.split('/')[-2],repository_url.split('/')[-1])
+
+ logger.debug('Called get_or_create_container_from_repository with repository_url="{}" and repository_tag="{}"'.format(repository_url,repository_tag))
+
+ # If building:
+ #{"message": "Successfully built 5a2089b2c334\n", "phase": "building"}
+ #{"message": "Successfully tagged r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3:latest\n", "phase": "building"}
+
+ # If reusing:
+ #{"message": "Reusing existing image (r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3), not building."}
+
+ # Build the Docker container for this repo
+ if repository_tag:
+ command = 'sudo jupyter-repo2docker --ref {} --user-id 1000 --user-name rosetta --no-run --json-logs {}'.format(repository_tag, repository_url)
+ else:
+ command = 'sudo jupyter-repo2docker --user-id 1000 --user-name rosetta --no-run --json-logs {}'.format(repository_url)
+ out = os_shell(command, capture=True)
+ if out.exit_code != 0:
+ logger.error(out.stderr)
+ raise ErrorMessage('Something went wrong when creating the Dockerfile for repository "{}"'.format(repository_url))
+
+ # Convert output to lines
+ out_lines = out.stderr.split('\n')
+
+ # Get rep2docker image name from output
+ last_line_message = json.loads(out_lines[-1])['message']
+ if 'Reusing existing image' in last_line_message:
+ repo2docker_image_name = last_line_message.split('(')[1].split(')')[0]
+ elif 'Successfully tagged' in last_line_message:
+ repo2docker_image_name = last_line_message.split(' ')[2]
+ else:
+ raise Exception('Cannot build')
+
+ # Set image registry, name and tag, Use "strip()" as sometimes the newline chars might jump in.
+ registry = os.environ.get('REGISTRY_HOST','proxy:5000').strip()
+ image_name = repository_name.lower().strip()
+ image_tag = repo2docker_image_name[-7:].strip() # The last part of the image name generated by repo2docker is the git short hash
+
+ # Re-tag image taking into account that if we are using the proxy as registry we use localhost or it won't work
+ if registry == 'proxy:5000':
+ push_registry = 'localhost:5000'
+ else:
+ push_registry = registry
+
+ out = os_shell('sudo docker tag {} {}/{}:{}'.format(repo2docker_image_name,push_registry,image_name,image_tag) , capture=True)
+ if out.exit_code != 0:
+ logger.error(out.stderr)
+ raise ErrorMessage('Something went wrong when tagging the container for repository "{}"'.format(repository_url))
+
+ # Push image to the (local) registry
+ out = os_shell('sudo docker push {}/{}:{}'.format(push_registry,image_name,image_tag) , capture=True)
+ if out.exit_code != 0:
+ logger.error(out.stderr)
+ raise ErrorMessage('Something went wrong when pushing the container for repository "{}"'.format(repository_url))
+
+ # Create the container if not already existent
+ try:
+ container = Container.objects.get(registry=registry, image_name=image_name, image_tag=image_tag)
+ except Container.DoesNotExist:
+
+ # Set default container name and description
+ container_name = repository_name
+ container_description = 'Built from {}'.format(repository_url)
+
+ # Get name repo name and description from GitHub (if repo is there)
+ if repository_url.startswith('https://github.com'):
+ try:
+ response = requests.get('https://api.github.com/repos/{}'.format(repository_name))
+ json_content = json.loads(response.content)
+ container_name = json_content['name'].title()
+ container_description = json_content['description']
+ if not container_description.endswith('.'):
+ container_description+='.'
+ container_description += ' Built from {}'.format(repository_url)
+ except:
+ pass
+
+ container = Container.objects.create(user = None,
+ name = container_name,
+ description = container_description,
+ registry = registry,
+ image_name = image_name,
+ image_tag = image_tag,
+ image_arch = 'amd64',
+ image_os = 'linux',
+ interface_port = '8888',
+ interface_protocol = 'http',
+ interface_transport = 'tcp/ip',
+ supports_custom_interface_port = False,
+ supports_interface_auth = False)
+ return container
+#=========================
+# New Binder Task
+#=========================
+
+@private_view
+def new_binder_task(request, repository):
+
+ # Init data
+ data={}
+ data['user'] = request.user
+ # Convert the Git repository as a Docker container
+ logger.debug('Got a new Binder task request for repository "%s"', repository)
+
+ # Set repository name/tag/url
+ repository_tag = repository.split('/')[-1]
+ repository_url = repository.replace('/'+repository_tag, '')
+ container = get_or_create_container_from_repository(repository_url, repository_tag)
+
+ # Set the container
+ data['task_container'] = container
+ # List all computing resources
+ data['computings'] = list(Computing.objects.filter(group=None)) + list(Computing.objects.filter(group__user=request.user))
+
+ data['step'] = 'two'
+ data['next_step'] = 'three'
+ # Render the new task page and handle the rest from there
+ return render(request, 'new_task.html', {'data': data})
diff --git a/services/webapp/code/rosetta/urls.py b/services/webapp/code/rosetta/urls.py
index 88b14fd..ed69eb3 100644
--- a/services/webapp/code/rosetta/urls.py
+++ b/services/webapp/code/rosetta/urls.py
@@ -85,6 +85,10 @@ urlpatterns = [
path('api/v1/base/agent/', core_app_api.agent_api.as_view(), name='agent_api'),
path('api/v1/filemanager/', core_app_api.FileManagerAPI.as_view(), name='filemanager_api'),
+ # Binder compatibility
+ path('v2/git/<path:repository>', core_app_views.new_binder_task),
+
+
]
diff --git a/services/webapp/requirements.txt b/services/webapp/requirements.txt
index ab0fd71..5f8fcb2 100644
--- a/services/webapp/requirements.txt
+++ b/services/webapp/requirements.txt
@@ -8,3 +8,4 @@ sendgrid==5.3.0
mozilla-django-oidc==1.2.4
uwsgi==2.0.19.1
python-magic==0.4.15
+jupyter-repo2docker==2022.2.0
--
GitLab