Skip to content
Snippets Groups Projects
Commit 349785e6 authored by Stefano Alberto Russo's avatar Stefano Alberto Russo
Browse files

Merge branch 'feature/uWSGI' into develop

parents 603f4ef8 dc84ed65
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 127 additions and 105 deletions
README.md
+ 3
− 2
View file @ 349785e6
......@@ -45,7 +45,9 @@ Clean
Example Webapp configuraion
- SAFEMODE=False
- DJANGO_LOG_LEVEL=CRITICAL
- DJANGO_DEV_SERVER=True
- DJANGO_DEBUG=True
- DJANGO_LOG_LEVEL=ERROR
- ROSETTA_LOG_LEVEL=ERROR
- ROSETTA_TUNNEL_HOST=localhost # Not http or https
- ROSETTA_WEBAPP_HOST=
......@@ -58,7 +60,6 @@ Example Webapp configuraion
- DJANGO_PUBLIC_HTTP_HOST=http://localhost # Public facing, with http or https
### Extras
List all running services
......
docker-compose-dev.yml
+ 1
− 0
View file @ 349785e6
......@@ -46,6 +46,7 @@ services:
hostname: webapp
environment:
- SAFEMODE=False
- DJANGO_DEV_SERVER=True
- DJANGO_DEBUG=True
- DJANGO_LOG_LEVEL=CRITICAL
- ROSETTA_LOG_LEVEL=DEBUG
......
services/webapp/code/rosetta/core_app/api.py
+ 54
− 74
View file @ 349785e6
......@@ -209,28 +209,27 @@ class UserViewSet(viewsets.ModelViewSet):
class agent_api(PublicGETAPI):
def _get(self, request):
task_uuid = request.GET.get('task_uuid', None)
if not task_uuid:
return HttpResponse('MISSING task_uuid')
from django.core.exceptions import ValidationError
try:
task_uuid = request.GET.get('task_uuid', None)
if not task_uuid:
return HttpResponse('MISSING task_uuid')
from django.core.exceptions import ValidationError
try:
task = Task.objects.get(uuid=task_uuid)
except (Task.DoesNotExist, ValidationError):
return HttpResponse('Unknown task uuid "{}"'.format(task_uuid))
task = Task.objects.get(uuid=task_uuid)
except (Task.DoesNotExist, ValidationError):
return HttpResponse('Unknown task uuid "{}"'.format(task_uuid))
from.utils import get_webapp_conn_string
webapp_conn_string = get_webapp_conn_string()
action = request.GET.get('action', None)
if not action:
# Return the agent code
agent_code='''
from.utils import get_webapp_conn_string
webapp_conn_string = get_webapp_conn_string()
action = request.GET.get('action', None)
if not action:
# Return the agent code
agent_code='''
import logging
import socket
try:
......@@ -285,64 +284,45 @@ else:
print(port)
'''
return HttpResponse(agent_code)
elif action=='set_ip_port':
task_ip = request.GET.get('ip', None)
if not task_ip:
return HttpResponse('IP not valid (got "{}")'.format(task_ip))
task_port = request.GET.get('port', None)
if not task_port:
return HttpResponse('Port not valid (got "{}")'.format(task_port))
try:
int(task_port)
except (TypeError, ValueError):
return HttpResponse('Port not valid (got "{}")'.format(task_port))
# Set fields
logger.info('Setting task "{}" to ip "{}" and port "{}"'.format(task.uuid, task_ip, task_port))
task.status = TaskStatuses.running
task.ip = task_ip
if task.container.supports_dynamic_ports:
task.port = int(task_port)
task.save()
# Notify the user that the task called back home
logger.info('Sending task ready mail notification to "{}"'.format(task.user.email))
mail_subject = 'Your Task "{}" is up and running'.format(task.container.name)
mail_text = 'Hello,\n\nyour Task "{}" on {} is up and running: {}/tasks/?uuid={}\n\nThe Rosetta notifications bot.'.format(task.container.name, task.computing, settings.DJANGO_PUBLIC_HTTP_HOST, task.uuid)
try:
send_email(to=task.user.email, subject=mail_subject, text=mail_text)
except Exception as e:
logger.error('Cannot send task ready email: "{}"'.format(e))
return HttpResponse('OK')
else:
return HttpResponse('Unknown action "{}"'.format(action))
except Exception as e:
logger.error(e)
return HttpResponse(agent_code)
elif action=='set_ip_port':
task_ip = request.GET.get('ip', None)
if not task_ip:
return HttpResponse('IP not valid (got "{}")'.format(task_ip))
task_port = request.GET.get('port', None)
if not task_port:
return HttpResponse('Port not valid (got "{}")'.format(task_port))
try:
int(task_port)
except (TypeError, ValueError):
return HttpResponse('Port not valid (got "{}")'.format(task_port))
# Set fields
logger.info('Setting task "{}" to ip "{}" and port "{}"'.format(task.uuid, task_ip, task_port))
task.status = TaskStatuses.running
task.ip = task_ip
if task.container.supports_dynamic_ports:
task.port = int(task_port)
task.save()
# Notify the user that the task called back home
logger.info('Sending task ready mail notification to "{}"'.format(task.user.email))
mail_subject = 'Your Task "{}" is up and running'.format(task.container.name)
mail_text = 'Hello,\n\nyour Task "{}" on {} is up and running: {}/tasks/?uuid={}\n\nThe Rosetta notifications bot.'.format(task.container.name, task.computing, settings.DJANGO_PUBLIC_HTTP_HOST, task.uuid)
try:
send_email(to=task.user.email, subject=mail_subject, text=mail_text)
except Exception as e:
logger.error('Cannot send task ready email: "{}"'.format(e))
return HttpResponse('OK')
else:
return HttpResponse('Unknown action "{}"'.format(action))
services/webapp/code/rosetta/core_app/computing_managers.py
+ 9
− 9
View file @ 349785e6
......@@ -205,7 +205,7 @@ class RemoteComputingManager(ComputingManager):
else:
binds += ',{}'.format(task.extra_binds)
run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp && mkdir -p /tmp/{}_data/home && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'wget {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export BASE_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
......@@ -262,7 +262,7 @@ class RemoteComputingManager(ComputingManager):
user = task.computing.get_conf_param('user')
# Stop the task remotely
stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.pid)
stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.pid)
out = os_shell(stop_command, capture=True)
if out.exit_code != 0:
if not 'No such process' in out.stderr:
......@@ -286,7 +286,7 @@ class RemoteComputingManager(ComputingManager):
user = task.computing.get_conf_param('user')
# View log remotely
view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}_data/task.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}_data/task.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
out = os_shell(view_log_command, capture=True)
if out.exit_code != 0:
......@@ -364,7 +364,7 @@ class SlurmComputingManager(ComputingManager):
else:
binds += ',{}'.format(task.extra_binds)
run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT && {} '.format(authstring)
run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
......@@ -431,7 +431,7 @@ class SlurmComputingManager(ComputingManager):
user = task.computing.get_conf_param('user')
# Stop the task remotely
stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid)
stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid)
out = os_shell(stop_command, capture=True)
if out.exit_code != 0:
raise Exception(out.stderr)
......@@ -454,7 +454,7 @@ class SlurmComputingManager(ComputingManager):
user = task.computing.get_conf_param('user')
# View log remotely
view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
out = os_shell(view_log_command, capture=True)
if out.exit_code != 0:
......@@ -518,7 +518,7 @@ class RemotehopComputingManager(ComputingManager):
else:
binds += ',{}'.format(task.extra_binds)
run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
run_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} /bin/bash -c \''.format(second_user, second_host)
if use_agent:
......@@ -592,7 +592,7 @@ class RemotehopComputingManager(ComputingManager):
second_user = task.computing.get_conf_param('second_user')
# Stop the task remotely
stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
stop_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
stop_command += 'kill -9 {}"'.format(task.pid)
......@@ -621,7 +621,7 @@ class RemotehopComputingManager(ComputingManager):
second_user = task.computing.get_conf_param('second_user')
# View log remotely
view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
view_log_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
view_log_command += 'cat \\\\\\$HOME/{}.log"'.format(task.uuid)
......
services/webapp/code/rosetta/core_app/decorators.py
+ 13
− 17
View file @ 349785e6
......@@ -51,18 +51,16 @@ def public_view(wrapped_view):
error_text = str(e)
else:
# Raise te exception if we are in debug mode
# Log the exception
logger.error(format_exception(e))
# Raise the exception if we are in debug mode
if settings.DEBUG:
raise
# Otherwise,
# Otherwise, mask it
else:
# first log the exception
logger.error(format_exception(e))
# and then mask it.
error_text = 'something went wrong'
error_text = 'something went wrong ({})'.format(e)
data = {'user': request.user,
'title': 'Error',
......@@ -111,18 +109,16 @@ def private_view(wrapped_view):
error_text = str(e)
else:
# Raise te exception if we are in debug mode
# Log the exception
logger.error(format_exception(e))
# Raise the exception if we are in debug mode
if settings.DEBUG:
raise
# Otherwise,
# Otherwise, mask it
else:
# first log the exception
logger.error(format_exception(e))
# and then mask it.
error_text = 'something went wrong'
error_text = 'something went wrong ({})'.format(e)
data = {'user': request.user,
'title': 'Error',
......
services/webapp/code/rosetta/settings.py
+ 2
− 1
View file @ 349785e6
......@@ -133,7 +133,8 @@ USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.2/howto/static-files/
STATIC_URL = '/static/'
STATIC_URL = '/static/' # URL path
STATIC_ROOT = '/rosetta/static' # Filesystem path
# REST framework settings
......
services/webapp/requirements.txt
+ 1
− 0
View file @ 349785e6
......@@ -6,3 +6,4 @@ django-rest-swagger==2.2.0
dateutils==0.6.6
sendgrid==5.3.0
mozilla-django-oidc==1.2.4
uwsgi==2.0.19.1
services/webapp/run_webapp.sh
+ 44
− 2
View file @ 349785e6
......@@ -36,6 +36,48 @@ if [[ "x$EXIT_CODE" != "x0" ]] ; then
fi
echo ""
if [[ "x$DJANGO_DEV_SERVER" == "xTrue" ]] ; then
# Run the (development) server
echo "Now starting the server and logging in /var/log/webapp/server.log."
exec python3 manage.py runserver 0.0.0.0:8080 2>> /var/log/webapp/server.log
echo "Now starting the development server and logging in /var/log/webapp/server.log."
exec python3 manage.py runserver 0.0.0.0:8080 2>> /var/log/webapp/server.log
else
# Move to the code dir
cd /opt/code
# Collect static
echo "Collecting static files..."
python3 manage.py collectstatic
# Run uWSGI
echo "Now starting the uWSGI server and logging in /var/log/webapp/server.log."
uwsgi --chdir=/opt/code \
--module=rosetta.wsgi \
--env DJANGO_SETTINGS_MODULE=rosetta.settings \
--master --pidfile=/tmp/project-master.pid \
--socket=127.0.0.1:49152 \
--static-map /static=/rosetta/static \
--http :8080 \
--disable-logging 2>> /var/log/webapp/server.log
fi
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment