Added support for uWSGI together with static files serivng. Added the...

Added support for uWSGI together with static files serivng. Added the DJANGO_DEV_SERVER env var to switch between the dev server and uWSGI. Small changes in error handling to play nicer with logs when not running in dev mode.

services/webapp/code/rosetta/core_app/api.py

@@ -209,7 +209,6 @@ class UserViewSet(viewsets.ModelViewSet):
 class agent_api(PublicGETAPI):
     
     def _get(self, request):
-        try:
         
         task_uuid = request.GET.get('task_uuid', None)
         if not task_uuid:
@@ -326,23 +325,4 @@ print(port)
             return HttpResponse('Unknown action "{}"'.format(action))
 
 
-        except Exception as e:
-            logger.error(e)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 

services/webapp/code/rosetta/core_app/computing_managers.py

@@ -205,7 +205,7 @@ class RemoteComputingManager(ComputingManager):
                 else:
                     binds += ',{}'.format(task.extra_binds)
             
-            run_command  = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
+            run_command  = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
             run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp && mkdir -p /tmp/{}_data/home && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid) 
             run_command += 'wget {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export BASE_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
             run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
@@ -262,7 +262,7 @@ class RemoteComputingManager(ComputingManager):
         user = task.computing.get_conf_param('user')
 
         # Stop the task remotely
-        stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.pid)
+        stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.pid)
         out = os_shell(stop_command, capture=True)
         if out.exit_code != 0:
             if not 'No such process' in out.stderr:
@@ -286,7 +286,7 @@ class RemoteComputingManager(ComputingManager):
         user = task.computing.get_conf_param('user')
 
         # View log remotely
-        view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}_data/task.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
+        view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}_data/task.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
 
         out = os_shell(view_log_command, capture=True)
         if out.exit_code != 0:
@@ -364,7 +364,7 @@ class SlurmComputingManager(ComputingManager):
                 else:
                     binds += ',{}'.format(task.extra_binds)
 
-            run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
+            run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
             run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
             run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT && {} '.format(authstring)
             run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
@@ -431,7 +431,7 @@ class SlurmComputingManager(ComputingManager):
         user = task.computing.get_conf_param('user')
 
         # Stop the task remotely
-        stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid)
+        stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid)
         out = os_shell(stop_command, capture=True)
         if out.exit_code != 0:
             raise Exception(out.stderr)
@@ -454,7 +454,7 @@ class SlurmComputingManager(ComputingManager):
         user = task.computing.get_conf_param('user')
 
         # View log remotely
-        view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
+        view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
 
         out = os_shell(view_log_command, capture=True)
         if out.exit_code != 0:
@@ -518,7 +518,7 @@ class RemotehopComputingManager(ComputingManager):
                 else:
                     binds += ',{}'.format(task.extra_binds)
 
-            run_command  = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
+            run_command  = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
             run_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} /bin/bash -c \''.format(second_user, second_host)
             
             if use_agent:
@@ -592,7 +592,7 @@ class RemotehopComputingManager(ComputingManager):
         second_user = task.computing.get_conf_param('second_user')
 
         # Stop the task remotely
-        stop_command  = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
+        stop_command  = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
         stop_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
         stop_command += 'kill -9 {}"'.format(task.pid)
 
@@ -621,7 +621,7 @@ class RemotehopComputingManager(ComputingManager):
         second_user = task.computing.get_conf_param('second_user')
 
         # View log remotely
-        view_log_command  = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
+        view_log_command  = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
         view_log_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)
         view_log_command += 'cat \\\\\\$HOME/{}.log"'.format(task.uuid)
 

services/webapp/code/rosetta/core_app/decorators.py

@@ -51,18 +51,16 @@ def public_view(wrapped_view):
                 error_text = str(e)
             else:
 
-                # Raise te exception if we are in debug mode
+                # Log the exception 
+                logger.error(format_exception(e))
+
+                # Raise the exception if we are in debug mode
                 if settings.DEBUG:
                     raise
 
-                # Otherwise,
+                # Otherwise, mask it
                 else:
-
-                    # first log the exception
-                    logger.error(format_exception(e))
-
-                    # and then mask it.
-                    error_text = 'something went wrong'
+                    error_text = 'something went wrong ({})'.format(e)
 
             data = {'user': request.user,
                     'title': 'Error',
@@ -111,18 +109,16 @@ def private_view(wrapped_view):
                     error_text = str(e)
                 else:
 
-                    # Raise te exception if we are in debug mode
+                    # Log the exception 
+                    logger.error(format_exception(e))
+    
+                    # Raise the exception if we are in debug mode
                     if settings.DEBUG:
                         raise
     
-                    # Otherwise,
+                    # Otherwise, mask it
                     else:
-
-                        # first log the exception
-                        logger.error(format_exception(e))
-
-                        # and then mask it.
-                        error_text = 'something went wrong'
+                        error_text = 'something went wrong ({})'.format(e)
 
                 data = {'user': request.user,
                         'title': 'Error',

services/webapp/code/rosetta/settings.py

@@ -133,7 +133,8 @@ USE_TZ = True
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/2.2/howto/static-files/
 
-STATIC_URL = '/static/'
+STATIC_URL = '/static/'         # URL path
+STATIC_ROOT = '/rosetta/static' # Filesystem path
 
 
 # REST framework settings

services/webapp/requirements.txt

@@ -6,3 +6,4 @@ django-rest-swagger==2.2.0
 dateutils==0.6.6
 sendgrid==5.3.0
 mozilla-django-oidc==1.2.4
+uwsgi==2.0.19.1

services/webapp/run_webapp.sh

@@ -36,6 +36,48 @@ if [[ "x$EXIT_CODE" != "x0" ]] ; then
 fi
 echo ""
 
+
+if [[ "x$DJANGO_DEV_SERVER" == "xTrue" ]] ; then
+    
 # Run the (development) server
-echo "Now starting the server and logging in /var/log/webapp/server.log."
+    echo "Now starting the development server and logging in /var/log/webapp/server.log."
     exec python3 manage.py runserver 0.0.0.0:8080 2>> /var/log/webapp/server.log
+
+else
+    # Move to the code dir
+    cd /opt/code
+    
+    # Collect static
+    echo "Collecting static files..."
+    python3 manage.py collectstatic
+
+    # Run uWSGI
+    echo "Now starting the uWSGI server and logging in /var/log/webapp/server.log."
+
+	uwsgi --chdir=/opt/code \
+	      --module=rosetta.wsgi \
+	      --env DJANGO_SETTINGS_MODULE=rosetta.settings \
+	      --master --pidfile=/tmp/project-master.pid \
+	      --socket=127.0.0.1:49152 \
+	      --static-map /static=/rosetta/static \
+	      --http :8080 \
+	      --disable-logging 2>> /var/log/webapp/server.log
+fi
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+