Added first Web App skeleton. Updated README.

.gitignore

@@ -14,3 +14,5 @@ __pycache__/
 # Data
 data*
 
+# DB conf
+images/webapp/db_conf.sh

README.md

@@ -27,13 +27,53 @@ Run
 
 	$ rosetta/run
 
-Check status
 
-	$ rosetta/ps
+Play
+
+    You can now point your browser to http://localhost:8080
+
+Clean
+
+	# rosetta/clean
+
+### Extras
+
+Check status (not yet fully supported)
+
+    # rosetta/status
+
+
+Run Web App unit tests (with Rosetta running)
+
+    ./run_webapp_unit_tests.sh
 
 
 ### Building errors
 
 It is common for the build process to fail with a "404 not found" error on an apt-get instrucions, as apt repositories often change their IP addresses. In such case, try:
 
-    $ rosetta/build nocache
+    $ rosetta/build nocache
\ No newline at end of file
+
+
+### Development mode
+
+Django development server is running on port 8080 of the "webapp" service.
+
+To enable live code changes, add or comment out the following in docker-compose.yaml under the "volumes" section of the "webapp" service:
+
+    - ./images/webapp/code:/opt/webapp_code
+    
+This will mount the code from images/webapp/code as a volume inside the webapp container itself allowing to make immediately effective codebase edits.
+
+Note that when you edit the Django ORM model, you need to rerun the migrate the database, either by just rerunning the webapp service:
+
+    $ rosetta/rerun webapp
+
+..ora by entering in the webapp service container and manually migrate:
+
+    $ rosetta/shell webapp
+    $ source /env.sh
+    $ source /db_conf.sh
+    $ cd /opt/webapp_code
+    $ python3 manage.py makemigrations
+    $ python3 manage.py migrate  
\ No newline at end of file

docker-compose.yml

@@ -33,3 +33,26 @@ services:
       - ./data/dregistry:/var/lib/registry
     ports:
       - "5000:5000"
+
+  webapp:
+    image: "rosetta/webapp"
+    container_name: webapp
+    hostname: webapp
+    environment:
+      - SAFEMODE=False
+      - DJANGO_LOG_LEVEL=CRITICAL
+      - ROSETTA_LOG_LEVEL=DEBUG 
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./data_rosetta/webapp/data:/data
+      - ./data_rosetta/webapp/log:/var/log/webapp
+      #- ./images/webapp/code:/opt/webapp_code
+
+
+
+
+
+
+
+

images/webapp/.gitignore

+# From https://github.com/github/gitignore/blob/master/Python.gitignore
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don’t work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+virtual/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# 
+.docker_bash_history

images/webapp/Dockerfile

+FROM rosetta/base
+MAINTAINER Stefano Alberto Russo <stefano.russo@gmail.com>
+
+# Always start with an apt-get update when extending base images,
+# otherwise apt repositories might get outdated (404 not found)
+# and building without cache does not re-build base images.
+RUN apt-get update
+
+#------------------------------
+# Apt requirements
+#------------------------------
+
+# Install Curl
+RUN apt-get install curl -y
+
+# Download get-pip script
+RUN curl -O https://bootstrap.pypa.io/get-pip.py
+
+# Install Python3 and Pip3 (python3-distutils required for pip3)
+RUN apt-get install python3 python3-distutils -y 
+
+# Install Python and pip in this order (first Python 3 and then Python 2), or 
+# you will end ap with python defaulting to python2 and pip defaulting to pip3
+# Otherwise, do somethign like "ln -s /usr/local/bin/pip3 /usr/local/bin/pip"
+
+# Install Python3 and Pip3 (ython3-distutils required for pip3)
+RUN apt-get install python3 python3-distutils -y 
+RUN python3 get-pip.py 'pip==10.0.1'
+
+# Install Python2 and Pip2
+RUN apt-get install python -y
+RUN python get-pip.py 'pip==10.0.1'
+
+# Python 3 dev (for pycrypto)
+RUN apt-get install python3-dev -y
+
+# Install postgres driver required for psycopg2
+RUN apt-get install libpq-dev -y
+
+
+#------------------------------
+# Install Django project
+#------------------------------
+
+# Prepare dir
+RUN mkdir /opt/webapp_code
+
+# Install Python requirements..
+COPY requirements.txt /tmp/
+RUN cd /opt/webapp_code && pip3 install -r /tmp/requirements.txt
+
+# Patch Django 2.2 non-ascii chars in /usr/local/lib/python3.6/dist-packages/django/views/templates/technical_500.html
+RUN sed -i 's/[\x80-\xFF]/./g' /usr/local/lib/python3.6/dist-packages/django/views/templates/technical_500.html
+
+# Install App code
+COPY code /opt/webapp_code
+
+# Fix permissions
+RUN chown -R rosetta:rosetta /opt/webapp_code
+
+# Copy db conf
+COPY db_conf.sh /db_conf.sh
+
+# Prepare for logs
+RUN mkdir /var/log/webapp/ && chown rosetta:rosetta /var/log/webapp/
+
+
+#------------------------------
+# Supervisord
+#------------------------------
+
+COPY run_webapp.sh /etc/supervisor/conf.d/
+RUN chmod 755 /etc/supervisor/conf.d/run_webapp.sh
+COPY supervisord_webapp.conf /etc/supervisor/conf.d/
+
+
+#------------------------------
+# Prestartup
+#------------------------------
+
+COPY prestartup_webapp.sh /prestartup/
+
+
+
+
+
+
+
+
+

images/webapp/code/manage.py

+#!/usr/bin/env python3
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'rosetta.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

images/webapp/code/migrations_vanilla/README.txt

+
+This is a folder with just an __init__.py file, used to initialize the migrations.
+

images/webapp/code/rosetta/base_app/api.py

+import logging
+ 
+# Django imports
+from django.http import HttpResponse
+from django.utils import timezone
+from django.contrib.auth import authenticate, login, logout
+from django.contrib.auth.models import User, Group
+
+from rest_framework.response import Response
+from rest_framework import status, serializers, viewsets
+from rest_framework.views import APIView
+
+ 
+# Project imports
+from rosetta.common import format_exception
+from rosetta.base_app.models import Profile
+ 
+# Setup logging
+logger = logging.getLogger(__name__)
+
+
+#==============================
+#  Common returns
+#==============================
+ 
+# Ok (with data)
+def ok200(data=None):
+    return Response({"results": data}, status=status.HTTP_200_OK)
+ 
+# Error 400
+def error400(data=None):
+    return Response({"detail": data}, status=status.HTTP_400_BAD_REQUEST)
+ 
+# Error 401
+def error401(data=None):
+    return Response({"detail": data}, status=status.HTTP_401_UNAUTHORIZED)
+ 
+# Error 404
+def error404(data=None):
+    return Response({"detail": data}, status=status.HTTP_404_NOT_FOUND)
+ 
+# Error 500
+def error500(data=None):
+    return Response({"detail": data}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+
+ 
+#==============================
+#  Authentication helper
+#==============================
+ 
+def rosetta_authenticate(request):
+
+    # Get data
+    user      = request.user if request.user.is_authenticated else None
+    username  = request.data.get('username', None)
+    password  = request.data.get('password', None)
+    authtoken = request.data.get('authtoken', None)
+
+    # Try standard user authentication
+    if user:
+        return user
+
+    # Try username/password  authentication
+    elif username or password:
+        
+        # Check we got both
+        if not username:
+            return error400('Got empty username')
+        if not password:
+            return error400('Got empty password')
+ 
+        # Authenticate
+        user = authenticate(username=username, password=password)
+        if not user:
+            return error401('Wrong username/password')  
+        else:
+            login(request, user)
+            return user
+
+    # Try auth toekn authentication 
+    elif authtoken:
+        try:
+            profile = Profile.objects.get(authtoken=authtoken)
+        except Profile.DoesNotExist:
+            return error400('Wrong auth token')
+        login(request, profile.user)
+        return profile.user
+    else:
+        return error401('This is a private API. Login or provide username/password or auth token')
+
+
+
+#==============================
+#  Base public API class
+#==============================
+ 
+class PublicPOSTAPI(APIView):
+    '''Base public POST API class'''
+ 
+    # POST
+    def post(self, request):
+        try:
+            return self._post(request)
+        except Exception as e:
+            logger.error(format_exception(e))
+            return error500('Got error in processing request: {}'.format(e))
+ 
+class PublicGETAPI(APIView):
+    '''Base public GET API class''' 
+    # GET
+    def get(self, request):
+        try:
+            return self._get(request)
+        except Exception as e:
+            logger.error(format_exception(e))
+            return error500('Got error in processing request: {}'.format(e))
+
+
+
+#==============================
+#  Base private API class
+#==============================
+ 
+class PrivatePOSTAPI(APIView):
+    '''Base private POST API class'''
+ 
+    # POST
+    def post(self, request):
+        try:
+            # Authenticate using rosetta authentication
+            response = rosetta_authenticate(request)
+             
+            # If we got a response return it, otherwise set it as the user.
+            if isinstance(response, Response):
+                return response
+            else:
+                self.user = response
+             
+            # Call API logic
+            return self._post(request)
+        except Exception as e:
+            logger.error(format_exception(e))
+            return error500('Got error in processing request: {}'.format(e))
+ 
+class PrivateGETAPI(APIView):
+    '''Base private GET API class'''
+
+    # GET  
+    def get(self, request):
+        try:
+            # Authenticate using rosetta authentication
+            response = rosetta_authenticate(request)
+             
+            # If we got a response return it, otherwise set it as the user.
+            if isinstance(response, Response):
+                return response
+            else:
+                self.user = response
+             
+            # Call API logic
+            return self._get(request)
+        except Exception as e:
+            logger.error(format_exception(e))
+            return error500('Got error in processing request: {}'.format(e))
+
+
+
+#==============================
+#  User & profile APIs
+#==============================
+
+class login_api(PrivateGETAPI, PrivatePOSTAPI):
+    """
+    get:
+    Returns the auth token.
+
+    post:
+    Authorize and returns the auth token.
+    """
+         
+    def _post(self, request):
+        return ok200({'authtoken': self.user.profile.authtoken})
+
+    def _get(self, request):
+        return ok200({'authtoken': self.user.profile.authtoken}) 
+ 
+ 
+class logout_api(PrivateGETAPI):
+    """
+    get:
+    Logout the user
+    """
+         
+    def _get(self, request):
+        logout(request)
+        return ok200()
+
+
+class UserViewSet(viewsets.ModelViewSet):
+    """
+    API endpoint that allows Users to be viewed or edited.
+    """
+
+    class UserSerializer(serializers.HyperlinkedModelSerializer):
+        class Meta:
+            model = User
+            fields = ('url', 'username', 'email', 'groups')
+
+    queryset = User.objects.all().order_by('-date_joined')    
+    serializer_class = UserSerializer
+
+
+
+
+
+
+
+
+

images/webapp/code/rosetta/base_app/management/commands/base_app_populate.py

+from django.core.management.base import BaseCommand
+from django.contrib.auth.models import User
+from ...models import Profile
+
+class Command(BaseCommand):
+    help = 'Adds the admin superuser with \'a\' password.'
+
+    def handle(self, *args, **options):
+
+        try:
+            User.objects.get(username='admin')
+            print('Not creating admin user as it already exist')
+        except User.DoesNotExist:
+            print('Creating admin user with default password')
+            admin = User.objects.create_superuser('admin', 'admin@example.com', 'a')
+            Profile.objects.create(user=admin)
+
+        try:
+            User.objects.get(username='testuser')
+            print('Not creating test user as it already exist')
+        except User.DoesNotExist:
+            print('Creating test user with default password')
+            testuser = User.objects.create_user('testuser', 'testuser@rosetta.platform', 'testpass')
+            Profile.objects.create(user=testuser, authtoken='129aac94-284a-4476-953c-ffa4349b4a50')
+            
\ No newline at end of file

images/webapp/code/rosetta/base_app/migrations

+/data/migrations/base_app
\ No newline at end of file

images/webapp/code/rosetta/base_app/models.py

+import uuid
+from django.db import models
+from django.contrib.auth.models import User
+
+
+#=========================
+#  Profile 
+#=========================
+
+class Profile(models.Model):
+    user      = models.OneToOneField(User, on_delete=models.CASCADE)
+    timezone  = models.CharField('User Timezone', max_length=36, default='UTC')
+    authtoken = models.CharField('User auth token', max_length=36, blank=True, null=True)
+
+    def save(self, *args, **kwargs):
+        if not self.authtoken:
+            self.authtoken = str(uuid.uuid4())
+        super(Profile, self).save(*args, **kwargs)
+
+    def __unicode__(self):
+        return str('Profile of user "{}"'.format(self.user.username))
+
+
+
+
+
+
+
+
+
+

images/webapp/code/rosetta/base_app/templates/login.html

+<html>
+    <head>
+        <title>Rosetta WebApp</title>
+    </head>
+    <body>
+        <div style="text-align:center; margin-top:50px">
+            {% if data.error %}
+            <div style="color:red">
+              {{ data.error }}
+            </div>
+            {% elif data.success %}
+            <div style="color:green">
+              {{data.success}}
+            </div>
+            {% else %}            
+            <br/>
+            {% endif %} 
+            
+            <form class="form-signin" role="form" action='/login/' method='POST'>
+            {% csrf_token %}
+            <input style="max-width:300px; margin: 5px auto" type="username" class="form-control" placeholder="Username" name='username' required autofocus>
+            <input style="max-width:300px; margin: 5px auto" type="password" class="form-control" placeholder="Password" name='password'>
+            <input style="width:100px; height:35px; margin: 5px auto; font-size:16px" type='submit' class="btn btn-success" value='Log in' />
+            </form>
+        </div>
+    </body>
+    
+
+</html>
+

images/webapp/code/rosetta/base_app/templates/main.html

+<html>
+    <head>
+        <title>Rosetta WebApp</title>
+    </head>
+    <body>
+        <div style="text-align:center; margin-top:50px">
+            <h1>Rosetta WebApp</h1>
+            {% if data.error %}
+            <div style="color:red">
+              {{ data.error }}
+            </div>
+            {% elif data.success %}
+            <div style="color:green">
+              {{data.success}}
+            </div>
+            {% else %}            
+            <br/>
+            {% endif %} 
+            
+            {% if request.user.is_authenticated %}
+            Logged in as <b>{{ request.user.username }}</b> | <a href="/logout">Logout</a>
+            {% else %}
+            <form class="form-signin" role="form" action='/login/' method='POST'>
+            {% csrf_token %}
+            <input style="max-width:300px; margin: 5px auto" type="username" class="form-control" placeholder="Username" name='username' required autofocus>
+            <input style="max-width:300px; margin: 5px auto" type="password" class="form-control" placeholder="Password" name='password'>
+            <input style="width:100px; height:35px; margin: 5px auto; font-size:16px" type='submit' class="btn btn-success" value='Log in' />
+            </form>
+            {% endif %}
+        </div>
+
+        <div style="max-width:500px; margin: 30px auto">
+            Modules:
+            <ul style="margin-top:5px; margin-bottom:20px">
+                <li><a href="/admin">Admin</a></li>
+                <li><a href="/api/v1/doc">Swagger APIs v1 Doc</a></li>
+            </ul>
+
+            REST APIs v1:
+            <ul style="margin-top:5px">
+                <li><a href="/api/v1/base/login/">/api/v1/base/login/</a></li>
+                <li><a href="/api/v1/base/logout/">/api/v1/base/logout/</a></li>
+            </ul>
+            
+        </div>
+        
+    </body>
+    
+
+</html>
+

images/webapp/code/rosetta/base_app/tests/common.py

+import json
+from django.test import TestCase
+from rest_framework.test import APIClient as Client
+from django.test.client import MULTIPART_CONTENT
+from rest_framework import status
+from rest_framework.reverse import reverse
+from django.contrib.auth.models import User
+
+class APIClient(Client):
+    # Add patch to test client object
+    def patch(self, path, data='', content_type=MULTIPART_CONTENT, follow=False, **extra):
+        return self.generic('PATCH', path, data, content_type, **extra)
+
+    # Add options to test client object
+    def options(self, path, data='', content_type=MULTIPART_CONTENT, follow=False, **extra):
+        return self.generic('OPTIONS', path, data, content_type, **extra)
+
+
+class BaseAPITestCase(TestCase):
+
+    def __init__(self, *args, **kwargs):
+        self.maxDiff = None
+        super(TestCase, self).__init__(*args, **kwargs)
+
+    def send_request(self, request_method, *args, **kwargs):
+
+        request_func = getattr(self.client, request_method)
+        status_code  = None
+       
+        if 'multipart' in kwargs and kwargs['multipart'] is True:
+            # Do nothing, this is a "special", multipart request 
+            pass
+        else:
+            if 'content_type' not in kwargs and request_method != 'get':
+                kwargs['content_type'] = 'application/json'
+            
+            if 'data' in kwargs and request_method != 'get' and kwargs['content_type'] == 'application/json':
+                data = kwargs.get('data', '')
+                kwargs['data'] = json.dumps(data)
+
+        if 'status_code' in kwargs:
+            status_code = kwargs.pop('status_code')
+
+        self.response = request_func(*args, **kwargs)
+
+        # Parse response
+        is_json = False
+
+        if 'content-type' in self.response._headers:
+            is_json = bool(filter(lambda x: 'json' in x, self.response._headers['content-type']))
+
+        try:
+            if is_json and self.response.content:
+                self.response.content_dict = json.loads(self.response.content)
+            else:
+                self.response.content_dict = {}
+            
+        except:
+            self.response.content_dict = {}
+        
+        if status_code:
+            if not self.response.status_code == status_code:
+                raise Exception('Error with response:' + str(self.response))
+        return self.response
+
+    def post(self, *args, **kwargs):
+        return self.send_request('post', *args, **kwargs)
+
+    def get(self, *args, **kwargs):
+        return self.send_request('get', *args, **kwargs)
+
+    def put(self, *args, **kwargs):
+        return self.send_request('put', *args, **kwargs)
+
+    def delete(self, *args, **kwargs):
+        return self.send_request('delete', *args, **kwargs)
+
+    def patch(self, *args, **kwargs):
+        return self.send_request('patch', *args, **kwargs)
+
+    def init(self):
+        self.client = APIClient()
+
+    def assertRedirects(self, *args, **kwargs):
+        super(BaseAPITestCase, self).assertRedirects(*args, **kwargs)
+
+
+
+
+
+
+
+

images/webapp/code/rosetta/base_app/tests/test_apis.py

+import json
+
+from django.contrib.auth.models import User
+        
+from .common import BaseAPITestCase
+from ..models import Profile
+
+class ApiTests(BaseAPITestCase):
+
+    def setUp(self):
+        
+        # Create test users
+        self.user = User.objects.create_user('testuser', password='testpass')
+        self.anotheruser = User.objects.create_user('anotheruser', password='anotherpass')
+
+        # Create test profile
+        Profile.objects.create(user=self.user, authtoken='ync719tce917tec197t29cn712eg')
+
+
+    def test_api_web_auth(self):
+        '''Test auth using login api''' 
+         
+        # No user at all
+        resp = self.post('/api/v1/base/login/', data={})
+        self.assertEqual(resp.status_code, 401)
+        self.assertEqual(json.loads(resp.content), {"detail": "This is a private API. Login or provide username/password or auth token"})
+
+        # Wrong user
+        resp = self.post('/api/v1/base/login/', data={'username':'wronguser', 'password':'testpass'})
+        self.assertEqual(resp.status_code, 401)
+        self.assertEqual(json.loads(resp.content), {"detail": "Wrong username/password"})
+
+        # Wrong pass
+        resp = self.post('/api/v1/base/login/', data={'username':'testuser', 'password':'wrongpass'})
+        self.assertEqual(resp.status_code, 401)
+        self.assertEqual(json.loads(resp.content), {"detail": "Wrong username/password"})
+
+        # Correct user
+        resp = self.post('/api/v1/base/login/', data={'username': 'testuser', 'password':'testpass'})
+        self.assertEqual(resp.status_code, 200)
+
+
+
+
+
+
+
+
+        
\ No newline at end of file

images/webapp/code/rosetta/base_app/views.py

+from django.shortcuts import render
+from django.http import HttpResponseRedirect
+from django.contrib.auth import authenticate, login, logout
+
+# Setup logging
+import logging
+logger = logging.getLogger(__name__)
+
+class ErrorMessage(Exception):
+    pass
+
+
+def main_view(request):
+    return render(request, 'main.html')
+ 
+
+def login_view(request):
+
+    data={}
+
+    if request.method == 'POST':
+        username = request.POST.get('username')
+        password = request.POST.get('password')
+        
+        if (not username) or (not password):
+            data['error'] = 'Empty username or password'
+        
+        if request.user.is_authenticated:
+            logout(request)
+            
+        user = authenticate(username=username, password=password)
+        if user:
+            login(request, user)
+            return HttpResponseRedirect('/')
+        else:
+            data['error'] = 'Check username and password'
+
+    
+    # Render the login page again with no other data than title
+    return render(request, 'login.html', {'data': data})
+
+
+
+def logout_view(request):
+    logout(request)
+    return HttpResponseRedirect('/')
+
+
+
+
+
+
+
+
+