Merge pull request #89 from bcgov/feature/DIV-1041

Add Redis storage POC

.env.example

@@ -11,3 +11,9 @@ DATABASE_PORT=
 CLAMAV_ENABLED=True
 CLAMAV_TCP_PORT=3310
 CLAMAV_TCP_ADDR=localhost
+
+# Redis settings
+REDIS_HOST=localhost
+REDIS_PORT=6379
+REDIS_DB=
+REDIS_PASSWORD=

README.md

@@ -50,7 +50,11 @@ To run this project in your development machine, follow these steps:
         docker run -d -p 5005:5001 aquavitae/weasyprint
         ```
 
-9. Open your browser and go to http://127.0.0.1:8000, you will be greeted with the eDivorce homepage.  In dev mode, you can log in with any username and the password 'divorce'.
+9. Start up docker containers:
+
+    `docker-compose up -d`
+
+10. Open your browser and go to http://127.0.0.1:8000, you will be greeted with the eDivorce homepage.  In dev mode, you can log in with any username and the password 'divorce'.
 
 ### SCSS Compilation
 

docker-compose.yml

@@ -10,3 +10,29 @@ services:
       - "3310:3310"
     restart: always
 
+  # Redis Server
+  redis:
+    container_name: edivorce-redis
+    image: redis
+    command: redis-server --requirepass admin
+    ports:
+      - "6379:6379"
+    volumes:
+      - data-redis:/data
+    restart: always
+
+  # Redis Commander
+  redis-commander:
+    container_name: edivorce-redis-commander
+    hostname: redis-commander
+    image: rediscommander/redis-commander:latest
+    restart: always
+    environment:
+      - REDIS_PORT=6379
+      - REDIS_HOST=redis
+      - REDIS_PASSWORD=admin
+    ports:
+      - "8082:8081"
+
+volumes:
+  data-redis:

edivorce/apps/core/redis.py

@@ -0,0 +1,103 @@
+"""
+Redis storages backend to help store binary data.
+"""
+import base64
+import io
+import redis
+import uuid
+import re
+
+from shutil import copyfileobj
+from tempfile import SpooledTemporaryFile
+
+from django.conf import settings
+from django.core.files.base import File
+from django.core.files.storage import Storage
+from django.utils.deconstruct import deconstructible
+
+EX_EXPIRY = 60*24*7  # 1 week expiry
+
+
+def generate_unique_filename(instance, filename):
+    return '{}_{}'.format(uuid.uuid4(), re.sub('[^0-9a-zA-Z]+', '_', filename))
+
+
+class RedisFile(File):
+
+    def __init__(self, name, storage):
+        self.name = name
+        self._storage = storage
+        self._file = None
+
+    def _get_file(self):
+        if self._file is None:
+            self._file = SpooledTemporaryFile()
+
+            # get from redis
+            content = self._storage.client.get(self.name)
+
+            # stored as base64 .. decode
+            content = base64.b64decode(content)
+
+            with io.BytesIO(content) as file_content:
+                copyfileobj(file_content, self._file)
+
+            self._file.seek(0)
+        return self._file
+
+    def _set_file(self, value):
+        self._file = value
+
+    file = property(_get_file, _set_file)
+
+
+@deconstructible
+class RedisStorage(Storage):
+    def __init__(self):
+        self.client = redis.Redis(
+            host=settings.REDIS_HOST,
+            port=settings.REDIS_PORT,
+            db=settings.REDIS_DB,
+            password=settings.REDIS_PASSWORD
+        )
+
+    def _full_key_name(self, name):
+        return name
+
+    def delete(self, name):
+        self.client.delete(self._full_key_name(name))
+
+    def exists(self, name):
+        return self.client.exists(self._full_key_name(name))
+
+    def listdir(self, path):
+        return '', ''
+
+    def size(self, name):
+        return ''
+
+    def url(self, name):
+        return ''
+
+    def _open(self, name, mode='rb'):
+        remote_file = RedisFile(self._full_key_name(name), self)
+        return remote_file
+
+    def _save(self, name, content):
+        content.open()
+
+        data = base64.b64encode(content.read())
+        self.client.set(self._full_key_name(name), data, ex=EX_EXPIRY)
+
+        content.close()
+        return name
+
+    def get_available_name(self, name, max_length=None):
+        """
+        Allow storage backend to generate a new name if there is already an existing file. Not used for this Redis
+        implementation.
+        """
+        name = self._full_key_name(name)
+        return super().get_available_name(name, max_length)
+
+

edivorce/apps/core/tests/test_storage.py

@@ -0,0 +1,94 @@
+from unittest import mock
+
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.test import TransactionTestCase
+from redis.exceptions import ConnectionError
+
+from edivorce.apps.core.redis import generate_unique_filename
+from edivorce.apps.poc.models import Document
+
+
+class UploadStorageTests(TransactionTestCase):
+
+    @mock.patch('redis.connection.ConnectionPool.get_connection')
+    def test_storage_connection_error(self, mock_redis):
+        mock_redis.side_effect = ConnectionError()
+
+        original_count = Document.objects.count()
+        connection_error = False
+
+        try:
+            file = SimpleUploadedFile('file.txt', b'this is some content')
+            test = Document()
+            test.file = file
+            test.save()
+        except ConnectionError:
+            connection_error = True
+
+        self.assertTrue(connection_error)
+        self.assertEqual(Document.objects.count(), original_count)
+
+    @mock.patch('edivorce.apps.core.redis.RedisStorage.get_available_name')
+    @mock.patch('edivorce.apps.core.redis.RedisStorage._save')
+    def test_storage_file_name_match(self, mock_redis_an, mock_redis_save):
+        mock_redis_an.return_value = 'file.txt'
+        mock_redis_save.return_value = 'file.txt'
+
+        file = SimpleUploadedFile('file.txt', b'this is some content')
+        test = Document()
+        test.file = file
+        test.save()
+
+        self.assertTrue(mock_redis_save.called)
+        self.assertEqual(test.filename, test.file.name)
+
+    @mock.patch('edivorce.apps.core.redis.RedisStorage.get_available_name')
+    @mock.patch('edivorce.apps.core.redis.RedisStorage._save')
+    def test_storage_redis_storage(self, mock_redis_an, mock_redis_save):
+        mock_redis_an.return_value = '6061bebb-f2be-4a74-8757-c4063f6f6993_file_txt'
+        mock_redis_save.return_value = 'file.txt'
+
+        file = SimpleUploadedFile('file.txt', b'this is some content')
+        test = Document()
+        test.file = file
+        test.save()
+
+        self.assertTrue(mock_redis_save.called)
+        self.assertEqual(Document.objects.count(), 1)
+        test = Document.objects.get(id=test.id)
+        self.assertEqual(test.filename, 'file.txt')
+        self.assertNotEqual(test.file.name, 'file.txt')
+
+    def test_storage_redis_key(self):
+        name = 'file.txt'
+        self.assertNotEqual(generate_unique_filename(None, name), name)
+
+        name = '../../../etc/passwd'
+        self.assertNotEqual(generate_unique_filename(None, name), name)
+        self.assertFalse('../../' in generate_unique_filename(None, name))
+
+        name = '../../../etc/passwd%00.png'
+        self.assertNotEqual(generate_unique_filename(None, name), name)
+        self.assertFalse('../../' in generate_unique_filename(None, name))
+
+        name = '..%2F..%2F..%2Fetc%2F'
+        self.assertNotEqual(generate_unique_filename(None, name), name)
+        self.assertFalse('../../' in generate_unique_filename(None, name))
+
+    @mock.patch('edivorce.apps.core.redis.RedisStorage.get_available_name')
+    @mock.patch('edivorce.apps.core.redis.RedisStorage._save')
+    @mock.patch('edivorce.apps.core.redis.RedisStorage.delete')
+    def test_storage_redis_delete(self, mock_redis_an, mock_redis_save, mock_redis_delete):
+        mock_redis_an.return_value = '6061bebb-f2be-4a74-8757-c4063f6f6993_file_txt'
+        mock_redis_save.return_value = 'file.txt'
+        mock_redis_delete.return_value = True
+
+        file = SimpleUploadedFile('file.txt', b'this is some content')
+        test = Document()
+        test.file = file
+        test.save()
+
+        test.delete()
+
+        self.assertTrue(mock_redis_delete.called)
+

edivorce/apps/core/urls.py

@@ -1,8 +1,6 @@
 from django.conf.urls import url
-from django.conf import settings
 
-from .views import main, system, pdf, api, localdev, poc
-from .decorators import bceid_required
+from .views import main, system, pdf, api, localdev
 
 
 urlpatterns = [
@@ -34,9 +32,4 @@ urlpatterns = [
     url(r'^question/(?P<step>.*)$', main.question, name="question_steps"),
     url(r'^current$', system.current, name="current"),
     url(r'^$', main.home, name="home"),
-]
-
-if settings.DEBUG:
-    urlpatterns = urlpatterns + [
-        url(r'poc/upload', bceid_required(poc.UploadScan.as_view()), name="poc-upload"),
-    ]
+]

edivorce/apps/core/views/poc.py

@@ -1,23 +0,0 @@
-from django.shortcuts import render
-from django.views.generic.edit import FormView
-from django import forms
-
-from ..validators import file_scan_validation
-
-"""
-Everything in this file is considered as proof of concept work and should not be used for production code.
-"""
-
-
-class UploadForm(forms.Form):
-    upload_file = forms.FileField(validators=[file_scan_validation])
-
-
-class UploadScan(FormView):
-    form_class = UploadForm
-    template_name = "poc/upload.html"
-
-    def form_valid(self, form):
-        context = self.get_context_data()
-        context['validation_success'] = True
-        return render(self.request, self.template_name, context)

edivorce/apps/poc/migrations/0001_initial.py

@@ -0,0 +1,22 @@
+# Generated by Django 2.2.15 on 2020-08-23 05:10
+
+from django.db import migrations, models
+import edivorce.apps.core.redis
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Document',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('file', models.FileField(storage=edivorce.apps.core.redis.RedisStorage, upload_to='docs_%Y_%m_%d_%H_%M_%S')),
+            ],
+        ),
+    ]

edivorce/apps/poc/migrations/0002_auto_20200823_0606.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.2.15 on 2020-08-23 06:06
+
+from django.db import migrations, models
+import edivorce.apps.core.redis
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('poc', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='document',
+            name='filename',
+            field=models.CharField(max_length=128, null=True),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='file',
+            field=models.FileField(storage=edivorce.apps.core.redis.RedisStorage(), upload_to=edivorce.apps.core.redis.generate_unique_filename),
+        ),
+    ]

edivorce/apps/poc/models.py

@@ -0,0 +1,27 @@
+from django.db import models
+from django.conf import settings
+
+from edivorce.apps.core import redis
+
+
+class Document(models.Model):
+    """
+    This is only a POC model and should not be loaded on a production system.
+    """
+    filename = models.CharField(max_length=128, null=True)  # saving the original filename separately
+    file = models.FileField(upload_to=redis.generate_unique_filename, storage=redis.RedisStorage())
+
+    def save(self, *args, **kwargs):
+        self.filename = self.file.name
+
+        super(Document, self).save(*args, **kwargs)
+
+    def delete(self, **kwargs):
+        """
+        Override delete so we can delete the Redis object when this instance is deleted.
+        :param kwargs:
+        :return:
+        """
+        self.file.delete(save=False)
+
+        super(Document, self).delete(**kwargs)

edivorce/apps/poc/templates/poc-sidebar.html

@@ -0,0 +1,15 @@
+{% load step_order %}
+{% load load_json %}
+
+<div class="col-flex progress-column">
+    <h4>POC</h4>
+    <a href="{% url 'poc-scan' %}" class="progress-question">
+        <span class="progress-icon"><i class="fa fa-share-alt" aria-hidden="true"></i></span>
+        <span class="progress-content">File Scanning</span>
+    </a>
+    <a href="{% url 'poc-storage' %}" class="progress-question">
+        <span class="progress-icon"><i class="fa fa-info" aria-hidden="true"></i></span>
+        <span class="progress-content">Redis Storage</span>
+    </a>
+
+</div>

edivorce/apps/poc/templates/poc/document_confirm_delete.html

@@ -0,0 +1,27 @@
+{% extends 'base.html' %}
+{% load input_field %}
+{% load step_order %}
+{% load load_json %}
+
+{% block title %}{{ block.super }}: POC{% endblock %}
+
+{% block progress %}{% include "poc-sidebar.html" %}{% endblock %}
+
+{% block content %}
+    <form method="post">{% csrf_token %}
+        <p>Are you sure you want to delete "{{ object }}"?</p>
+        <input type="submit" value="Confirm">
+    </form>
+{% endblock %}
+
+{% block formbuttons %}
+
+{% endblock %}
+
+{% block sidebarNav %}
+<!-- no sidebar -->
+{% endblock %}
+
+{% block sidebar %}
+<!-- no sidebar -->
+{% endblock %}

edivorce/apps/core/templates/poc/upload.html → edivorce/apps/poc/templates/scan.html

@@ -5,7 +5,7 @@
 
 {% block title %}{{ block.super }}: POC{% endblock %}
 
-{% block progress %}{% include "partials/progress.html" %}{% endblock %}
+{% block progress %}{% include "poc-sidebar.html" %}{% endblock %}
 
 {% block content %}
     <h1><small>Proof of Concept:</small>File scanning</h1>

edivorce/apps/poc/templates/storage.html

@@ -0,0 +1,72 @@
+{% extends 'base.html' %}
+{% load input_field %}
+{% load step_order %}
+{% load load_json %}
+
+{% block title %}{{ block.super }}: POC{% endblock %}
+
+{% block progress %}{% include "poc-sidebar.html" %}{% endblock %}
+
+{% block content %}
+    <h1><small>Proof of Concept:</small>File storage</h1>
+
+    <form id="form" method="post" enctype="multipart/form-data">
+        {% csrf_token %}
+
+        <div class="question-well">
+            <h3>Upload a file to store in Redis.</h3>
+            <div class="form-group {% if form.file.errors %}has-error{% elif validation_success %}has-success{% endif %}">
+                <div>{{ form.file }}</div>
+                {% if form.file.errors %}
+                    <span class="help-block">
+                    {% for err in form.file.errors %}{{ err }}{% endfor %}
+                    </span>
+                {% endif %}
+                {% if validation_success %}<span class="help-block">No viruses found</span>{% endif %}
+            </div>
+        </div>
+
+        <div class="form-buttons clearfix">
+            <button type="submit" class="btn btn-primary pull-right">Submit</button>
+        </div>
+
+        <div class="question-well">
+            <h3>Stored documents</h3>
+            <table class="table">
+                <thead>
+                    <tr>
+                        <th>File name</th>
+                        <th>Redis key</th>
+                        <th></th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for document in documents %}
+                    <tr>
+                        <td class="fact-sheet-question">
+                            <a href="{% url "poc-storage-download" document.id %}" target="_blank">
+                            {{ document.filename|default:'' }}
+                            </a>
+                        </td>
+                        <td class="fact-sheet-question">{{ document.file.name }}</td>
+                        <td><a href="{% url "poc-storage-delete" document.id %}" class="delete-link">Delete</a></td>
+                    </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+        </div>
+
+    </form>
+{% endblock %}
+
+{% block formbuttons %}
+
+{% endblock %}
+
+{% block sidebarNav %}
+<!-- no sidebar -->
+{% endblock %}
+
+{% block sidebar %}
+<!-- no sidebar -->
+{% endblock %}

edivorce/apps/poc/urls.py

@@ -0,0 +1,11 @@
+from django.conf.urls import url
+
+from edivorce.apps.poc import views
+from ..core.decorators import bceid_required
+
+urlpatterns = [
+    url(r'scan', bceid_required(views.UploadScan.as_view()), name="poc-scan"),
+    url(r'storage/doc/(?P<document_id>\d+)', bceid_required(views.view_document_file), name="poc-storage-download"),
+    url(r'storage/delete/(?P<pk>\d+)', bceid_required(views.UploadStorageDelete.as_view()), name="poc-storage-delete"),
+    url(r'storage', bceid_required(views.UploadStorage.as_view()), name="poc-storage"),
+]

edivorce/apps/poc/views.py

@@ -0,0 +1,52 @@
+from django.shortcuts import render
+from django.views.generic.edit import FormView, CreateView, DeleteView
+from django import forms
+from django.http import HttpResponse
+
+from edivorce.apps.core.validators import file_scan_validation
+from edivorce.apps.poc.models import Document
+
+"""
+Everything in this file is considered as proof of concept work and should not be used for production code.
+"""
+
+
+class UploadScanForm(forms.Form):
+    upload_file = forms.FileField(validators=[file_scan_validation])
+
+
+class UploadScan(FormView):
+    form_class = UploadScanForm
+    template_name = "scan.html"
+
+    def form_valid(self, form):
+        context = self.get_context_data()
+        context['validation_success'] = True
+        return render(self.request, self.template_name, context)
+
+
+class UploadStorage(CreateView):
+    model = Document
+    fields = ['file']
+    template_name = "storage.html"
+    success_url = '/poc/storage'
+
+    def get_context_data(self, **kwargs):
+        kwargs['documents'] = Document.objects.all()
+        return super(UploadStorage, self).get_context_data(**kwargs)
+
+
+class UploadStorageDelete(DeleteView):
+    model = Document
+    success_url = '/poc/storage'
+
+
+def view_document_file(request, document_id):
+    doc = Document.objects.get(id=document_id)
+
+    content_type = 'application/pdf' if 'pdf' in doc.file.name else 'image/jpeg'
+
+    response = HttpResponse(doc.file.read(), content_type=content_type)
+    response['Content-Disposition'] = 'attachment; filename={}'.format(doc.filename)
+
+    return response

edivorce/settings/base.py

@@ -53,6 +53,12 @@ INSTALLED_APPS = (
     'sass_processor',
 )
 
+# add the POC app only if applicable
+if ENVIRONMENT in ['localdev', 'dev', 'test', 'minishift']:
+    INSTALLED_APPS += (
+        'edivorce.apps.poc',
+    )
+
 MIDDLEWARE = (
     'edivorce.apps.core.middleware.basicauth_middleware.BasicAuthMiddleware',
     'debug_toolbar.middleware.DebugToolbarMiddleware',
@@ -155,3 +161,9 @@ LOGOUT_URL = '/accounts/logout/'
 CLAMAV_ENABLED = env.bool('CLAMAV_ENABLED', True)
 CLAMAV_TCP_PORT = env.int('CLAMAV_TCP_PORT', 3310)
 CLAMAV_TCP_ADDR = env('CLAMAV_TCP_ADDR', 'localhost')
+
+# Redis settings
+REDIS_HOST = env('REDIS_HOST', 'localhost')
+REDIS_PORT = env.int('REDIS_PORT', 6379)
+REDIS_DB = env('REDIS_DB', '')
+REDIS_PASSWORD = env('REDIS_PASSWORD', '')

edivorce/urls.py

@@ -8,6 +8,7 @@ urlpatterns = []
 if settings.ENVIRONMENT in ['localdev', 'dev', 'test', 'minishift']:
     import debug_toolbar
     urlpatterns.append(url(r'^__debug__/', include(debug_toolbar.urls)),)
+    urlpatterns.append(url(r'^poc/', include('edivorce.apps.poc.urls')))
 
 if settings.ENVIRONMENT in ['localdev', 'minishift']:
     urlpatterns.append(url(r'^admin/', admin.site.urls))

requirements.txt

@@ -17,6 +17,7 @@ psycopg2==2.8.5
 python-dotenv==0.14.0
 pytz==2020.1
 rcssmin==1.0.6
+redis==3.5.3
 requests==2.24.0
 rjsmin==1.1.0
 six==1.15.0