From 270a77a2db76eaf96cae1a8baae4ff1e9100fdd6 Mon Sep 17 00:00:00 2001 From: Mike Olund Date: Sun, 19 Feb 2017 17:06:30 -0800 Subject: [PATCH] Updates to the Django project to support parallel localdev and OpenShift configurations --- README.md | 170 ++++++++--------------------- edivorce/apps/core/views/system.py | 9 ++ edivorce/settings/base.py | 44 ++++---- edivorce/settings/local.py | 17 ++- edivorce/settings/openshift.py | 71 ++++++++++++ edivorce/settings/production.py | 1 - edivorce/wsgi.py | 16 --- manage.py | 7 +- requirements.txt | 16 ++- setup.py | 28 +++++ wsgi.py | 21 ++++ 11 files changed, 233 insertions(+), 167 deletions(-) create mode 100644 edivorce/apps/core/views/system.py create mode 100644 edivorce/settings/openshift.py delete mode 100644 edivorce/settings/production.py delete mode 100644 edivorce/wsgi.py create mode 100755 setup.py create mode 100644 wsgi.py diff --git a/README.md b/README.md index 11ee4170..6ceb6df7 100644 --- a/README.md +++ b/README.md @@ -1,122 +1,58 @@ -# Openshift quickstart: Django +# eDivorce -This is a [Django](http://www.djangoproject.com) project that you can use as the starting point to develop your own and deploy it on an [OpenShift](https://github.com/openshift/origin) cluster. +This is a [Django](http://www.djangoproject.com) project forked from the [openshift/django-ex](https://github.com/openshift/django-ex) repository. -The steps in this document assume that you have access to an OpenShift deployment that you can deploy applications on. - -## What has been done for you - -This is a minimal Django 1.8 project. It was created with these steps: - -1. Create a virtualenv -2. Manually install Django and other dependencies -3. `pip freeze > requirements.txt` -4. `django-admin startproject project .` -3. Update `project/settings.py` to configure `SECRET_KEY`, `DATABASE` and `STATIC_ROOT` entries -4. `./manage.py startapp welcome`, to create the welcome page's app - -From this initial state you can: -* create new Django apps -* remove the `welcome` app -* rename the Django project -* update settings to suit your needs -* install more Python libraries and add them to the `requirements.txt` file - - -## Special files in this repository - -Apart from the regular files created by Django (`project/*`, `welcome/*`, `manage.py`), this repository contains: - -``` -openshift/ - OpenShift-specific files -├── scripts - helper scripts -└── templates - application templates - -requirements.txt - list of dependencies -``` +eDivorce was developed by the British Columbia Ministry of Justice to help self represented litigants fill out the paperwork for their divorce. It replaces existing fillable PDF forms with a friendly web interface. +The steps in this document assume that you have access to an OpenShift deployment that you can deploy applications on. ## Local development +Prerequesites: +* Docker +* Python 3.5 + To run this project in your development machine, follow these steps: 1. (optional) Create and activate a [virtualenv](https://virtualenv.pypa.io/) (you may want to use [virtualenvwrapper](http://virtualenvwrapper.readthedocs.org/)). -2. Fork this repo and clone your fork: +2. Clone this repo: - `git clone https://github.com/openshift/django-ex.git` + `git clone https://github.com/bcgov/eDivorce.git` 3. Install dependencies: `pip install -r requirements.txt` -4. Create a development database: - - `./manage.py migrate` - -5. If everything is alright, you should be able to start the Django development server: - - `./manage.py runserver` - -6. Open your browser and go to http://127.0.0.1:8000, you will be greeted with a welcome page. - - -## Deploying to OpenShift - -To follow the next steps, you need to be logged in to an OpenShift cluster and have an OpenShift project where you can work on. - - -### Using an application template - -The directory `openshift/templates/` contains OpenShift application templates that you can add to your OpenShift project with: - - oc create -f openshift/templates/.json - -The template `django.json` contains just a minimal set of components to get your Django application into OpenShift. +4. Create an environment settings file by copying `.env.example` to `.env` (`.env` will be ignored by Git) -The template `django-postgresql.json` contains all of the components from `django.json`, plus a PostgreSQL database service and an Image Stream for the Python base image. For simplicity, the PostgreSQL database in this template uses ephemeral storage and, therefore, is not production ready. +5. Create a development database: -After adding your templates, you can go to your OpenShift web console, browse to your project and click the create button. Create a new app from one of the templates that you have just added. - -Adjust the parameter values to suit your configuration. Most times you can just accept the default values, however you will probably want to set the `GIT_REPOSITORY` parameter to point to your fork and the `DATABASE_*` parameters to match your database configuration. - -Alternatively, you can use the command line to create your new app, assuming your OpenShift deployment has the default set of ImageStreams defined. Instructions for installing the default ImageStreams are available [here](http://docs.openshift.org/latest/admin_guide/install/first_steps.html). If you are defining the set of ImageStreams now, remember to pass in the proper cluster-admin credentials and to create the ImageStreams in the 'openshift' namespace: - - oc new-app openshift/templates/django.json -p SOURCE_REPOSITORY_URL= - -Your application will be built and deployed automatically. If that doesn't happen, you can debug your build: - - oc get builds - # take build name from the command above - oc build-logs + `./manage.py migrate` -And you can see information about your deployment too: +6. If everything is alright, you should be able to start the Django development server: - oc describe dc/django-example + `./manage.py runserver 0.0.0.0:8000` -In the web console, the overview tab shows you a service, by default called "django-example", that encapsulates all pods running your Django application. You can access your application by browsing to the service's IP address and port. You can determine these by running +7. Start the [Weasyprint server](https://hub.docker.com/r/aquavitae/weasyprint/) server on port 5005 - oc get svc + 1. Bind the IP address 10.200.10.1 to the lo0 interface on your Mac computer. Weasyprint has been configured to use this IP address to request CSS files from Django *(You should only have to do this once)*. + ``` + sudo ifconfig lo0 alias 10.200.10.1/24 + ``` + 1. Start docker + ``` + docker run -d -p 5005:5001 aquavitae/weasyprint + ``` -### Without an application template -Templates give you full control of each component of your application. -Sometimes your application is simple enough and you don't want to bother with templates. In that case, you can let OpenShift inspect your source code and create the required components automatically for you: +8. Open your browser and go to http://127.0.0.1:8000, you will be greeted with the eDivorce homepage. -```bash -$ oc new-app centos/python-35-centos7~https://github.com/openshift/django-ex -imageStreams/python-35-centos7 -imageStreams/django-ex -buildConfigs/django-ex -deploymentConfigs/django-ex -services/django-ex -A build was created - you can run `oc start-build django-ex` to start it. -Service "django-ex" created at 172.30.16.213 with port mappings 8080. -``` -You can access your application by browsing to the service's IP address and port. +## OpenShift deployment +See: `openshift/README.md` ## Logs @@ -129,17 +65,6 @@ You can look at the combined stdout and stderr of a given pod with this command: This can be useful to observe the correct functioning of your application. -## Special environment variables - -### APP_CONFIG - -You can fine tune the gunicorn configuration through the environment variable `APP_CONFIG` that, when set, should point to a config file as documented [here](http://docs.gunicorn.org/en/latest/settings.html). - -### DJANGO_SECRET_KEY - -When using one of the templates provided in this repository, this environment variable has its value automatically generated. For security purposes, make sure to set this to a random string as documented [here](https://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-SECRET_KEY). - - ## One-off command execution At times you might want to manually execute some command in the context of a running application in OpenShift. @@ -147,27 +72,34 @@ You can drop into a Python shell for debugging, create a new user for the Django You can do all that by using regular CLI commands from OpenShift. To make it a little more convenient, you can use the script `openshift/scripts/run-in-container.sh` that wraps some calls to `oc`. -In the future, the `oc` CLI tool might incorporate changes -that make this script obsolete. +In the future, the `oc` CLI tool might incorporate changes that make this script obsolete. Here is how you would run a command in a pod specified by label: +1. Log in to the Openshift instance + + ``` + oc login --token= + ``` + +1. Select the project where you want to run the command + + ``` + oc project + ``` + 1. Inspect the output of the command below to find the name of a pod that matches a given label: oc get pods -l -2. Open a shell in the pod of your choice. Because of how the images produced +1. Open a shell in the pod of your choice. Because of how the images produced with CentOS and RHEL work currently, we need to wrap commands with `bash` to enable any Software Collections that may be used (done automatically inside every bash shell). oc exec -p -it -- bash -3. Finally, execute any command that you need and exit the shell. - -Related GitHub issues: -1. https://github.com/GoogleCloudPlatform/kubernetes/issues/8876 -2. https://github.com/openshift/origin/issues/2001 +1. Finally, execute any command that you need and exit the shell. The wrapper script combines the steps above into one. You can use it like this: @@ -192,20 +124,10 @@ Or both together: ## Data persistence -You can deploy this application without a configured database in your OpenShift project, in which case Django will use a temporary SQLite database that will live inside your application's container, and persist only until you redeploy your application. - -After each deploy you get a fresh, empty, SQLite database. That is fine for a first contact with OpenShift and perhaps Django, but sooner or later you will want to persist your data across deployments. - -To do that, you should add a properly configured database server or ask your OpenShift administrator to add one for you. Then use `oc env` to update the `DATABASE_*` environment variables in your DeploymentConfig to match your database settings. - -Redeploy your application to have your changes applied, and open the welcome page again to make sure your application is successfully connected to the database server. - - -## Looking for help - -If you get stuck at some point, or think that this document needs further details or clarification, you can give feedback and look for help using the channels mentioned in [the OpenShift Origin repo](https://github.com/openshift/origin), or by filing an issue. - +For local development a SQLite database will be used. For OpenShift deployments data will be stored in a PostgreSQL database, with data files residing on a persistent volume. ## License -This code is dedicated to the public domain to the maximum extent permitted by applicable law, pursuant to [CC0](http://creativecommons.org/publicdomain/zero/1.0/). +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT. +See the Apache Version 2.0 License for specific language governing permissions and limitations under the License. diff --git a/edivorce/apps/core/views/system.py b/edivorce/apps/core/views/system.py new file mode 100644 index 00000000..c3d2781d --- /dev/null +++ b/edivorce/apps/core/views/system.py @@ -0,0 +1,9 @@ +from django.http import HttpResponse +from edivorce.apps.core.models import Question + + +def health(request): + """ + OpenShift health check + """ + return HttpResponse(Question.objects.count()) diff --git a/edivorce/settings/base.py b/edivorce/settings/base.py index a49340de..0a141984 100644 --- a/edivorce/settings/base.py +++ b/edivorce/settings/base.py @@ -11,12 +11,11 @@ https://docs.djangoproject.com/en/1.8/ref/settings/ """ import os - from decouple import config from unipath import Path # Build paths inside the project like this: os.path.join(BASE_DIR, ...) - +PROJECT_ROOT = Path(__file__).parent.parent.parent BASE_DIR = Path(__file__).parent.parent # Quick-start development settings - unsuitable for production @@ -30,8 +29,9 @@ SECRET_KEY = config( DEBUG = config('DEBUG', default=False, cast=bool) -ALLOWED_HOSTS = ['localhost', '127.0.0.1'] - +# TODO: Set ALLOWED_HOSTS this to the actual host headers used by the local/dev/test/prod instances & health probes +# ALLOWED_HOSTS = ['localhost', '127.0.0.1'] +ALLOWED_HOSTS = ['*'] # Application definition @@ -42,8 +42,10 @@ INSTALLED_APPS = ( 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'rest_framework', 'debug_toolbar', 'edivorce.apps.core', + 'compressor', ) MIDDLEWARE_CLASSES = ( @@ -54,6 +56,7 @@ MIDDLEWARE_CLASSES = ( 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'edivorce.apps.core.middleware.bceid_middleware.BceidMiddleware', 'django.middleware.security.SecurityMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware', ) @@ -76,23 +79,7 @@ TEMPLATES = [ }, ] -WSGI_APPLICATION = 'edivorce.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/1.8/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': config('DATABASE_ENGINE'), - 'NAME': config('DATABASE_NAME'), - 'USER': config('DATABASE_USER'), - 'PASSWORD': config('DATABASE_PASSWORD'), - 'HOST': config('DATABASE_HOST'), - 'PORT': config('DATABASE_PORT'), - } -} - +WSGI_APPLICATION = 'wsgi.application' # Internationalization # https://docs.djangoproject.com/en/1.8/topics/i18n/ @@ -115,3 +102,18 @@ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles/') STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage' + +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + # other finders.. + 'compressor.finders.CompressorFinder', +) + +# django-libsass + +COMPRESS_PRECOMPILERS = ( + ('text/x-scss', 'django_libsass.SassCompiler'), +) + +FORCE_SCRIPT_NAME = '' diff --git a/edivorce/settings/local.py b/edivorce/settings/local.py index f7a77b2d..d74cb355 100644 --- a/edivorce/settings/local.py +++ b/edivorce/settings/local.py @@ -1 +1,16 @@ -from base import * +from .base import * +from decouple import config + +DATABASES = { + 'default': { + 'ENGINE': config('DATABASE_ENGINE'), + 'NAME': config('DATABASE_NAME'), + 'USER': config('DATABASE_USER'), + 'PASSWORD': config('DATABASE_PASSWORD'), + 'HOST': config('DATABASE_HOST'), + 'PORT': config('DATABASE_PORT'), + } +} + +WEASYPRINT_URL = 'http://localhost:5005' +WEASYPRINT_CSS_LOOPBACK = 'http://10.200.10.1:8000' diff --git a/edivorce/settings/openshift.py b/edivorce/settings/openshift.py new file mode 100644 index 00000000..70a59a50 --- /dev/null +++ b/edivorce/settings/openshift.py @@ -0,0 +1,71 @@ +from .base import * + +def openshift_db_config(): + ''' + Database config based on the django-ex openshift sample application + ''' + service_name = os.getenv('DATABASE_SERVICE_NAME', '').upper() + + engines = { + 'sqlite': 'django.db.backends.sqlite3', + 'postgresql': 'django.db.backends.postgresql_psycopg2', + 'mysql': 'django.db.backends.mysql', + } + + if service_name: + engine = engines.get(os.getenv('DATABASE_ENGINE'), engines['sqlite']) + else: + engine = engines['sqlite'] + name = os.getenv('DATABASE_NAME') + if not name and engine == engines['sqlite']: + name = os.path.join(PROJECT_ROOT, 'db.sqlite3') + + return { + 'ENGINE': engine, + 'NAME': name, + 'USER': os.getenv('DATABASE_USER'), + 'PASSWORD': os.getenv('DATABASE_PASSWORD'), + 'HOST': os.getenv('{}_SERVICE_HOST'.format(service_name)), + 'PORT': os.getenv('{}_SERVICE_PORT'.format(service_name)), + } + + +DATABASES = { + 'default': openshift_db_config() +} + +WEASYPRINT_URL = 'http://weasyprint:5001' +WEASYPRINT_CSS_LOOPBACK = 'http://edivorce-django:8080' + +# Django Compressor offline compression (triggered by setup.py during OpenShift build) +COMPRESS_ENABLED = True +COMPRESS_OFFLINE = True + +# The app will be served out of a subdirectory of justice.gov.bc.ca +# PROD: /divorce +# TEST: /divorce-test +# DEV: /divorce-dev +# +# See nginx-proxy/conf.d/server.conf for related settings +# +OPENSHIFT_ENVIRONMENT_TYPE = os.getenv('ENVIRONMENT_TYPE') + +PROXY_URL_PREFIX = '' + +if OPENSHIFT_ENVIRONMENT_TYPE == 'dev': + PROXY_URL_PREFIX = "/divorce-dev" + DEBUG = True + +if OPENSHIFT_ENVIRONMENT_TYPE == 'test': + PROXY_URL_PREFIX = "/divorce-test" + + +if OPENSHIFT_ENVIRONMENT_TYPE == 'prod': + PROXY_URL_PREFIX = "/divorce" + + +FORCE_SCRIPT_NAME = PROXY_URL_PREFIX + '/' +STATIC_URL = PROXY_URL_PREFIX + '/static/' +WEASYPRINT_CSS_LOOPBACK += PROXY_URL_PREFIX + + diff --git a/edivorce/settings/production.py b/edivorce/settings/production.py deleted file mode 100644 index 9f20de55..00000000 --- a/edivorce/settings/production.py +++ /dev/null @@ -1 +0,0 @@ -from settings.base import * diff --git a/edivorce/wsgi.py b/edivorce/wsgi.py deleted file mode 100644 index d6279d02..00000000 --- a/edivorce/wsgi.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -WSGI config for project project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/ -""" - -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "edivorce.settings.local") - -application = get_wsgi_application() diff --git a/manage.py b/manage.py index 85df8762..866389e0 100755 --- a/manage.py +++ b/manage.py @@ -1,9 +1,14 @@ #!/usr/bin/env python import os import sys +import decouple if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "edivorce.settings.local") + + if decouple.config("LOCAL_DEV", default=False, cast=bool): + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "edivorce.settings.local") + else: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "edivorce.settings.openshift") from django.core.management import execute_from_command_line diff --git a/requirements.txt b/requirements.txt index 4a00dff1..f13c98f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,12 @@ -Django==1.8.17 -Unipath==1.1 astroid==1.4.9 backports.functools-lru-cache==1.3 configparser==3.5.0 +Django==1.8.17 +django-appconf==1.0.2 +django-compressor==2.1 django-debug-toolbar==1.5 +django-libsass==0.7 +djangorestframework==3.5.3 enum34==1.1.6 flake8==3.2.1 futures==3.0.5 @@ -11,18 +14,25 @@ greenlet==0.4.11 gunicorn==19.4.5 isort==4.2.5 lazy-object-proxy==1.2.2 +libsass==0.12.3 mccabe==0.5.3 msgpack-python==0.4.8 neovim==0.1.12 +olefile==0.44 +Pillow==4.0.0 psycopg2==2.6.1 pycodestyle==2.2.0 pyflakes==1.3.0 pylint==1.6.4 python-decouple==3.0 +rcssmin==1.0.6 reportlab==3.3.0 +requests==2.13.0 +rjsmin==1.0.12 six==1.10.0 sqlparse==0.2.2 trollius==2.1 +Unipath==1.1 +wheel==0.24.0 whitenoise==3.0 wrapt==1.10.8 -wsgiref==0.1.2 diff --git a/setup.py b/setup.py new file mode 100755 index 00000000..eaa93b72 --- /dev/null +++ b/setup.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python + +# NOTE: this isn't a real setup.py file. It has been added here +# so additional steps could be injected into the OpenShift/Docker S2I build. +# +# If a file called "setup.py" exists, then the S2I assemble script will run it +# during the build. +# +# TODO: I have added Django-Compressor code to wsgi.py so it runs during deployment +# instead. I'm just keeping the code here until I am sure the other approach +# is working. This script currently does NOTHING + +import os + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "edivorce.settings.openshift") +from django.core.management import execute_from_command_line + +print('') +print('NOTICE: THIS SCRIPT DOES NOT SET UP THE PROJECT!!') +print('If you are trying to set up the project, please run the following command instead:') +print('$ pip install -r requirements.txt') +print('') +print('Executing additional OpenShift S2I assemble tasks defined in setup.py') +print('') + +# Run Django-Compressor offline compression +# print('Running Django Compressor offline compression') +# execute_from_command_line(['manage.py', 'compress']) diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 00000000..aa2126ad --- /dev/null +++ b/wsgi.py @@ -0,0 +1,21 @@ +""" +WSGI config for project project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/1.8/howto/deployment/wsgi/ +""" + +import os +import decouple +from django.core.wsgi import get_wsgi_application +from django.core.management import execute_from_command_line + +if decouple.config("LOCAL_DEV", default=False, cast=bool): + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "edivorce.settings.local") +else: + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "edivorce.settings.openshift") + execute_from_command_line(['manage.py', 'compress', '--force']) + +application = get_wsgi_application()