diff --git a/django/website/chn_spreadsheet/test_files/sample_excel.xlsx b/django/website/chn_spreadsheet/test_files/sample_excel.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..c1d7b5faee1fe3e5f8d44e4c3fad192fbb8a4689
Binary files /dev/null and b/django/website/chn_spreadsheet/test_files/sample_excel.xlsx differ
diff --git a/django/website/chn_spreadsheet/tests.py b/django/website/chn_spreadsheet/tests.py
index 7ce503c2dd97ba78597f6ff6e4393132753573f6..9da30add0754f7a5ef704c0c6bec6548a682fe03 100644
--- a/django/website/chn_spreadsheet/tests.py
+++ b/django/website/chn_spreadsheet/tests.py
@@ -1,3 +1,215 @@
-from django.test import TestCase
+import datetime
+import decimal
+from os import path
+import pytest
-# Create your tests here.
+from django.utils.translation import ugettext as _
+
+from .utils import (
+ get_profile, get_columns_map, order_columns, get_fields_and_types,
+ parse_date, normalize_row, get_rows_iterator, convert_row, process_rows,
+ SheetProfile, SheetImportException
+)
+
+
+TEST_BASE_DIR = path.abspath(path.dirname(__file__))
+TEST_DIR = path.join(TEST_BASE_DIR, 'test_files')
+
+
+COLUMN_LIST = [
+ {
+ 'name': 'Province',
+ 'type': 'location',
+ 'field': 'message.location',
+ },
+ {
+ 'name': 'Message',
+ 'type': 'text',
+ 'field': 'message.content',
+ },
+]
+
+
+@pytest.mark.django_db
+def test_get_profile_returns_profile():
+ label = "unknownpoll"
+ profile = {'name': 'Empty profile'}
+
+ SheetProfile.objects.create(label=label, profile=profile)
+
+ sprofile = get_profile(label)
+ assert sprofile == profile
+
+
+@pytest.mark.django_db
+def test_get_profile_raises_on_unknown_label():
+ with pytest.raises(SheetImportException) as excinfo:
+ get_profile('unknownlabel')
+ assert excinfo.value.message == _('Misconfigured service. Source "unknownlabel" does not exist')
+
+
+def test_get_columns_map():
+ expected_result = {
+ 'Province': {
+ 'type': 'location',
+ 'field': 'message.location'
+ },
+ 'Message': {
+ 'type': 'text',
+ 'field': 'message.content'
+ },
+ }
+ result = get_columns_map(COLUMN_LIST)
+ assert result == expected_result
+
+
+def test_get_rows_iterator_raises_on_non_excel_files():
+ with pytest.raises(SheetImportException) as excinfo:
+ get_rows_iterator('not_a_file', 'excel')
+ assert excinfo.value.message == _('Expected excel file. Received file in an unrecognized format.')
+
+ with pytest.raises(SheetImportException) as excinfo:
+ get_rows_iterator(None, 'pdf')
+ assert excinfo.value.message == _('Unsupported file format: pdf')
+
+
+def test_get_rows_iterator_works_on_excel_files():
+ file_path = path.join(TEST_DIR, 'sample_excel.xlsx')
+ f = open(file_path, 'rb')
+ rows = list(get_rows_iterator(f, 'excel'))
+
+ # 2x2 spreadsheet
+ assert len(rows) == 2
+ assert len(rows[0]) == 2
+ assert len(rows[1]) == 2
+
+
+def _make_columns_row(column_list):
+ row = [d.copy() for d in column_list]
+ for col in row:
+ del col['name'] # Unify with first row version
+ return row
+
+
+def test_order_columns_with_no_first_row_returns_original_order():
+ expected = _make_columns_row(COLUMN_LIST)
+ ordered = order_columns(COLUMN_LIST)
+ assert ordered == expected
+
+
+def test_order_columns_with_first_row_return_first_row_order():
+ cleaned = _make_columns_row(COLUMN_LIST)
+
+ first_row = ['Message', 'Province']
+ ordered = order_columns(COLUMN_LIST, first_row)
+ assert ordered == [cleaned[1], cleaned[0]]
+
+
+def test_get_fields_and_types():
+ fields, types = get_fields_and_types(COLUMN_LIST)
+ expected_types = ['location', 'text']
+ expected_fields = ['message.location', 'message.content']
+
+ assert fields == expected_fields
+ assert types == expected_types
+
+
+def test_successful_runs_of_parse_date():
+ dates = (
+ '05/01/2015',
+ '5.1.2015',
+ '5/1/15',
+ '05-01-2015',
+ datetime.datetime(2015, 1, 5, 0, 0)
+ )
+ expected = datetime.date(2015, 1, 5)
+ for date in dates:
+ assert parse_date(date) == expected
+
+
+def test_exception_raised_on_faulty_dates():
+ bad_date = '05x01-2015'
+ with pytest.raises(ValueError):
+ parse_date(bad_date)
+
+
+def test_convert_row():
+ row = ['Short message', '5', '10.4', '1.5.2015', 'Something else']
+ types = ('text', 'integer', 'number', 'date', 'ignore')
+
+ number = decimal.Decimal('10.4')
+ date = datetime.date(2015, 5, 1)
+
+ converted = convert_row(row, types, 4)
+ assert converted == ['Short message', 5, number, date]
+
+
+def test_convert_row_raises_on_unknown_type():
+ row = ['Short message']
+ types = ['location']
+
+ with pytest.raises(SheetImportException) as excinfo:
+ convert_row(row, types, 5)
+ assert excinfo.value.message == _(u"Unknown data type 'location' on row 5 ")
+
+
+def test_convert_row_raises_on_malformed_value():
+ row = ['not_integer']
+ types = ['integer']
+
+ with pytest.raises(SheetImportException) as excinfo:
+ convert_row(row, types, 3)
+ assert excinfo.value.message == _(u"Can not process value 'not_integer' of type 'integer' on row 3 ")
+
+
+def test_normalize_row_differences():
+ class Cell(object):
+ def __init__(self, value):
+ self.value = value
+
+ row = [5, 'London', Cell('1.1.2015')]
+ result = normalize_row(row)
+ assert result == [5, 'London', '1.1.2015']
+
+
+def test_normalize_row_works_with_none():
+ assert normalize_row(None) is None
+
+
+def __test_process_rows_without_or_with_header(with_header):
+ def _rows_generator():
+ rows = [
+ ('Province', 'Message'),
+ ('London', 'Short message'),
+ ('Cambridge', 'What?'),
+ ]
+ if not with_header:
+ rows = rows[1:]
+ for row in rows:
+ yield row
+
+ columns = [d.copy() for d in COLUMN_LIST]
+ columns[0]['type'] = 'text'
+ rows = _rows_generator()
+
+ objects = process_rows(rows, columns, with_header)
+ expected_objects = [
+ {
+ 'message.location': 'London',
+ 'message.content': 'Short message'
+ },
+ {
+ 'message.location': 'Cambridge',
+ 'message.content': 'What?'
+ },
+ ]
+
+ assert objects == expected_objects
+
+
+def test_process_rows_without_header():
+ __test_process_rows_without_or_with_header(False)
+
+
+def test_process_rows_with_header():
+ __test_process_rows_without_or_with_header(True)
diff --git a/django/website/chn_spreadsheet/utils.py b/django/website/chn_spreadsheet/utils.py
index 1e137406f3c94d79ec9c67c05dc8caff82419c81..0b716679f6423e4abcabf09106baba35584afea1 100644
--- a/django/website/chn_spreadsheet/utils.py
+++ b/django/website/chn_spreadsheet/utils.py
@@ -60,7 +60,9 @@ def order_columns(profile_columns, first_row=None):
error_msg = _('Unknown column: %s') % label
raise SheetImportException(error_msg)
else:
- columns = profile_columns[:]
+ columns = [d.copy() for d in profile_columns]
+ for col in columns:
+ del col['name'] # Unify with first row version
return columns
@@ -72,7 +74,12 @@ def get_fields_and_types(columns):
def parse_date(value):
- return dateutil.parser.parse(value, dayfirst=True).date()
+ if isinstance(value, basestring):
+ date_time = dateutil.parser.parse(value, dayfirst=True)
+ else:
+ date_time = value
+
+ return date_time.date()
def convert_row(orig_values, types, row_number):
@@ -102,21 +109,20 @@ def convert_row(orig_values, types, row_number):
def normalize_row(raw_row):
# Unify difference between CSV and openpyxl cells
- row = []
- for val in raw_row:
- value = val.value if hasattr(val, "value") else val
- row.append(value)
- return row
+ if raw_row:
+ row = []
+ for val in raw_row:
+ value = val.value if hasattr(val, "value") else val
+ row.append(value)
+ return row
+ return None
-def process_rows(spreadsheet, profile):
- file_format = profile.get('format')
- rows = get_rows_iterator(spreadsheet, file_format)
-
- # If skip_header, then use profile's order of columns, otherwise
- # use header line to check mapping and define order
- first_row = rows.next() if profile['skip_header'] else None
- columns = order_columns(profile['columns'], normalize_row(first_row))
+def process_rows(rows, profile_columns, skip_header=False):
+ # If there is no header (skip_header=False), then use profile's order of
+ # columns, otherwise use header line to check mapping and define order
+ first_row = rows.next() if skip_header else None
+ columns = order_columns(profile_columns, normalize_row(first_row))
fields, types = get_fields_and_types(columns)
@@ -138,5 +144,10 @@ def save_rows(objects, data_type):
def store_spreadsheet(label, fobject):
profile = get_profile(label)
- items = process_rows(fobject, profile)
+
+ file_format = profile.get('format')
+ skip_header = profile.get('skip_header', False)
+
+ rows = get_rows_iterator(fobject, file_format)
+ items = process_rows(rows, profile['columns'], skip_header)
return save_rows(items, 'message')
diff --git a/django/website/hid/templates/hid/sources.html b/django/website/hid/templates/hid/sources.html
index 3ed1ea58916558d70efc2b6d8572350ff63deb18..6944649b901aa6447f1c929d14cd2428c1e899bd 100644
--- a/django/website/hid/templates/hid/sources.html
+++ b/django/website/hid/templates/hid/sources.html
@@ -13,7 +13,7 @@
<form action="{% url "sources-upload" %}" method="post" enctype="multipart/form-data" class="item-source-actions pull-right">
{% csrf_token %}
- <a class="btn btn-primary btn-block" value="View/Edit data" type="button" href="{% url "sources-edit" source.src %}">View/Edit data</a>
+ <a class="btn btn-primary btn-block" value="View/Edit data" type="button" href="{% url "data-view" %}">View/Edit data</a>
{% bootstrap_form source.form show_label=False %}
{% bootstrap_button "Upload" button_type="submit" value="Upload" button_class="item-source-upload btn-block btn-primary" %}
</form>
diff --git a/django/website/hid/templates/hid/view.html b/django/website/hid/templates/hid/view.html
new file mode 100644
index 0000000000000000000000000000000000000000..ac20d756409996bd6d59500b619d59ac8591bc4b
--- /dev/null
+++ b/django/website/hid/templates/hid/view.html
@@ -0,0 +1,8 @@
+{% extends "base.html" %}
+{% load i18n %}
+
+{% block maincontent %}
+<h2>View data</h2>
+<p>{% trans "This is the view data page." %}</p>
+
+{% endblock maincontent %}
diff --git a/django/website/hid/urls.py b/django/website/hid/urls.py
index eb4ab3515ec3714e48c17ef3db837b525c5f4573..34b0e36002c8521b45de022bffcdf120a8e5b5a4 100644
--- a/django/website/hid/urls.py
+++ b/django/website/hid/urls.py
@@ -10,5 +10,6 @@ urlpatterns = patterns('',
url(r'^sources/upload/$', login_required(UploadSpreadsheetView.as_view()), name='sources-upload'),
url(r'^sources/(?P<label>\w+)/$', login_required(ListSources.as_view()), name='sources-edit'),
url(r'^sources/$', login_required(ListSources.as_view()), name='sources'),
+ url(r'^view/$', login_required(TemplateView.as_view(template_name='hid/view.html')), name="data-view"),
url(r'^$', login_required(TemplateView.as_view(template_name='hid/dashboard.html')), name="dashboard"),
)
diff --git a/django/website/hid/views.py b/django/website/hid/views.py
index 83aef1720ef9582cb0b74df40ae20b41b476987e..275f5e6bf5c811be350d5debf2e974aa5ab44cb3 100644
--- a/django/website/hid/views.py
+++ b/django/website/hid/views.py
@@ -33,7 +33,7 @@ class UploadSpreadsheetView(FormView):
template_name = 'hid/upload.html'
def get_success_url(self):
- return reverse("sources")
+ return reverse("data-view")
def form_valid(self, form):
data = form.cleaned_data
diff --git a/django/website/media/images/favicon.ico b/django/website/media/images/favicon.ico
index 61b6eb304e5f45d11289fe4fbc5c78daaac886a9..6945e4856b67dc0f20cf114df510c7cca6d1dafa 100644
Binary files a/django/website/media/images/favicon.ico and b/django/website/media/images/favicon.ico differ
diff --git a/django/website/templates/base.html b/django/website/templates/base.html
index ac298b9a4ee9f70411ad57b76ce6127ed2c34894..08661ec7a1ef4a195019c3f5b0866eb852136728 100644
--- a/django/website/templates/base.html
+++ b/django/website/templates/base.html
@@ -17,10 +17,11 @@
<![endif]-->
{% block bootstrap3_extra_head %}{% endblock %}
<link href="{{ STATIC_URL }}css/styles.css" media="all" rel="stylesheet" />
+ <link rel="shortcut icon" href="{{STATIC_URL}}images/favicon.ico?v=1"/>
</head>
<body>
-
+
{% block content %}
<meta name="viewport" content="width=device-width,initial-scale=1">
<div class="container-fluid">
@@ -51,7 +52,7 @@
</div>
</div>
{% endblock content %}
-
+
{% block bootstrap3_extra_js %}
{% bootstrap_javascript jquery=True %}
{% endblock %}