Merge branch 'excel_import' into datalayer

django/website/chn_spreadsheet/tests.py

-from django.test import TestCase
+import datetime
+import decimal
+from os import path
+import pytest
 
-# Create your tests here.
+from django.utils.translation import ugettext as _
+
+from .utils import (
+    get_profile, get_columns_map, order_columns, get_fields_and_types,
+    parse_date, normalize_row, get_rows_iterator, convert_row, process_rows,
+    SheetProfile, SheetImportException
+)
+
+
+TEST_BASE_DIR = path.abspath(path.dirname(__file__))
+TEST_DIR = path.join(TEST_BASE_DIR, 'test_files')
+
+
+COLUMN_LIST = [
+    {
+        'name': 'Province',
+        'type': 'location',
+        'field': 'message.location',
+    },
+    {
+        'name': 'Message',
+        'type': 'text',
+        'field': 'message.content',
+    },
+]
+
+
+@pytest.mark.django_db
+def test_get_profile_returns_profile():
+    label = "unknownpoll"
+    profile = {'name': 'Empty profile'}
+
+    SheetProfile.objects.create(label=label, profile=profile)
+
+    sprofile = get_profile(label)
+    assert sprofile == profile
+
+
+@pytest.mark.django_db
+def test_get_profile_raises_on_unknown_label():
+    with pytest.raises(SheetImportException) as excinfo:
+        get_profile('unknownlabel')
+    assert excinfo.value.message == _('Misconfigured service. Source "unknownlabel" does not exist')
+
+
+def test_get_columns_map():
+    expected_result = {
+        'Province': {
+            'type': 'location',
+            'field': 'message.location'
+        },
+        'Message': {
+            'type': 'text',
+            'field': 'message.content'
+        },
+    }
+    result = get_columns_map(COLUMN_LIST)
+    assert result == expected_result
+
+
+def test_get_rows_iterator_raises_on_non_excel_files():
+    with pytest.raises(SheetImportException) as excinfo:
+        get_rows_iterator('not_a_file', 'excel')
+    assert excinfo.value.message == _('Expected excel file. Received file in an unrecognized format.')
+
+    with pytest.raises(SheetImportException) as excinfo:
+        get_rows_iterator(None, 'pdf')
+    assert excinfo.value.message == _('Unsupported file format: pdf')
+
+
+def test_get_rows_iterator_works_on_excel_files():
+    file_path = path.join(TEST_DIR, 'sample_excel.xlsx')
+    f = open(file_path, 'rb')
+    rows = list(get_rows_iterator(f, 'excel'))
+
+    # 2x2 spreadsheet
+    assert len(rows) == 2
+    assert len(rows[0]) == 2
+    assert len(rows[1]) == 2
+
+
+def _make_columns_row(column_list):
+    row = [d.copy() for d in column_list]
+    for col in row:
+        del col['name']  # Unify with first row version
+    return row
+
+
+def test_order_columns_with_no_first_row_returns_original_order():
+    expected = _make_columns_row(COLUMN_LIST)
+    ordered = order_columns(COLUMN_LIST)
+    assert ordered == expected
+
+
+def test_order_columns_with_first_row_return_first_row_order():
+    cleaned = _make_columns_row(COLUMN_LIST)
+
+    first_row = ['Message', 'Province']
+    ordered = order_columns(COLUMN_LIST, first_row)
+    assert ordered == [cleaned[1], cleaned[0]]
+
+
+def test_get_fields_and_types():
+    fields, types = get_fields_and_types(COLUMN_LIST)
+    expected_types = ['location', 'text']
+    expected_fields = ['message.location', 'message.content']
+
+    assert fields == expected_fields
+    assert types == expected_types
+
+
+def test_successful_runs_of_parse_date():
+    dates = (
+        '05/01/2015',
+        '5.1.2015',
+        '5/1/15',
+        '05-01-2015',
+        datetime.datetime(2015, 1, 5, 0, 0)
+    )
+    expected = datetime.date(2015, 1, 5)
+    for date in dates:
+        assert parse_date(date) == expected
+
+
+def test_exception_raised_on_faulty_dates():
+    bad_date = '05x01-2015'
+    with pytest.raises(ValueError):
+        parse_date(bad_date)
+
+
+def test_convert_row():
+    row = ['Short message', '5', '10.4', '1.5.2015', 'Something else']
+    types = ('text', 'integer', 'number', 'date', 'ignore')
+
+    number = decimal.Decimal('10.4')
+    date = datetime.date(2015, 5, 1)
+
+    converted = convert_row(row, types, 4)
+    assert converted == ['Short message', 5, number, date]
+
+
+def test_convert_row_raises_on_unknown_type():
+    row = ['Short message']
+    types = ['location']
+
+    with pytest.raises(SheetImportException) as excinfo:
+        convert_row(row, types, 5)
+    assert excinfo.value.message == _(u"Unknown data type 'location' on row 5 ")
+
+
+def test_convert_row_raises_on_malformed_value():
+    row = ['not_integer']
+    types = ['integer']
+
+    with pytest.raises(SheetImportException) as excinfo:
+        convert_row(row, types, 3)
+    assert excinfo.value.message == _(u"Can not process value 'not_integer' of type 'integer' on row 3 ")
+
+
+def test_normalize_row_differences():
+    class Cell(object):
+        def __init__(self, value):
+            self.value = value
+
+    row = [5, 'London', Cell('1.1.2015')]
+    result = normalize_row(row)
+    assert result == [5, 'London', '1.1.2015']
+
+
+def test_normalize_row_works_with_none():
+    assert normalize_row(None) is None
+
+
+def __test_process_rows_without_or_with_header(with_header):
+    def _rows_generator():
+        rows = [
+            ('Province', 'Message'),
+            ('London', 'Short message'),
+            ('Cambridge', 'What?'),
+        ]
+        if not with_header:
+            rows = rows[1:]
+        for row in rows:
+            yield row
+
+    columns = [d.copy() for d in COLUMN_LIST]
+    columns[0]['type'] = 'text'
+    rows = _rows_generator()
+
+    objects = process_rows(rows, columns, with_header)
+    expected_objects = [
+        {
+            'message.location': 'London',
+            'message.content': 'Short message'
+        },
+        {
+            'message.location': 'Cambridge',
+            'message.content': 'What?'
+        },
+    ]
+
+    assert objects == expected_objects
+
+
+def test_process_rows_without_header():
+    __test_process_rows_without_or_with_header(False)
+
+
+def test_process_rows_with_header():
+    __test_process_rows_without_or_with_header(True)

django/website/chn_spreadsheet/utils.py

@@ -60,7 +60,9 @@ def order_columns(profile_columns, first_row=None):
                 error_msg = _('Unknown column: %s') % label
                 raise SheetImportException(error_msg)
     else:
-        columns = profile_columns[:]
+        columns = [d.copy() for d in profile_columns]
+        for col in columns:
+            del col['name']  # Unify with first row version
 
     return columns
 
@@ -72,7 +74,12 @@ def get_fields_and_types(columns):
 
 
 def parse_date(value):
-    return dateutil.parser.parse(value, dayfirst=True).date()
+    if isinstance(value, basestring):
+        date_time = dateutil.parser.parse(value, dayfirst=True)
+    else:
+        date_time = value
+
+    return date_time.date()
 
 
 def convert_row(orig_values, types, row_number):
@@ -102,21 +109,20 @@ def convert_row(orig_values, types, row_number):
 
 def normalize_row(raw_row):
     # Unify difference between CSV and openpyxl cells
-    row = []
-    for val in raw_row:
-        value = val.value if hasattr(val, "value") else val
-        row.append(value)
-    return row
+    if raw_row:
+        row = []
+        for val in raw_row:
+            value = val.value if hasattr(val, "value") else val
+            row.append(value)
+        return row
+    return None
 
 
-def process_rows(spreadsheet, profile):
-    file_format = profile.get('format')
-    rows = get_rows_iterator(spreadsheet, file_format)
-
-    # If skip_header, then use profile's order of columns, otherwise
-    # use header line to check mapping and define order
-    first_row = rows.next() if profile['skip_header'] else None
-    columns = order_columns(profile['columns'], normalize_row(first_row))
+def process_rows(rows, profile_columns, skip_header=False):
+    # If there is no header (skip_header=False), then use profile's order of
+    # columns, otherwise use header line to check mapping and define order
+    first_row = rows.next() if skip_header else None
+    columns = order_columns(profile_columns, normalize_row(first_row))
 
     fields, types = get_fields_and_types(columns)
 
@@ -138,5 +144,10 @@ def save_rows(objects, data_type):
 
 def store_spreadsheet(label, fobject):
     profile = get_profile(label)
-    items = process_rows(fobject, profile)
+
+    file_format = profile.get('format')
+    skip_header = profile.get('skip_header', False)
+
+    rows = get_rows_iterator(fobject, file_format)
+    items = process_rows(rows, profile['columns'], skip_header)
     return save_rows(items, 'message')

django/website/hid/templates/hid/sources.html

@@ -13,7 +13,7 @@
 
             <form action="{% url "sources-upload" %}" method="post" enctype="multipart/form-data" class="item-source-actions pull-right">
                 {% csrf_token %}
-                <a class="btn btn-primary btn-block" value="View/Edit data" type="button" href="{% url "sources-edit" source.src %}">View/Edit data</a>
+                <a class="btn btn-primary btn-block" value="View/Edit data" type="button" href="{% url "data-view" %}">View/Edit data</a>
                 {% bootstrap_form source.form show_label=False %}
                 {% bootstrap_button "Upload" button_type="submit" value="Upload" button_class="item-source-upload btn-block btn-primary" %}
             </form>

django/website/hid/templates/hid/view.html

+{% extends "base.html" %}
+{% load i18n %}
+
+{% block maincontent %}
+<h2>View data</h2>
+<p>{% trans "This is the view data page." %}</p>
+
+{% endblock maincontent %}

django/website/hid/urls.py

@@ -10,5 +10,6 @@ urlpatterns = patterns('',
     url(r'^sources/upload/$', login_required(UploadSpreadsheetView.as_view()), name='sources-upload'),
     url(r'^sources/(?P<label>\w+)/$', login_required(ListSources.as_view()), name='sources-edit'),
     url(r'^sources/$', login_required(ListSources.as_view()), name='sources'),
+    url(r'^view/$', login_required(TemplateView.as_view(template_name='hid/view.html')), name="data-view"),
     url(r'^$', login_required(TemplateView.as_view(template_name='hid/dashboard.html')), name="dashboard"),
 )

django/website/hid/views.py

@@ -33,7 +33,7 @@ class UploadSpreadsheetView(FormView):
     template_name = 'hid/upload.html'
 
     def get_success_url(self):
-        return reverse("sources")
+        return reverse("data-view")
 
     def form_valid(self, form):
         data = form.cleaned_data

django/website/templates/base.html

@@ -17,10 +17,11 @@
   <![endif]-->
   {% block bootstrap3_extra_head %}{% endblock %}
   <link href="{{ STATIC_URL }}css/styles.css" media="all" rel="stylesheet" />
+  <link rel="shortcut icon" href="{{STATIC_URL}}images/favicon.ico?v=1"/>
 </head>
 
 <body>
-    
+
     {% block content %}
     <meta name="viewport" content="width=device-width,initial-scale=1">
     <div class="container-fluid">
@@ -51,7 +52,7 @@
         </div>
     </div>
     {% endblock content %}
-    
+
     {% block bootstrap3_extra_js %}
 {% bootstrap_javascript jquery=True %}
     {% endblock %}