Merge branch 'django22-spike' into 'staging'

Django 2.2 and Python 3 upgrade

See merge request !157

.gitlab-ci.yml

 ---
 
-image: python:2.7
+image: python:3.6
 
 services:
   - mysql:5.7

Pipfile

@@ -4,23 +4,22 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [requires]
-python_version = "2.7"
+python_version = "3.6"
 
 [packages]
 "django-bootstrap3" = "*"
-"django-tables2" = "==2.0.6"
+"django-tables2" = "*"
 "django.js" = {editable = true,git = "https://git@github.com/aptivate/django.js.git",ref = "ca328a94b00023bd64f4fc1c908675edaaf2ac19"}
 "linecache2" = "*"
-Django = ">1.11.22,<2.0"
+Django = ">2.2.8,<3.0"
 Pillow = "*"
 cssmin = "*"
 django-assets = "*"
 django-braces = "*"
 django-constance = {extras = ["database"],version = "*"}
-django-filter = "==1.1.0"
+django-filter = "*"
 django-floppyforms = "*"
 django-form-utils = "*"
-django-spreadsheetresponsemixin = "*"
 django-widget-tweaks = "*"
 djangorestframework = "*"
 djangorestframework-bulk = "*"
@@ -30,7 +29,7 @@ openpyxl = "*"
 python-dateutil = "*"
 pytz = "*"
 rest-pandas = "*"
-pymysql = "*"
+mysqlclient = "*"
 django-debug-toolbar-template-timings = "*"
 
 [dev-packages]
@@ -41,11 +40,11 @@ isort = "*"
 mock = "*"
 py = "*"
 pylava = "*"
-pyparsing = "==2.4.1.1"
+pyparsing = "*"
 pytest-cov = "*"
 pytest-django = "*"
 pytest-env = "*"
 pytest-pythonpath = "*"
 werkzeug = "*"
-pydocstyle = "==3.0.0"
+pydocstyle = "*"
 django-dynamic-fixture = "*"

internewshid/chn_spreadsheet/importer.py

 import datetime
-import sys
 from decimal import Decimal
 
 from django.utils import six
@@ -91,7 +90,7 @@ class Importer(object):
 
         # If there is no header (skip_header=False), then use profile's order of
         # columns, otherwise use header line to check mapping and define order
-        first_row = self.normalize_row(rows.next()) if skip_header else None
+        first_row = self.normalize_row(next(rows)) if skip_header else None
         columns = self.order_columns(first_row)
 
         objects = []
@@ -102,13 +101,13 @@ class Importer(object):
                 if any(values):
                     item = self.process_row(values, columns)
                     item['_row_number'] = i
-                    for taxonomy, term in meta_data.iteritems():
+                    for taxonomy, term in meta_data.items():
                         self._append_term_to_item(item, taxonomy, term)
 
                     objects.append(item)
 
             except SheetImportException as e:
-                raise type(e), type(e)(str(e) + 'in row {0} '.format(i)), sys.exc_info()[2]
+                raise type(e)(str(e) + 'in row {0} '.format(i)) from e
 
         return objects
 
@@ -160,18 +159,40 @@ class Importer(object):
         return num_saved
 
     def _get_spreadsheet_error_message(self, row, exc_inst):
-        # TODO: A TransportException during transport.items.add_terms()
-        # (eg "Term matching query does not exist") will not
-        # be handled correctly here as 'item' won't exist
-
-        status_code = exc_inst.message.pop('status_code')
-        item = exc_inst.message.pop('item') if 'item' in exc_inst.message else {}
+        # The exception will have different formats:
+        # Example of failure during transport.items.create():
+        # {
+        #  'enumerator': [ErrorDetail(string='Ensure this field has no more than 190 characters.', code='max_length')],
+        #  'status_code': 400,
+        #  'item': {
+        #      'timestamp': datetime.datetime(2018, 8, 9, 12, 14, 26, 766000, tzinfo=tzoffset(None, 21600)),
+        #      'body': 'the community members want more food.',
+        #      'gender': 'female',
+        #      'location': 'Camp 4',
+        #      'enumerator': '<a very long string>',
+        #      'external_id': '97f61035-6feb-40a1-9e7e-15c0f65cfdb5'
+        #   }
+        # }
+
+        # Example of failure during transport.items.add_terms():
+        # {
+        #  'detail': 'Term matching query does not exist.',
+        #  'status_code': 400,
+        #  'terms': {'taxonomy': 'age-ranges', 'name': ''},
+        #  'item_id': 4
+        # }
+
+        exc_inst.message.pop('status_code', None)
+        exc_inst.message.pop('item_id', None)
+        item = exc_inst.message.pop('item', {})
+        detail = exc_inst.message.pop('detail', None)
+        terms = exc_inst.message.pop('terms', None)
 
         messages = []
 
         field_to_column_map = self._get_field_to_column_map()
 
-        for field, errors in exc_inst.message.iteritems():
+        for field, errors in exc_inst.message.items():
             for error in errors:
                 messages.append(
                     _("Column: '{0}' ({1})\nError ({2}): '{3}'\n\nValue: {4}").format(
@@ -183,6 +204,14 @@ class Importer(object):
                     )
                 )
 
+        if detail and terms:
+            taxonomy = terms['taxonomy']
+            name = terms['name']
+
+            messages.append(
+                _(f"Error: {detail}\nTaxonomy: {taxonomy}\nName: {name}\n")
+            )
+
         return _("There was a problem with row {0} of the spreadsheet:\n{1}").format(
             row, '\n'.join(messages)
         )
@@ -232,13 +261,13 @@ class CellConverter(object):
             return converters[self.type](self.value)
         except Exception as e:
             message = _("{0}: Can not process value '{1}' of type '{2}' ").format(str(e), self.value, self.type)
-            raise SheetImportException(message), None, sys.exc_info()[2]
+            raise SheetImportException(message) from e
 
     def convert_date(self):
         if self.value is None:
             return None
 
-        if isinstance(self.value, basestring):
+        if isinstance(self.value, str):
             date_time = self.parse_date()
         else:
             date_time = self.value

internewshid/chn_spreadsheet/migrations/0001_initial.py

 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import models, migrations
 import jsonfield.fields
 

internewshid/chn_spreadsheet/migrations/0002_add_geopoll_config.py

 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0003_update_geopoll_config.py

 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0004_add_rapidpro_config.py

 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0005_update_geopoll_config.py

 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0006_update_rapidpro_config.py

 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0007_update_rapidpro_config.py

 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0008_update_geopoll_config.py

 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0009_update_rapidpro_config.py

 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0010_update_rapidpro_config.py

 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0011_update_geopoll_config.py

 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0012_auto_20181008_1321.py

 # -*- coding:  utf-8 -*-
-# Generated by Django 1.11.15 on 2018-10-08 12: 21
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0013_auto_20181019_1607.py

 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.15 on 2018-10-19 15:07
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/migrations/0014_auto_20191216_1054.py

 # -*- coding: utf-8 -*-
-# Generated by Django 1.11.23 on 2019-12-16 04:54
-from __future__ import unicode_literals
-
 from django.db import migrations
 
 

internewshid/chn_spreadsheet/models.py

@@ -19,5 +19,5 @@ class SheetProfile(models.Model):
     profile = JSONField(
         load_kwargs={'object_pairs_hook': collections.OrderedDict})
 
-    def __unicode__(self):
+    def __str__(self):
         return self.label

internewshid/chn_spreadsheet/tests/etc_tests.py

@@ -14,7 +14,7 @@ TEST_DIR = path.join(TEST_BASE_DIR, 'test_files')
 @pytest.fixture
 def django_db_setup(django_db_setup, django_db_blocker):
     with django_db_blocker.unblock():
-        call_command('loaddata', 'bangladesh-refugee-crisis.json')
+        call_command('loaddata', 'spreadsheet-profiles.json')
 
 
 @pytest.mark.django_db  # noqa