Merge branch 'staging' into id-removal

docs/architecture.md

@@ -95,7 +95,7 @@ A tabbed page has a `name` and one or more tab instances. These determine the pa
 The `Settings` are specified in a blob of JSON with the following format:
 ```
 {
-  "source":"kobo",
+  "collection_type":"kobo",
   "columns":[
     "select_item",
     "created",
@@ -115,7 +115,7 @@ The `Settings` are specified in a blob of JSON with the following format:
 }
 ```
 
-`source`
+`collection_type`
 : Matches the `SheetProfile` `label` above
 
 `columns`

internewshid/chn_spreadsheet/example_spreadsheet_channel_config.json

 {
     "name": "spreadsheet",
-    "sources": [
+    "collection_types": [
         {
             "label": "geopoll",
             "name": "Geopoll",

internewshid/chn_spreadsheet/migrations/0014_auto_20191216_1054.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-12-16 04:54
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def migrate_source_columns_to_collection_type(apps, schema_editor):
+    SheetProfile = apps.get_model('chn_spreadsheet', 'SheetProfile')
+    for sheet_profile in SheetProfile.objects.all():
+        migrated_columns = []
+
+        for column in sheet_profile.profile['columns']:
+            if 'field' in column and column['field'] == 'source':
+                column['field'] = 'collection_type'
+            if 'name' in column and column['name'] == 'Source':
+                column['name'] = 'Collection Type'
+            migrated_columns.append(column)
+
+        sheet_profile.profile['columns'] = migrated_columns
+        sheet_profile.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('chn_spreadsheet', '0013_auto_20181019_1607'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_source_columns_to_collection_type),
+    ]

internewshid/chn_spreadsheet/tests/kobo_master_tests.py

@@ -37,7 +37,7 @@ def test_kobo_master_items_imported(importer, django_db_setup):
     assert items[1]['age'] == '41'
     assert items[1]['location'] == 'Camp 1W'
     assert items[1]['enumerator'] == 'osman'
-    assert items[1]['source'] == 'sample source'
+    assert items[1]['collection_type'] == 'sample collection type'
     assert isinstance(items[1]['timestamp'], datetime.datetime)
 
     tags = []

internewshid/data_layer/migrations/0020_message_collection_type.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-12-16 04:03
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_layer', '0019_message_sub_location'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='message',
+            name='collection_type',
+            field=models.CharField(blank=True, max_length=190),
+        ),
+    ]

internewshid/data_layer/migrations/0021_auto_20191216_1007.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-12-16 04:07
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def migrate_source_to_collection_type(apps, schema_editor):
+    Message = apps.get_model('data_layer', 'Message')
+    for message in Message.objects.all():
+        if hasattr(message, 'source'):
+            message.collection_type = message.source
+            message.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_layer', '0020_message_collection_type'),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_source_to_collection_type),
+    ]

internewshid/data_layer/migrations/0022_remove_message_source.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-12-16 04:10
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_layer', '0021_auto_20191216_1007'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='message',
+            name='source',
+        ),
+    ]

internewshid/data_layer/models.py

@@ -47,7 +47,7 @@ class Message(DataLayerModel):
     gender = models.CharField(max_length=100, blank=True)
     age = models.CharField(max_length=100, blank=True)
     enumerator = models.CharField(max_length=190, blank=True)
-    source = models.CharField(max_length=190, blank=True)
+    collection_type = models.CharField(max_length=190, blank=True)
     external_id = models.CharField(max_length=80, blank=True)
 
     def apply_terms(self, terms):

internewshid/data_layer/tests/factories.py

@@ -18,7 +18,7 @@ class ItemFactory(DjangoModelFactory):
     age = fuzzy.FuzzyText()
     gender = fuzzy.FuzzyText()
     enumerator = fuzzy.FuzzyText()
-    source = fuzzy.FuzzyText()
+    collection_type = fuzzy.FuzzyText()
     location = fuzzy.FuzzyText()
     timestamp = fuzzy.FuzzyDateTime(
         timezone.now() + timedelta(days=-365)

internewshid/hid/apps.py

@@ -10,6 +10,7 @@ class HidAppConfig(AppConfig):
         from hid.filters import (
             AgeRangeFilter,
             CategoryFilter,
+            CollectionTypeFilter,
             EnumeratorFilter,
             ExternalIdFilter,
             FeedbackTypeFilter,
@@ -17,7 +18,6 @@ class HidAppConfig(AppConfig):
             LocationFilter,
             SearchFilter,
             SubLocationFilter,
-            SourceFilter,
             TagsFilter,
             TimeRangeFilter,
         )
@@ -34,7 +34,7 @@ class HidAppConfig(AppConfig):
         register_filter('gender', GenderFilter())
         register_filter('age_range', AgeRangeFilter())
         register_filter('enumerator', EnumeratorFilter())
-        register_filter('source', SourceFilter())
+        register_filter('collection_type', CollectionTypeFilter())
         register_filter('tags', TagsFilter())
         register_filter('feedback_type', FeedbackTypeFilter())
         register_filter('external_id', ExternalIdFilter())

internewshid/hid/filters.py

@@ -82,11 +82,11 @@ class EnumeratorFilter(object):
             filters.update(enumerator=enumerator)
 
 
-class SourceFilter(object):
+class CollectionTypeFilter(object):
     def apply(self, filters, query_dict, **kwargs):
-        source = query_dict.get('source', None)
-        if source is not None:
-            filters.update(source=source)
+        collection_type = query_dict.get('collection_type', None)
+        if collection_type is not None:
+            filters.update(collection_type=collection_type)
 
 
 class FeedbackTypeFilter(object):

internewshid/hid/fixtures/bangladesh-refugee-crisis.json

@@ -296,7 +296,7 @@
       "model": "chn_spreadsheet.sheetprofile",
       "pk": 4
     },
-    {"model": "chn_spreadsheet.sheetprofile", "pk": 5, "fields": {"label": "kobo_master", "profile": "{\"name\":\"(Redundant)\",\"format\":\"excel\",\"label\":\"kobo_master\",\"skip_header\":1,\"taxonomies\":{},\"columns\":[{\"field\":\"timestamp\",\"type\":\"date\",\"name\":\"Date\",\"date_format\":\"%Y-%m-%dT%H: %M: %S.%f%z\"},{\"field\":\"body\",\"type\":\"text\",\"name\":\"Feedback\"},{\"field\":\"translation\",\"type\":\"text\",\"name\":\"Translation\"},{\"field\":\"ignore\",\"type\":\"ignore\",\"name\":\"Theme\"},{\"field\":\"terms\",\"type\":\"taxonomy\",\"name\":\"Tags\",\"taxonomy\":\"tags\"},{\"field\":\"ignore\",\"type\":\"ignore\",\"name\":\"FB Category\"},{\"field\":\"gender\",\"type\":\"text\",\"name\":\"Gender\"},{\"field\":\"age\",\"type\":\"text\",\"name\":\"Age\"},{\"field\":\"location\",\"type\":\"text\",\"name\":\"Location\"},{\"field\":\"sub_location\",\"type\":\"text\",\"name\":\"Sub-Location\"},{\"field\":\"enumerator\",\"type\":\"text\",\"name\":\"Ennumerator\"},{\"field\":\"source\",\"type\":\"text\",\"name\":\"Source\"}]}"}},
+    {"model": "chn_spreadsheet.sheetprofile", "pk": 5, "fields": {"label": "kobo_master", "profile": "{\"name\":\"(Redundant)\",\"format\":\"excel\",\"label\":\"kobo_master\",\"skip_header\":1,\"taxonomies\":{},\"columns\":[{\"field\":\"timestamp\",\"type\":\"date\",\"name\":\"Date\",\"date_format\":\"%Y-%m-%dT%H: %M: %S.%f%z\"},{\"field\":\"body\",\"type\":\"text\",\"name\":\"Feedback\"},{\"field\":\"translation\",\"type\":\"text\",\"name\":\"Translation\"},{\"field\":\"ignore\",\"type\":\"ignore\",\"name\":\"Theme\"},{\"field\":\"terms\",\"type\":\"taxonomy\",\"name\":\"Tags\",\"taxonomy\":\"tags\"},{\"field\":\"ignore\",\"type\":\"ignore\",\"name\":\"FB Category\"},{\"field\":\"gender\",\"type\":\"text\",\"name\":\"Gender\"},{\"field\":\"age\",\"type\":\"text\",\"name\":\"Age\"},{\"field\":\"location\",\"type\":\"text\",\"name\":\"Location\"},{\"field\":\"sub_location\",\"type\":\"text\",\"name\":\"Sub-Location\"},{\"field\":\"enumerator\",\"type\":\"text\",\"name\":\"Ennumerator\"},{\"field\":\"collection_type\",\"type\":\"text\",\"name\":\"Collection Type\"}]}"}},
     {
         "fields": {
             "name": "main"
@@ -311,7 +311,7 @@
             "name": "feedback",
             "page": 1,
             "position": 2,
-            "settings": "{\"label\":\"Feedback\",\"columns\":[\"select_item\",\"timestamp\",\"body\",\"translation\",\"category\",\"tags\",\"feedback_type\",\"gender\",\"age\",\"location\",\"sub_location\",\"enumerator\",\"source\"],\"filters\":{\"terms\":[]},\"dynamic_filters\":[\"time_range\",\"category\",\"tags\",\"gender\",\"feedback_type\",\"age_range\",\"location\",\"sub_location\",\"enumerator\",\"source\",\"search\"],\"categories\":[\"bangladesh-refugee-crisis-sectors\"]}",
+            "settings": "{\"label\":\"Feedback\",\"columns\":[\"select_item\",\"timestamp\",\"body\",\"translation\",\"category\",\"tags\",\"feedback_type\",\"gender\",\"age\",\"location\",\"sub_location\",\"enumerator\",\"collection_type\"],\"filters\":{\"terms\":[]},\"dynamic_filters\":[\"time_range\",\"category\",\"tags\",\"gender\",\"feedback_type\",\"age_range\",\"location\",\"sub_location\",\"enumerator\",\"collection_type\",\"search\"],\"categories\":[\"bangladesh-refugee-crisis-sectors\"]}",
             "tab_type": "view-and-edit-table"
         },
         "model": "tabbed_page.tabinstance",

internewshid/hid/fixtures/sudan.json

@@ -74,7 +74,7 @@
       "model": "chn_spreadsheet.sheetprofile",
       "pk": 4
     },
-    {"model": "chn_spreadsheet.sheetprofile", "pk": 5, "fields": {"label": "kobo_master", "profile": "{\"name\":\"(Redundant)\",\"format\":\"excel\",\"label\":\"kobo_master\",\"skip_header\":1,\"taxonomies\":{},\"columns\":[{\"field\":\"timestamp\",\"type\":\"date\",\"name\":\"Date\",\"date_format\":\"%Y-%m-%dT%H: %M: %S.%f%z\"},{\"field\":\"body\",\"type\":\"text\",\"name\":\"Feedback\"},{\"field\":\"translation\",\"type\":\"text\",\"name\":\"Translation\"},{\"field\":\"ignore\",\"type\":\"ignore\",\"name\":\"Theme\"},{\"field\":\"terms\",\"type\":\"taxonomy\",\"name\":\"Tags\",\"taxonomy\":\"tags\"},{\"field\":\"ignore\",\"type\":\"ignore\",\"name\":\"FB Category\"},{\"field\":\"gender\",\"type\":\"text\",\"name\":\"Gender\"},{\"field\":\"age\",\"type\":\"text\",\"name\":\"Age\"},{\"field\":\"location\",\"type\":\"text\",\"name\":\"Location\"},{\"field\":\"enumerator\",\"type\":\"text\",\"name\":\"Ennumerator\"},{\"field\":\"source\",\"type\":\"text\",\"name\":\"Source\"}]}"}},
+    {"model": "chn_spreadsheet.sheetprofile", "pk": 5, "fields": {"label": "kobo_master", "profile": "{\"name\":\"(Redundant)\",\"format\":\"excel\",\"label\":\"kobo_master\",\"skip_header\":1,\"taxonomies\":{},\"columns\":[{\"field\":\"timestamp\",\"type\":\"date\",\"name\":\"Date\",\"date_format\":\"%Y-%m-%dT%H: %M: %S.%f%z\"},{\"field\":\"body\",\"type\":\"text\",\"name\":\"Feedback\"},{\"field\":\"translation\",\"type\":\"text\",\"name\":\"Translation\"},{\"field\":\"ignore\",\"type\":\"ignore\",\"name\":\"Theme\"},{\"field\":\"terms\",\"type\":\"taxonomy\",\"name\":\"Tags\",\"taxonomy\":\"tags\"},{\"field\":\"ignore\",\"type\":\"ignore\",\"name\":\"FB Category\"},{\"field\":\"gender\",\"type\":\"text\",\"name\":\"Gender\"},{\"field\":\"age\",\"type\":\"text\",\"name\":\"Age\"},{\"field\":\"location\",\"type\":\"text\",\"name\":\"Location\"},{\"field\":\"enumerator\",\"type\":\"text\",\"name\":\"Ennumerator\"},{\"field\":\"collection_type\",\"type\":\"text\",\"name\":\"Collection Type\"}]}"}},
     {
         "fields": {
             "name": "main"
@@ -89,7 +89,7 @@
             "name": "feedback",
             "page": 1,
             "position": 2,
-            "settings": "{\"label\":\"Feedback\",\"columns\":[\"select_item\",\"timestamp\",\"body\",\"translation\",\"category\",\"tags\",\"feedback_type\",\"gender\",\"age\",\"location\",\"enumerator\",\"source\",\"external_id\"],\"filters\":{\"terms\":[]},\"dynamic_filters\":[\"time_range\",\"category\",\"tags\",\"gender\",\"feedback_type\",\"age_range\",\"location\",\"enumerator\",\"source\",\"external_id\"],\"categories\":[\"sudan\"]}",
+            "settings": "{\"label\":\"Feedback\",\"columns\":[\"select_item\",\"timestamp\",\"body\",\"translation\",\"category\",\"tags\",\"feedback_type\",\"gender\",\"age\",\"location\",\"enumerator\",\"collection_type\",\"external_id\"],\"filters\":{\"terms\":[]},\"dynamic_filters\":[\"time_range\",\"category\",\"tags\",\"gender\",\"feedback_type\",\"age_range\",\"location\",\"enumerator\",\"collection_type\",\"external_id\"],\"categories\":[\"sudan\"]}",
             "tab_type": "view-and-edit-table"
         },
         "model": "tabbed_page.tabinstance",

internewshid/hid/forms/item.py

@@ -37,7 +37,7 @@ class AddEditItemForm(forms.Form):
         widget=forms.Textarea,
         required=False
     )
-    source = forms.CharField(
+    collection_type = forms.CharField(
         widget=forms.Textarea,
         required=False
     )

internewshid/hid/migrations/0005_initial_tabbed_page.py

@@ -35,7 +35,7 @@ def create_initial_view_and_edit_page(apps, schema_editor):
                 'columns': ['select_item', 'created', 'timestamp', 'body',
                             'category'],
                 'label': 'Questions',
-                'source': 'geopoll',
+                'collection_type': 'geopoll',
                 'categories': ['ebola-questions'],
                 'filters': {
                     'terms': ['item-types:question']
@@ -55,7 +55,7 @@ def create_initial_view_and_edit_page(apps, schema_editor):
                 'columns': ['select_item', 'created', 'timestamp', 'body',
                             'network_provider'],
                 'label': 'Rumors',
-                'source': 'rapidpro',
+                'collection_type': 'rapidpro',
                 'filters': {
                     'terms': ['item-types:rumor']
                 }

internewshid/hid/tables.py

@@ -89,10 +89,10 @@ class ItemTable(tables.Table):
         verbose_name=_('Enumerator'),
         attrs={'th': {'id': 'header-enumerator'}}
     )
-    source = tables.TemplateColumn(
-        template_name='hid/source_column.html',
-        verbose_name=_('Source'),
-        attrs={'th': {'id': 'header-source'}}
+    collection_type = tables.TemplateColumn(
+        template_name='hid/collection_type_column.html',
+        verbose_name=_('Collection Type'),
+        attrs={'th': {'id': 'header-collection-type'}}
     )
     external_id = tables.TemplateColumn(
         template_name='hid/external_id_column.html',

internewshid/hid/tabs/view_and_edit_table.py

@@ -97,9 +97,9 @@ class ViewAndEditTableTab(object):
         if enumerators and enumerators == 'All Enumerators':
             filters.pop('enumerator')
 
-        source = filters.get('source')
-        if source and source == 'All Sources':
-            filters.pop('source')
+        collection_type = filters.get('collection_type')
+        if collection_type and collection_type == 'All Collection Types':
+            filters.pop('collection_type')
 
         limit = kwargs.get('per_page', 100)
         filters['limit'] = limit
@@ -200,9 +200,9 @@ class ViewAndEditTableTab(object):
         enumerators = transport_items.list_options('enumerator')
         return {'items': enumerators}
 
-    def _get_source_options(self, items_list, **kwargs):
-        sources = transport_items.list_options('source')
-        return {'items': sources}
+    def _get_collection_type_options(self, items_list, **kwargs):
+        collection_types = transport_items.list_options('collection_type')
+        return {'items': collection_types}
 
     def _get_feedback_type_options(self):
         terms = transport_terms.list(taxonomy='item-types')
@@ -250,7 +250,7 @@ class ViewAndEditTableTab(object):
         sub_location_options = self._get_sub_location_options(items, **kwargs)
         gender_options = self._get_gender_options(items, **kwargs)
         enumerator_options = self._get_enumerator_options(items, **kwargs)
-        source_options = self._get_source_options(items, **kwargs)
+        collection_type_options = self._get_collection_type_options(items, **kwargs)
         feedback_type_options = self._get_feedback_type_options()
 
         per_page = int(kwargs.get('per_page', 100))
@@ -277,7 +277,7 @@ class ViewAndEditTableTab(object):
             'add_button_for': self._get_item_type_filter(kwargs),
             'type_label': kwargs.get('label', '?'),
             'table': table,
-            'source': kwargs.get('source'),
+            'collection_type': kwargs.get('collection_type'),
             'actions': actions,
             'category_options': category_options,
             'feedback_type_options': feedback_type_options,
@@ -285,7 +285,7 @@ class ViewAndEditTableTab(object):
             'sub_locations': sub_location_options,
             'gender': gender_options,
             'enumerator': enumerator_options,
-            'source_filters': source_options,
+            'collection_type_filters': collection_type_options,
             'next': reverse('tabbed-page', kwargs={
                 'name': tab_instance.page.name,
                 'tab_name': tab_instance.name

internewshid/hid/templates/hid/add_edit_item.html

@@ -66,7 +66,7 @@
 
                 </div>
                 <div class="item-content">
-                    <div class="item-source-wrapper">
+                    <div class="item-collection-type-wrapper">
                         {% if form.body %}
                         <div class="item-original-text {% if form.body.errors %}has-error{% endif %}">
                             <label>
@@ -80,7 +80,7 @@
                         </div>
                         {% endif %}
 
-                        <div class="item-source-meta">
+                        <div class="item-collection-type-meta">
 
                             <div class="item-reported-date {% if form.timestamp.errors %}has-error{% endif %}">
                                 <label>
@@ -165,15 +165,15 @@
                             </div>
                             {% endif %}
 
-                            {% if form.source %}
-                            <div class="item-source {% if form.source.errors %}has-error{% endif %}">
+                            {% if form.collection_type %}
+                            <div class="item-collection-type {% if form.collection_type.errors %}has-error{% endif %}">
                                 <label>
-                                    {% trans "Source" %}
-                                    {% if form.source.errors %}
-                                    <span class='help-block'>({{ form.source.errors|join:", " }})</span>
+                                    {% trans "Collection Type" %}
+                                    {% if form.collection_type.errors %}
+                                    <span class='help-block'>({{ form.collection_type.errors|join:", " }})</span>
                                     {% endif %}
                                 </label>
-                                {{ form.source|add_class:'form-control'|attr:"Source"|attr:"rows:1"|attr:"cols:3"|attr:"style:resize:none;" }}
+                                {{ form.collection_type|add_class:'form-control'|attr:"Collection Type"|attr:"rows:1"|attr:"cols:3"|attr:"style:resize:none;" }}
                             </div>
                             {% endif %}
                         </div>