Merge branch 'keyword-search' into 'staging'

Keyword search

See merge request !137

internewshid/hid/apps.py

@@ -15,6 +15,7 @@ class HidAppConfig(AppConfig):
             FeedbackTypeFilter,
             GenderFilter,
             LocationFilter,
+            SearchFilter,
             SourceFilter,
             TagsFilter,
             TimeRangeFilter,
@@ -35,6 +36,7 @@ class HidAppConfig(AppConfig):
         register_filter('tags', TagsFilter())
         register_filter('feedback_type', FeedbackTypeFilter())
         register_filter('external_id', ExternalIdFilter())
+        register_filter('search', SearchFilter())
 
         register_tab('view-and-edit-table', ViewAndEditTableTab())
 

internewshid/hid/filters.py

@@ -98,3 +98,11 @@ class ExternalIdFilter(object):
 
         if pattern:
             filters.update(external_id_pattern=pattern)
+
+
+class SearchFilter(object):
+    def apply(self, filters, query_dict, **kwargs):
+        search = query_dict.get('search', None)
+
+        if search:
+            filters.update(search=search)

internewshid/hid/fixtures/bangladesh-refugee-crisis.json

@@ -311,7 +311,7 @@
             "name": "feedback",
             "page": 1,
             "position": 2,
-            "settings": "{\"label\":\"Feedback\",\"columns\":[\"select_item\",\"timestamp\",\"body\",\"translation\",\"category\",\"tags\",\"feedback_type\",\"gender\",\"age\",\"location\",\"enumerator\",\"source\",\"external_id\"],\"filters\":{\"terms\":[]},\"dynamic_filters\":[\"time_range\",\"category\",\"tags\",\"gender\",\"feedback_type\",\"age_range\",\"location\",\"enumerator\",\"source\",\"external_id\"],\"categories\":[\"bangladesh-refugee-crisis-sectors\"]}",
+            "settings": "{\"label\":\"Feedback\",\"columns\":[\"select_item\",\"timestamp\",\"body\",\"translation\",\"category\",\"tags\",\"feedback_type\",\"gender\",\"age\",\"location\",\"enumerator\",\"source\",\"external_id\"],\"filters\":{\"terms\":[]},\"dynamic_filters\":[\"time_range\",\"category\",\"tags\",\"gender\",\"feedback_type\",\"age_range\",\"location\",\"enumerator\",\"source\",\"external_id\",\"search\"],\"categories\":[\"bangladesh-refugee-crisis-sectors\"]}",
             "tab_type": "view-and-edit-table"
         },
         "model": "tabbed_page.tabinstance",

internewshid/hid/templates/hid/tabs/filters/search.html

+{% load i18n %}
+<div class="filter-search input-group">
+  <label class="control-label" for="search-filter">{% trans "Keywords" %}</label>
+  <input id="search-filter" type="text" placeholder="Keywords" name="search" class="form-control" value="{{ request.GET.search }}">
+</div>

internewshid/hid/tests/views_tests.py

@@ -623,6 +623,42 @@ def test_table_items_filtered_by_external_id():
     assert not_matching['id'] not in ids
 
 
+@pytest.mark.django_db
+def test_table_items_filtered_by_keywords():
+    matching_1 = transport.items.create({
+        'body': 'Latrine hurricane camp',
+        'translation': 'Translation 1',
+    })
+
+    matching_2 = transport.items.create({
+        'body': 'Body 2',
+        'translation': 'Camp food latrines',
+    })
+
+    not_matching = transport.items.create({
+        'body': 'Not matching',
+        'translation': 'Violence segregation food',
+    })
+
+    page = TabbedPageFactory()
+    tab_instance = TabInstanceFactory(page=page)
+    request = MagicMock(session={'THREADED_FILTERS': {}},
+                        GET={'search': 'Latrine camp'})
+    tab = ViewAndEditTableTab()
+    context_data = tab.get_context_data(
+        tab_instance, request, categories=[],
+        dynamic_filters=['search']
+    )
+
+    table = context_data['table']
+
+    ids = [t['id'] for t in table.data.data]
+
+    assert matching_1['id'] in ids
+    assert matching_2['id'] in ids
+    assert not_matching['id'] not in ids
+
+
 @pytest.mark.django_db
 def test_dynamic_filters_read_from_tab_instance():
     page = TabbedPageFactory(name='main')

internewshid/rest_api/tests/item_list_view_tests.py

@@ -329,3 +329,159 @@ def test_ordering_by_body():
     assert payload[1]['body'] == "item 2"
     assert payload[2]['body'] == "item 3"
     assert payload[3]['body'] == "item 4"
+
+
+@pytest.mark.django_db
+def test_filter_message_by_keyword():
+    create_item(
+        body="""Latrine ipsum dolor sit amet, consectetur adipiscing elit.
+Pellentesque vitae ipsum a magna rutrum facilisis. Fusce vitae dolor dolor.
+Nullam."""
+    )
+
+    create_item(
+        body="""Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Suspendisse ut orci diam. Donec scelerisque id massa vitae laoreet. Ut sit."""
+    )
+
+    create_item(
+        body="""Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Pellentesque ac orci felis. Pellentesque hendrerit laoreet dolor nec euismod.
+ Fusce pretium."""
+    )
+
+    create_item(
+        body="""Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+ Donec at justo sit amet ante LATRINE semper tempus. Suspendisse vulputate
+ urna nec."""
+    )
+
+    create_item(
+        body="""Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Mauris nec mauris vestibulum, laoreet mi ut, facilisis massa. Pellentesque
+ quam tortor. latrines"""
+    )
+
+    payload = get(
+        data={
+            'search': 'latrine',
+        }
+    ).data
+
+    assert len(payload) == 3
+
+    assert 'Latrine' in payload[0]['body']
+    assert 'LATRINE' in payload[1]['body']
+    assert 'latrines' in payload[2]['body']
+
+
+@pytest.mark.django_db
+def test_filter_translation_by_keyword():
+    create_item(
+        body="item 1",
+        translation="""Latrine ipsum dolor sit amet, consectetur adipiscing elit.
+Pellentesque vitae ipsum a magna rutrum facilisis. Fusce vitae dolor dolor.
+Nullam."""
+    )
+
+    create_item(
+        body="item 2",
+        translation="""Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Suspendisse ut orci diam. Donec scelerisque id massa vitae laoreet. Ut sit."""
+    )
+
+    create_item(
+        body="item 3",
+        translation="""Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Pellentesque ac orci felis. Pellentesque hendrerit laoreet dolor nec euismod.
+ Fusce pretium."""
+    )
+
+    create_item(
+        body="item 4",
+        translation="""Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+ Donec at justo sit amet ante LATRINE semper tempus. Suspendisse vulputate
+ urna nec."""
+    )
+
+    create_item(
+        body="item 5",
+        translation="""Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+        Mauris nec mauris vestibulum, laoreet mi ut, facilisis massa.
+        Pellentesque quam tortor. latrines""",
+    )
+
+    payload = get(
+        data={
+            'search': 'latrine',
+        }
+    ).data
+
+    assert len(payload) == 3
+
+    assert 'Latrine' in payload[0]['translation']
+    assert 'LATRINE' in payload[1]['translation']
+    assert 'latrines' in payload[2]['translation']
+
+
+@pytest.mark.django_db
+def test_filtering_a_message_must_match_all_keywords():
+    item_1 = create_item(
+        body="""Latrine ipsum dolor sit amet, consectetur adipiscing elit.
+Pellentesque vitae ipsum a magna rutrum facilisis. Fusce vitae dolor dolor.
+Nullam."""
+    )
+
+    create_item(
+        body="""Lorem ipsum dolor sit amet, latrine consectetur adipiscing elit.
+Suspendisse ut orci diam. Donec scelerisque id massa vitae laoreet. Ut sit."""
+    )
+
+    create_item(
+        body="""Lorem ipsum dolor sit amet, magna consectetur adipiscing elit.
+Pellentesque ac orci felis. Pellentesque hendrerit laoreet dolor nec euismod.
+ Fusce pretium."""
+    )
+
+    payload = get(
+        data={
+            'search': 'latrine magna',
+        }
+    ).data
+
+    assert len(payload) == 1
+
+    assert payload[0]['body'] == item_1.data['body']
+
+
+@pytest.mark.django_db
+def test_filtering_a_translation_must_match_all_keywords():
+    item_1 = create_item(
+        body="item 1",
+        translation="""Latrine ipsum dolor sit amet, consectetur adipiscing elit.
+Pellentesque vitae ipsum a magna rutrum facilisis. Fusce vitae dolor dolor.
+Nullam."""
+    )
+
+    create_item(
+        body="item 2",
+        translation="""Lorem ipsum dolor sit amet, latrine consectetur adipiscing elit.
+Suspendisse ut orci diam. Donec scelerisque id massa vitae laoreet. Ut sit."""
+    )
+
+    create_item(
+        body="item 3",
+        translation="""Lorem ipsum dolor sit amet, magna consectetur adipiscing elit.
+Pellentesque ac orci felis. Pellentesque hendrerit laoreet dolor nec euismod.
+ Fusce pretium."""
+    )
+
+    payload = get(
+        data={
+            'search': 'latrine magna',
+        }
+    ).data
+
+    assert len(payload) == 1
+
+    assert payload[0]['body'] == "item 1"

internewshid/rest_api/views.py

-from django.db.models import Count
+from django.db.models import Count, Q
 from django.utils.translation import ugettext as _
 
 from rest_framework import status, viewsets
@@ -106,6 +106,21 @@ class ItemViewSet(viewsets.ModelViewSet, BulkDestroyModelMixin):
         if external_id_pattern is not None:
             items = items.filter(external_id__contains=external_id_pattern)
 
+        search = self.request.query_params.get('search', None)
+        if search is not None:
+            # Either the translation or body must match all the keywords
+            # Anything more sophisticated and we should use a search backend
+            words = search.split()
+            body_q = reduce(
+                lambda x, y: x & y, [Q(body__icontains=w) for w in words]
+            )
+            translation_q = reduce(
+                lambda x, y: x & y, [Q(translation__icontains=w)
+                                     for w in words]
+            )
+
+            items = items.filter(body_q | translation_q)
+
         ordering = self.request.query_params.get('ordering', '-timestamp')
         items = items.order_by(ordering)