WIP key value store

Pipfile

@@ -10,7 +10,7 @@ python_version = "3.6"
 asgiref = "*"
 "django-bootstrap3" = "*"
 "django-tables2" = "*"
-"django.js" = {ref = "ca328a94b00023bd64f4fc1c908675edaaf2ac19",git = "https://git@github.com/aptivate/django.js.git",editable = true}
+"django.js" = {editable = true,git = "https://git@github.com/aptivate/django.js.git",ref = "ca328a94b00023bd64f4fc1c908675edaaf2ac19"}
 "linecache2" = "*"
 Django = ">2.2.8,<3.0"
 Pillow = "*"
@@ -32,6 +32,7 @@ pytz = "*"
 rest-pandas = "*"
 mysqlclient = "*"
 django-debug-toolbar-template-timings = "*"
+ipdb = "*"
 
 [dev-packages]
 django-debug-toolbar = "*"

internewshid/chn_spreadsheet/importer.py

@@ -135,6 +135,11 @@ class Importer(object):
                 if value:
                     self._append_term_to_item(
                         item, 'tags', value.strip())
+            elif col['type'] == 'keyvalue':
+                key, value = converter.convert_value()
+                if value:
+                    self._append_keyvalue_to_item(
+                        item, key, value)
             else:
                 converter.add_to(item)
 
@@ -146,6 +151,10 @@ class Importer(object):
         term = self._get_term_dict(taxonomy, name)
         item.setdefault('terms', []).append(term)
 
+    def _append_keyvalue_to_item(self, item, key, value):
+        keyvalue = {'key': key, 'value': value}
+        item.setdefault('keyvalues', []).append(keyvalue)
+
     def _get_term_dict(self, taxonomy, name):
         return {'taxonomy': taxonomy, 'name': name}
 
@@ -155,11 +164,15 @@ class Importer(object):
         for obj in objects:
             row = obj.pop('_row_number', '')
             terms = obj.pop('terms', [])
+            keyvalues = obj.pop('keyvalues', [])
             try:
                 item = transport.items.create(obj)
                 for term in terms:
                     transport.items.add_terms(
                         item['id'], term['taxonomy'], term['name'])
+                for keyvalue in keyvalues:
+                    transport.items.add_keyvalue(
+                        item['id'], keyvalue['key'], keyvalue['value'])
             except ItemNotUniqueException:
                 pass
 
@@ -254,6 +267,7 @@ class CellConverter(object):
         self.value = value
         self.type = col_spec['type']
         self.field = col_spec['field']
+        self.name = col_spec.get('name')
         self.date_format = col_spec.get('date_format', None)
 
     def add_to(self, object_dict):
@@ -268,7 +282,7 @@ class CellConverter(object):
             'integer': lambda x: int(x),
             'number': lambda x: Decimal(x),
             'taxonomy': lambda x: x if x else '',
-            'keyvalue': lambda x: x if x else '',
+            'keyvalue': lambda x: (self.name, x) if x else '',
             'protection_concern': lambda x: 'Protection Concern' if x.lower() == 'yes' else ''
         }
         if self.type not in converters:

internewshid/chn_spreadsheet/tests/importer_tests.py

@@ -139,6 +139,7 @@ def test_order_columns_with_first_row_return_first_row_order(importer):
     assert ordered == [cleaned[2], cleaned[0], cleaned[1]]
 
 
+@pytest.mark.skip("importing all columns")  #Now importing all missing columns as key value pairs
 def test_order_columns_ignores_extra_columns_in_first_row(importer):
     cleaned = _make_columns_row(COLUMN_LIST)
     first_row = ['Message', 'Province', 'Sub-Province', 'None', 'None', 'None']
@@ -147,7 +148,7 @@ def test_order_columns_ignores_extra_columns_in_first_row(importer):
 
     assert ordered == [cleaned[2], cleaned[0], cleaned[1]]
 
-
+@pytest.mark.skip("importing all columns")  #Now importing all missing columns as key value pairs
 def test_order_columns_ignores_none_and_missing_columns_in_first_row(importer):
     first_row = ['Province', None]
 

internewshid/chn_spreadsheet/tests/key_values_tests.py

@@ -30,7 +30,6 @@ def test_kobo_keyvalue_imported(importer, django_db_setup, taxonomies):  # noqa
     assert num_saved > 0
 
     items = transport.items.list_items()['results']
-    pprint(items)
     assert len(items) == num_saved
 
     # default ordering is by timestamp desc
@@ -40,6 +39,8 @@ def test_kobo_keyvalue_imported(importer, django_db_setup, taxonomies):  # noqa
     assert items[2]['language'] == 'English'
     assert items[2]['risk'] == '1'
     assert items[2]['values']['KV1'] == 'one'
+    assert items[1]['values']['KV1'] == '2020-12-12 00:00:00'
+    assert items[0]['values']['KV1'] == '123'
     assert isinstance(items[2]['timestamp'], datetime.datetime)
 
 

internewshid/rest_api/serializers.py

 from rest_framework import serializers, validators
 
-from data_layer.models import Item
+from data_layer.models import Item, Value
 from taxonomies.models import Taxonomy, Term
 
 from .fields import IgnoreMicrosecondsDateTimeField
@@ -79,6 +79,10 @@ class ItemSerializer(serializers.ModelSerializer):
 
     timestamp = IgnoreMicrosecondsDateTimeField()
     terms = TermSerializer(many=True, required=False)
+    values = serializers.SerializerMethodField()
+
+    def get_values(self, item):
+        return {kv.key.key:kv.value for kv in item.values.all()}
 
     def create(self, validated_data):
         """ Create an item with nested metadata terms."""

internewshid/rest_api/views.py

@@ -12,7 +12,7 @@ from rest_framework.response import Response
 from rest_framework_bulk.mixins import BulkDestroyModelMixin
 from rest_pandas import PandasView
 
-from data_layer.models import Item
+from data_layer.models import Item, Key, Value
 from taxonomies.models import Taxonomy, Term
 
 from .serializers import (
@@ -32,7 +32,6 @@ class ItemViewSet(viewsets.ModelViewSet, BulkDestroyModelMixin):
         'sub_location',
         'language',
         'risk',
-        'values',
         'gender',
         'age',
         'contributor',
@@ -184,6 +183,23 @@ class ItemViewSet(viewsets.ModelViewSet, BulkDestroyModelMixin):
         serializer = ItemSerializer(item)
         return Response(serializer.data, status=status.HTTP_200_OK)
 
+
+    @action(methods=['post'], detail=True)
+    def add_keyvalue(self, request, item_pk):
+        try:
+            item = Item.objects.get(pk=item_pk)
+        except Item.DoesNotExist as e:
+            data = {'detail': str(e)}
+            return Response(data, status=status.HTTP_404_NOT_FOUND)
+
+        keyvalue = request.data
+        key, _ = Key.objects.get_or_create(key=keyvalue['key'])
+        value = Value.objects.create(key=key, value=keyvalue['value'], message=item)
+
+        serializer = ItemSerializer(item)
+        return Response(serializer.data, status=status.HTTP_200_OK)
+
+
     @action(methods=['post'], detail=True)
     def delete_all_terms(self, request, item_pk):
         taxonomy_slug = request.data['taxonomy']

internewshid/transport/items.py

@@ -180,13 +180,27 @@ def add_terms(item_id, taxonomy_slug, names):
 
     if status.is_success(response.status_code):
         return response.data
-    else:
-        response.data['status_code'] = response.status_code
-        response.data['terms'] = terms
-        response.data['item_id'] = item_id
-        raise TransportException(response.data)
 
-    return response.data
+    response.data['status_code'] = response.status_code
+    response.data['terms'] = terms
+    response.data['item_id'] = item_id
+    raise TransportException(response.data)
+
+
+def add_keyvalue(item_id, key, value):
+    view = get_view({'post': 'add_keyvalue'})
+
+    keyvalue = {'key': key, 'value': value}
+    request = request_factory.post('', keyvalue)
+    response = view(request, item_pk=item_id)
+
+    if status.is_success(response.status_code):
+        return response.data
+
+    response.data['status_code'] = response.status_code
+    response.data['value'] = keyvalue
+    response.data['item_id'] = item_id
+    raise TransportException(response.data)
 
 
 def delete_all_terms(item_id, taxonomy_slug):