Allow destroying locally-stored submissions after

onadata/apps/restservice/__init__.py

@@ -1,2 +1,7 @@
-SERVICE_CHOICES = ((u'f2dhis2', u'f2dhis2'), (u'generic_json', u'JSON POST'),
-                   (u'generic_xml', u'XML POST'), (u'bamboo', u'bamboo'))
+from django.conf import settings
+
+SERVICE_CHOICES = [(u'f2dhis2', u'f2dhis2'), (u'generic_json', u'JSON POST'),
+                   (u'generic_xml', u'XML POST'), (u'bamboo', u'bamboo')]
+
+if settings.ENABLE_DESTRUCTIVE_REST_SERVICE:
+    SERVICE_CHOICES.append((u'destructive_json', u'DESTRUCTIVE JSON POST'))

onadata/apps/restservice/services/destructive_json.py

@@ -0,0 +1,58 @@
+import hashlib
+import json
+import requests
+from lxml import etree
+from StringIO import StringIO
+
+from django.conf import settings
+from reversion.models import Version
+
+from onadata.apps.restservice.RestServiceInterface import RestServiceInterface
+
+
+class ServiceDefinition(RestServiceInterface):
+    ''' This service ERASES all submission data in Postgres and Mongo except
+    whatever is contained within the XML elements specified in
+    `XML_CHILDREN_OF_ROOT_TO_KEEP`. The SHA256 of the submission's original
+    JSON is also saved. That original JSON is then POSTed to the specified URL.
+    Use with extreme caution! '''
+
+    XML_CHILDREN_OF_ROOT_TO_KEEP = ('formhub', 'meta')
+    id = u'destructive_json'
+    verbose_name = u'DESTRUCTIVE JSON POST'
+
+    def redact_submission(self, parsed_instance, append_elements=[]):
+        instance = parsed_instance.instance
+        xml_parser = etree.parse(StringIO(instance.xml))
+        xml_root = xml_parser.getroot()
+        # remove all user-generated data!
+        for child in xml_root.getchildren():
+            if child.tag not in self.XML_CHILDREN_OF_ROOT_TO_KEEP:
+                xml_root.remove(child)
+
+        # append any specified new elements before saving
+        for el_to_append in append_elements:
+            xml_root.append(el_to_append)
+
+        instance.xml = etree.tounicode(xml_root)
+        del instance._parser # has cached copy of xml
+        del parsed_instance._dict_cache # cached copy of json
+        instance.save()
+
+        # delete revisions!
+        Version.objects.get_for_object(instance).delete()
+
+    def send(self, url, parsed_instance):
+        post_data = json.dumps(parsed_instance.to_dict_for_mongo())
+        hasher = hashlib.sha256()
+        hasher.update(post_data)
+        post_data_hash = hasher.hexdigest()
+        hash_element = etree.Element('sha256')
+        hash_element.text = post_data_hash
+
+        self.redact_submission(parsed_instance, [hash_element])
+
+        headers = {"Content-Type": "application/json"}
+        response = requests.post(
+            url, headers=headers, data=post_data, timeout=60)
+        response.raise_for_status()

onadata/apps/restservice/utils.py

@@ -1,9 +1,33 @@
+import logging
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from onadata.apps.restservice import SERVICE_CHOICES
 from onadata.apps.restservice.models import RestService
 
+valid_service_names = [x[0] for x in SERVICE_CHOICES]
 
 def call_service(parsed_instance):
     # lookup service
     instance = parsed_instance.instance
+
+    # check for a forced service
+    if settings.FORCE_REST_SERVICE_NAME:
+        if not settings.FORCE_REST_SERVICE_URL:
+            raise ImproperlyConfigured(
+                'You must specify FORCE_REST_SERVICE_URL when setting '
+                'FORCE_REST_SERVICE_NAME'
+            )
+        if settings.FORCE_REST_SERVICE_NAME not in valid_service_names:
+            raise ImproperlyConfigured(
+                'FORCE_REST_SERVICE_NAME specifies a service not listed in '
+                'onadata.apps.restservice.SERVICE_CHOICES'
+            )
+        RestService.objects.get_or_create(
+            xform=instance.xform,
+            name=settings.FORCE_REST_SERVICE_NAME,
+            service_url=settings.FORCE_REST_SERVICE_URL
+        )
+
     services = RestService.objects.filter(xform=instance.xform)
     # call service send with url and data parameters
     for sv in services:
@@ -12,8 +36,16 @@ def call_service(parsed_instance):
             service = sv.get_service_definition()()
             service.send(sv.service_url, parsed_instance)
         except:
-            # TODO: Handle gracefully | requeue/resend
-            pass
+            if settings.FAILED_REST_SERVICE_BLOCKS_SUBMISSION:
+                raise
+            else:
+                # TODO: Handle gracefully | requeue/resend
+                logging.warning(
+                    u'RestService {} failed; service_url={}'.format(
+                        sv.name, sv.service_url
+                    ),
+                    exc_info=True
+                )
 
 
 def call_ziggy_services(ziggy_instance, uuid):

onadata/settings/kc_environ.py

@@ -147,7 +147,7 @@
 
 # Optional Sentry configuration: if desired, be sure to install Raven and set
 # RAVEN_DSN in the environment
-if 'RAVEN_DSN' in os.environ:
+if os.environ.get('RAVEN_DSN', False):
     try:
         import raven
     except ImportError:
@@ -232,6 +232,23 @@
     },
 }
 
+# WARNING! Enabling this allows users to THROW AWAY ALL SUBMISSION DATA!
+# When used, the Destructive REST Service places the ENTIRE responsibility for
+# storing incoming submissions on the user-specified external service
+ENABLE_DESTRUCTIVE_REST_SERVICE = os.environ.get(
+    'ENABLE_DESTRUCTIVE_REST_SERVICE', 'False') == 'True'
+
+# Force all submissions to be sent to the specified type of REST service.
+# Choose from among `onadata.apps.restservice.SERVICE_CHOICES` and set
+# `FORCE_REST_SERVICE_URL` to the desired HTTP endpoint
+FORCE_REST_SERVICE_NAME = os.environ.get('FORCE_REST_SERVICE_NAME', False)
+FORCE_REST_SERVICE_URL = os.environ.get('FORCE_REST_SERVICE_URL', False)
+
+# When set to True, Raise an unhandled exception (and return a HTTP 500 to the
+# client) if any external REST service fails during the submission process
+FAILED_REST_SERVICE_BLOCKS_SUBMISSION = os.environ.get(
+    'FAILED_REST_SERVICE_BLOCKS_SUBMISSION', 'False') == 'True'
+
 ### ISSUE 242 TEMPORARY FIX ###
 # See https://github.com/kobotoolbox/kobocat/issues/242
 ISSUE_242_MINIMUM_INSTANCE_ID = os.environ.get(