refactoring 2

.gitignore

@@ -0,0 +1,92 @@
+*.swp
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+.venv/
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject

es.py

@@ -0,0 +1,136 @@
+import urllib2
+
+# Sends data to ES.
+# Raises exceptions: yes.
+def send_to_es(CONFIG, index_name, data):
+    # HTTP bulk insert toward ES
+
+    url = '{}/{}/{}/_bulk'.format(
+        CONFIG['ES_URL'],
+        index_name,
+        CONFIG['ES_Type']
+    )
+
+    try:
+        http_res = urllib2.urlopen(url, data)
+    except Exception as e:
+        raise Exception(
+            'Error while executing HTTP bulk insert on {} - {}'.format(
+                index_name, str(e)
+            )
+        )
+
+    # Interpreting HTTP bulk insert response
+
+    http_plaintext = http_res.read()
+
+    if(http_res.getcode() != 200):
+        raise Exception(
+            'Bulk insert on {} failed - '
+            'HTTP status code = {} - '
+            'Response {}'.format(
+                index_name, http_res.getcode(), http_plaintext
+            )
+        )
+
+    try:
+        json_res = json.loads(http_plaintext)
+    except Exception as e:
+        raise Exception(
+            'Error while decoding JSON HTTP response - '
+            '{} - '
+            'first 100 characters: {}'.format(
+                str(e),
+                http_plaintext[:100],
+            )
+        )
+
+    if json_res['errors']:
+        raise Exception(
+            'Bulk insert on {} failed to process '
+            'one or more documents'.format(index_name)
+        )
+
+# Checks if index_name exists.
+# Returns: True | False.
+# Raises exceptions: yes.
+def does_index_exist(index_name, CONFIG):
+    url = '{}/{}'.format(CONFIG['ES_URL'], index_name)
+
+    try:
+            head_req = urllib2.Request(url)
+            head_req.get_method = lambda : 'HEAD'
+            http_res = urllib2.urlopen(head_req)
+            return http_res.getcode() == 200
+    except urllib2.HTTPError as err:
+        if err.code == 404:
+            return False
+        else:
+            raise Exception(
+                'Error while checking if {} index exists: {}'.format(
+                    index_name, str(err)
+                )
+            )
+
+# Creates index 'index_name' using template given in config.
+# Raises exceptions: yes.
+def create_index(index_name, CONF_DIR, CONFIG):
+
+    # index already exists?
+    if does_index_exist(index_name, CONFIG):
+        return
+
+    # index does not exist, creating it
+    tpl_path = '{}/{}'.format(CONF_DIR, CONFIG['ES_IndexTemplateFileName'])
+
+    try:
+        with open(tpl_path, "r") as f:
+            tpl = f.read()
+    except Exception as e:
+        raise Exception(
+            'Error while reading index template from file {}: {}'.format(
+                tpl_path, str(e)
+            )
+        )
+
+    url = '{}/{}'.format(CONFIG['ES_URL'], index_name)
+
+    last_err = None
+    try:
+        http_res = urllib2.urlopen(url, tpl)
+    except Exception as e:
+        # something went wrong: does index exist anyway?
+        last_err = str(e)
+        pass
+
+    if does_index_exist(index_name, CONFIG):
+        return
+
+    err = "An error occurred while creating index {} from template {}: "
+    if last_err:
+        err += last_err
+    else:
+        err += "error unknown"
+    raise Exception(err.format(index_name, tpl_path))
+
+def prepare_for_http_auth(CONFIG):
+    if CONFIG['ES_AuthType'] != 'none':
+        pwdman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+        pwdman.add_password(
+            None,
+            CONFIG['ES_URL'],
+            CONFIG['ES_UserName'],
+            CONFIG['ES_Password']
+        )
+
+        if CONFIG['ES_AuthType'] == 'basic':
+            auth_handler = urllib2.HTTPBasicAuthHandler(pwdman)
+        elif CONFIG['ES_AuthType'] == 'digest':
+            auth_handler = urllib2.HTTPDigestAuthHandler(pwdman)
+        else:
+            raise Exception(
+                'Unexpected authentication type: {}'.format(CONFIG['ES_AuthType'])
+            )
+
+        opener = urllib2.build_opener(auth_handler)
+        urllib2.install_opener(opener)