Implement a simple take on translating RFS bulk requests to use index…

…-type rewrite rules from the type mappings sanitization transformer.

Signed-off-by: Greg Schohn <[email protected]>

transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/NameMappingClasspathResourceLocator.java

@@ -74,7 +74,7 @@ public String getString(String fullName, Charset encoding, JinjavaInterpreter in
         try {
             return resourceCache.get(new ResourceCacheKey(fullName, encoding));
         } catch (ExecutionException e) {
-            throw new IOException("Failed to get resource content from cache", e);
+            throw new IOException("Failed to get resource content named `" + fullName + "`from cache", e);
         }
     }
 }

transformation/transformationPlugins/jsonMessageTransformers/jsonJinjavaTransformer/src/main/java/org/opensearch/migrations/transform/jinjava/RegexReplaceException.java

@@ -28,4 +28,4 @@ public String getMessage() {
                 .add("replacement='" + replacement + "'")
                 .add("rewrittenReplacement='" + rewrittenReplacement + "'");
     }
-}
+}

transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/documentBackfillItems.j2

@@ -0,0 +1,21 @@
+{# see https://github.com/opensearch-project/opensearch-migrations/pull/1110 for the format of these messages #}
+{%- include "typeMappings/rewriteBulkRequest.j2" -%}
+{%- import "typeMappings/rewriteIndexForTarget.j2" as transidx -%}
+
+{%- set parameters = source_document.index -%}
+{{  log_value('ERROR', parameters) }}
+{{  log_value('WARN', source_document.source) }}
+
+{%- set type_name = parameters['_type'] -%}
+{%- if type_name -%}
+    {%- set target_index = transidx.convert_source_index_to_target(parameters['_index'], type_name, input_map.index_mappings, input_map.regex_index_mappings) if type_name -%}
+    {%- if target_index -%}
+        {
+          {{ rewrite_index_parameters(parameters, target_index) }},
+          "source": {{ source_document.source | tojson }}
+        }
+    {%- endif -%}
+{%- else -%}
+    {%- import "typeMappings/preserveAll.j2" as preserve -%}
+    {{- preserve.make_keep_json()  -}}
+{%- endif -%}

transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/main/resources/jinjava/typeMappings/rewriteBulkRequest.j2

@@ -19,9 +19,19 @@
 {%- macro get_index() -%}index{% endmacro %}
 {%- macro get_update() -%}update{% endmacro %}
 
+{%- macro rewrite_command_parameters(command, parameters, target_index) -%}
+    {%- if target_index -%}
+        "{{ invoke_macro("get_"+command) }}": {{ retarget_command_parameters(parameters, target_index) }}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro rewrite_index_parameters(parameters, target_index) -%}
+    {{ rewrite_command_parameters('index', parameters, target_index) }}
+{%- endmacro -%}
+
 {%- macro rewrite_command(command, parameters, target_index, doc) -%}
     {%- if target_index -%}
-        { "{{ invoke_macro("get_"+command) }}": {{ retarget_command_parameters(parameters, target_index) }} },
+        { {{ rewrite_command_parameters(command, parameters, target_index) }} },
         {{ doc | tojson }}
     {%- endif -%}
 {%- endmacro -%}

transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationDocBackfillTest.java

@@ -0,0 +1,43 @@
+package org.opensearch.migrations.transform;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+
+import org.opensearch.migrations.testutils.JsonNormalizer;
+
+import lombok.Lombok;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper;
+
+@Slf4j
+public class TypeMappingsSanitizationDocBackfillTest {
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @Test
+    public void test() throws Exception {
+        var testString = "{\n" +
+                "  \"index\": { \"_index\": \"performance\", \"_type\": \"network\", \"_id\": \"1\" },\n" +
+                "  \"source\": { \"field1\": \"value1\" }\n" +
+                "}";
+
+        var expectedString = "{\n" +
+            "  \"index\": { \"_index\": \"network\", \"_id\": \"1\" },\n" +
+            "  \"source\": { \"field1\": \"value1\" }\n" +
+            "}";
+
+
+        var regexIndexMappings = List.of(List.of(".*", "", ""));
+        var indexTypeMappingRewriter = new TypeMappingsSanitizationTransformer(null, regexIndexMappings);
+        var resultObj = indexTypeMappingRewriter.transformJson(OBJECT_MAPPER.readValue(testString, LinkedHashMap.class));
+        log.atInfo().setMessage("resultStr = {}").addArgument(() -> {
+            try {
+                return OBJECT_MAPPER.writeValueAsString(resultObj);
+            } catch (Exception e) {
+                throw Lombok.sneakyThrow(e);
+            }
+        }).log();
+        Assertions.assertEquals(JsonNormalizer.fromString(expectedString), JsonNormalizer.fromObject(resultObj));
+    }
+}

transformation/transformationPlugins/jsonMessageTransformers/jsonTypeMappingsSanitizationTransformer/src/test/java/org/opensearch/migrations/transform/TypeMappingsSanitizationTransformerBulkTest.java

@@ -36,7 +36,7 @@ static void initialize() throws IOException {
     }
 
     @Test
-    public void testBulk() throws Exception {
+    public void testBulkRequest() throws Exception {
         var testString =
             "{\n" +
                 "  \"" + JsonKeysForHttpMessage.METHOD_KEY + "\": \"PUT\",\n" +