Merge pull request #383 from simonsystem/add-ability-to-fetch-only-on…

…e-resource

#minor feat: add ability to fetch only specific resources by name

.github/workflows/build_and_test.yaml

@@ -94,6 +94,7 @@ jobs:
           wait_for_pod_ready "sidecar-5xx"
           wait_for_pod_ready "sidecar-pythonscript"
           wait_for_pod_ready "sidecar-pythonscript-logfile"
+          wait_for_pod_ready "sidecar-pythonscript-resource-name"
           wait_for_pod_ready "sidecar-logtofile-pythonscript"
           wait_for_pod_ready "dummy-server-pod"
 
@@ -114,6 +115,16 @@ jobs:
               wait_for_pod_log $p $r
             done
           done
+
+          # monitor only specified resources for sidecar-pythonscript-resource-name
+          pods=("sidecar-pythonscript-resource-name")
+          resources=("sample-configmap" "sample-secret-binary")
+          for p in ${pods[*]}; do
+            for r in ${resources[*]}; do
+              wait_for_pod_log $p $r
+            done
+          done
+
           # 10 more seconds after the last thing appeared in the logs.
           sleep 10
       - name: Retrieve pod logs
@@ -124,6 +135,7 @@ jobs:
           kubectl logs sidecar-5xx > /tmp/logs/sidecar-5xx.log
           kubectl logs sidecar-pythonscript > /tmp/logs/sidecar-pythonscript.log
           kubectl logs sidecar-pythonscript-logfile > /tmp/logs/sidecar-pythonscript-logfile.log
+          kubectl logs sidecar-pythonscript-resource-name > /tmp/logs/sidecar-pythonscript-resource-name.log
           kubectl logs dummy-server-pod > /tmp/logs/dummy-server.log
       - name: Upload artifacts (pod logs)
         uses: actions/upload-artifact@v4
@@ -232,6 +244,7 @@ jobs:
           # Total is (9 + 7)
           test $(cat /tmp/logs/sidecar-pythonscript.log | grep "Hello from python script!" | wc -l) = "9" &&
           test $(cat /tmp/logs/sidecar-pythonscript-logfile.log | grep "Hello from python script!" | wc -l) = "9" &&
+          test $(cat /tmp/logs/sidecar-pythonscript-resource-name.log | grep "Hello from python script!" | wc -l) = "2" &&
           kubectl exec sidecar-logtofile-pythonscript -- sh -c "test -e /opt/logs/sidecar.log" &&
           test $(kubectl exec sidecar-logtofile-pythonscript -- sh -c 'cat /opt/logs/sidecar.log | grep "Hello from python script!" | wc -l') = "16"
       - name: Verify sidecar files after update

README.md

@@ -71,6 +71,7 @@ If the filename ends with `.url` suffix, the content will be processed as a URL
 | `FOLDER_ANNOTATION`        | The annotation the sidecar will look for in configmaps to override the destination folder for files. The annotation _value_ can be either an absolute or a relative path. Relative paths will be relative to `FOLDER`.                                                                                                              | false    | `k8s-sidecar-target-directory`            | string  |
 | `NAMESPACE`                | Comma separated list of namespaces. If specified, the sidecar will search for config-maps inside these namespaces. It's also possible to specify `ALL` to search in all namespaces.                                                                                                                                                 | false    | namespace in which the sidecar is running | string  |
 | `RESOURCE`                 | Resource type, which is monitored by the sidecar. Options: `configmap`, `secret`, `both`                                                                                                                                                                                                                                            | false    | `configmap`                               | string  |
+| `RESOURCE_NAME`            | Comma separated list of resource names, which are monitored by the sidecar. Items can be prefixed by the namespace and the resource type. E.g. `secret/resource-name` or `namespace/secret/resource-name`. Setting this will result `method` set to `WATCH` being treated as `SLEEP`                                             | false    | -                                         | string  |
 | `METHOD`                   | If `METHOD` is set to `LIST`, the sidecar will just list config-maps/secrets and exit. With `SLEEP` it will list all config-maps/secrets, then sleep for `SLEEP_TIME` seconds. Anything else will continuously watch for changes (see https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes). | false    | -                                         | string  |
 | `SLEEP_TIME`               | How many seconds to wait before updating config-maps/secrets when using `SLEEP` method.                                                                                                                                                                                                                                             | false    | `60`                                      | integer |
 | `REQ_URL`                  | URL to which send a request after a configmap/secret got reloaded                                                                                                                                                                                                                                                                   | false    | -                                         | URI     |

src/resources.py

@@ -31,6 +31,11 @@
     RESOURCE_CONFIGMAP: "list_config_map_for_all_namespaces"
 }})
 
+_read_namespace = {
+    RESOURCE_SECRET: "read_namespaced_secret",
+    RESOURCE_CONFIGMAP: "read_namespaced_config_map"
+}
+
 _resources_version_map = {
     RESOURCE_SECRET: {},
     RESOURCE_CONFIGMAP: {},
@@ -98,26 +103,48 @@ def _get_destination_folder(metadata, default_folder, folder_annotation):
 
 def list_resources(label, label_value, target_folder, request_url, request_method, request_payload,
                    namespace, folder_annotation, resource, unique_filenames, script, enable_5xx,
-                   ignore_already_processed):
+                   ignore_already_processed, resource_name):
     v1 = client.CoreV1Api()
-    # Filter resources based on label and value or just label
-    label_selector = f"{label}={label_value}" if label_value else label
 
-    additional_args = {
-        'label_selector': label_selector
-    }
+    additional_args = {}
+
     if namespace != "ALL":
         additional_args['namespace'] = namespace
 
     logger.info(f"Performing list-based sync on {resource} resources: {additional_args}")
 
-    ret = getattr(v1, _list_namespace[namespace][resource])(**additional_args)
+    resource_names = []
+
+    if namespace != "ALL" and resource_name:
+        for rn in resource_name.split(","):
+            splitted_rn = list(reversed(rn.split("/")))
+            if len(splitted_rn) == 3 and splitted_rn[2] != namespace:
+                continue
+            if len(splitted_rn) == 2 and splitted_rn[1] != resource:
+                continue
+            resource_names.append(splitted_rn[0])
+
+    if namespace != "ALL" and resource_names:
+        items = []
+        for rn in resource_names:
+            additional_args['name'] = rn
+            try:
+                ret = getattr(v1, _read_namespace[resource])(**additional_args)
+                items.append(ret)
+            except ApiException as e:
+                if e.status != 404:
+                    raise e
+
+    else:
+        additional_args['label_selector'] = f"{label}={label_value}" if label_value else label
+        ret = getattr(v1, _list_namespace[namespace][resource])(**additional_args)
+        items = ret.items
 
     files_changed = False
     exist_keys = set()
 
     # For all the found resources
-    for item in ret.items:
+    for item in items:
         metadata = item.metadata
         exist_keys.add(metadata.namespace + metadata.name)
 
@@ -362,14 +389,20 @@ def _watch_resource_iterator(label, label_value, target_folder, request_url, req
             request(request_url, request_method, enable_5xx, request_payload)
 
 
-def _watch_resource_loop(mode, *args):
+def _watch_resource_loop(mode, label, label_value, target_folder, request_url, request_method, request_payload,
+                         namespace, folder_annotation, resource, unique_filenames, script, enable_5xx,
+                         ignore_already_processed, resource_name):
     while True:
         try:
-            if mode == "SLEEP":
-                list_resources(*args)
+            if mode == "SLEEP" or (namespace != 'ALL' and resource_name):
+                list_resources(label, label_value, target_folder, request_url, request_method, request_payload,
+                               namespace, folder_annotation, resource, unique_filenames, script, enable_5xx,
+                               ignore_already_processed, resource_name)
                 sleep(int(os.getenv("SLEEP_TIME", 60)))
             else:
-                _watch_resource_iterator(*args)
+                _watch_resource_iterator(label, label_value, target_folder, request_url, request_method, request_payload,
+                                         namespace, folder_annotation, resource, unique_filenames, script, enable_5xx,
+                                         ignore_already_processed)
         except ApiException as e:
             if e.status != 500:
                 logger.error(f"ApiException when calling kubernetes: {e}\n")
@@ -390,11 +423,11 @@ def _watch_resource_loop(mode, *args):
 
 def watch_for_changes(mode, label, label_value, target_folder, request_url, request_method, request_payload,
                       current_namespace, folder_annotation, resources, unique_filenames, script, enable_5xx,
-                      ignore_already_processed):
+                      ignore_already_processed, resource_name):
     processes = _start_watcher_processes(current_namespace, folder_annotation, label,
                                          label_value, request_method, mode, request_payload, resources,
                                          target_folder, unique_filenames, script, request_url, enable_5xx,
-                                         ignore_already_processed)
+                                         ignore_already_processed, resource_name)
 
     while True:
         died = False
@@ -414,14 +447,14 @@ def watch_for_changes(mode, label, label_value, target_folder, request_url, requ
 
 def _start_watcher_processes(namespace, folder_annotation, label, label_value, request_method,
                              mode, request_payload, resources, target_folder, unique_filenames, script, request_url,
-                             enable_5xx, ignore_already_processed):
+                             enable_5xx, ignore_already_processed, resource_name):
     processes = []
     for resource in resources:
         for ns in namespace.split(','):
             proc = Process(target=_watch_resource_loop,
                            args=(mode, label, label_value, target_folder, request_url, request_method, request_payload,
                                  ns, folder_annotation, resource, unique_filenames, script, enable_5xx,
-                                 ignore_already_processed)
+                                 ignore_already_processed, resource_name)
                            )
             proc.daemon = True
             proc.start()

src/sidecar.py

@@ -21,6 +21,7 @@
 LABEL = "LABEL"
 LABEL_VALUE = "LABEL_VALUE"
 RESOURCE = "RESOURCE"
+RESOURCE_NAME = "RESOURCE_NAME"
 REQ_PAYLOAD = "REQ_PAYLOAD"
 REQ_URL = "REQ_URL"
 REQ_METHOD = "REQ_METHOD"
@@ -70,6 +71,9 @@ def main():
     resources = ("secret", "configmap") if resources == "both" else (resources,)
     logger.debug(f"Selected resource type: {resources}")
 
+    resource_name = os.getenv(RESOURCE_NAME, "")
+    logger.debug(f"Selected resource name: {resource_name}")
+
     request_method = os.getenv(REQ_METHOD)
     request_url = os.getenv(REQ_URL)
 
@@ -127,11 +131,11 @@ def main():
             for ns in namespace.split(','):
                 list_resources(label, label_value, target_folder, request_url, request_method, request_payload,
                                ns, folder_annotation, res, unique_filenames, script, enable_5xx,
-                               ignore_already_processed)
+                               ignore_already_processed, resource_name)
     else:
         watch_for_changes(method, label, label_value, target_folder, request_url, request_method, request_payload,
                           namespace, folder_annotation, resources, unique_filenames, script, enable_5xx,
-                          ignore_already_processed)
+                          ignore_already_processed, resource_name)
 
 
 def _initialize_kubeclient_configuration():

test/resources/sidecar.yaml

@@ -260,6 +260,44 @@ spec:
         name: logger-config
         defaultMode: 0777
 
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sidecar-pythonscript-resource-name
+  namespace: default
+spec:
+  serviceAccountName: sample-acc
+  containers:
+    - name: sidecar
+      image: kiwigrid/k8s-sidecar:testing
+      volumeMounts:
+        - name: shared-volume
+          mountPath: /tmp/
+        - name: script-volume
+          mountPath: /opt/script.py
+          subPath: script.py
+      env:
+        - name: LABEL
+          value: "findme"
+        - name: FOLDER
+          value: /tmp/
+        - name: RESOURCE
+          value: both
+        - name: RESOURCE_NAME
+          value: configmap/sample-configmap,secret/sample-secret-binary
+        - name: SCRIPT
+          value: "/opt/script.py"
+        - name: LOG_LEVEL
+          value: "DEBUG"
+  volumes:
+    - name: shared-volume
+      emptyDir: { }
+    - name: script-volume
+      configMap:
+        name: script-configmap
+        defaultMode: 0777
+
 ---
 apiVersion: v1
 kind: Pod
@@ -388,4 +426,4 @@ metadata:
 type: Opaque
 stringData:
   username: "user1"
-  password: "abcdefghijklmnopqrstuvwxyz"
+  password: "abcdefghijklmnopqrstuvwxyz"