[release-0.10] Cherry pick rsynctls toomanyfiles

custom-scorecard-tests/config.yaml

@@ -195,6 +195,16 @@ stages:
     storage:
       spec:
         mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_rsync_tls_normal_manyfiles.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:release-0.10
+    labels:
+      suite: volsync-e2e
+      test: test_rsync_tls_normal_manyfiles.yml
+    storage:
+      spec:
+        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_rsync_tls_priv.yml

custom-scorecard-tests/scorecard/patches/e2e-tests-stage1.yaml

@@ -181,6 +181,16 @@
     storage:
       spec:
         mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_rsync_tls_normal_manyfiles.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:release-0.10
+    labels:
+      suite: volsync-e2e
+      test: test_rsync_tls_normal_manyfiles.yml
+    storage:
+      spec:
+        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_rsync_tls_priv.yml

mover-rsync-tls/client.sh

@@ -149,15 +149,14 @@ while [[ $rc -ne 0 && $RETRY -lt $MAX_RETRIES ]]; do
       /diskrsync-tcp $BLOCK_SOURCE --source --target-address 127.0.0.1 --port $STUNNEL_LISTEN_PORT
       rc=$?
     else
-        shopt -s dotglob  # Make * include dotfiles
-        if [[ -n "$(ls -A -- ${SOURCE}/*)" ]]; then
+        ls -A "${SOURCE}"/ > /tmp/filelist.txt
+        if [[ -s /tmp/filelist.txt ]]; then
             # 1st run preserves as much as possible, but excludes the root directory
-            rsync -aAhHSxz --exclude=lost+found --itemize-changes --info=stats2,misc2 ${SOURCE}/* rsync://127.0.0.1:$STUNNEL_LISTEN_PORT/data
+            rsync -aAhHSxz -r --exclude=lost+found --itemize-changes --info=stats2,misc2 --files-from=/tmp/filelist.txt ${SOURCE}/ rsync://127.0.0.1:$STUNNEL_LISTEN_PORT/data
         else
             echo "Skipping sync of empty source directory"
         fi
         rc_a=$?
-        shopt -u dotglob  # Back to default * behavior
 
         # To delete extra files, must sync at the directory-level, but need to avoid
         # trying to modify the directory itself. This pass will only delete files

test-e2e/roles/write_to_pvc/tasks/main.yml

@@ -11,6 +11,10 @@
   loop_control:
     loop_var: var_check
 
+- name: determine number of files to create
+  ansible.builtin.set_fact:
+    create_file_count: "{{ file_count | default(1) }}"
+
 - name: Create Job
   kubernetes.core.k8s:
     state: present

test-e2e/roles/write_to_pvc/templates/job.yml.j2

@@ -24,6 +24,13 @@ spec:
               mkdir -p `dirname /mnt/{{ path }}`
               echo '{{ data }}' > '/mnt/{{ path }}'
               stat '/mnt/{{ path }}'
+
+              counter=1
+              while [ $counter -lt "{{ create_file_count }}" ]; do
+                echo '{{ data }}' > '/mnt/{{ path }}'-${counter}
+                counter=$((counter+1))
+              done
+
               sync
           securityContext:
             allowPrivilegeEscalation: false

test-e2e/test_rsync_tls_normal_manyfiles.yml

@@ -0,0 +1,252 @@
+---
+- hosts: localhost
+  tags:
+    - e2e
+    - rsync_tls
+    - manyfiles
+    - unprivileged
+    - volumepopulator
+  tasks:
+    - name: Create namespace
+      include_role:
+        name: create_namespace
+
+    - name: Probe cluster information
+      include_role:
+        name: gather_cluster_info
+
+    - name: Define podSecurityContext
+      ansible.builtin.set_fact:
+        podSecurityContext:
+          fsGroup: 5678
+          runAsGroup: 5678
+          runAsNonRoot: true
+          runAsUser: 1234
+          seccompProfile:
+            type: RuntimeDefault
+      when: not cluster_info.is_openshift
+
+    - name: Create ReplicationDestination (w/ mSC)
+      kubernetes.core.k8s:
+        state: present
+        definition:
+          apiVersion: volsync.backube/v1alpha1
+          kind: ReplicationDestination
+          metadata:
+            name: test
+            namespace: "{{ namespace }}"
+          spec:
+            rsyncTLS:
+              copyMethod: Snapshot
+              capacity: 1Gi
+              accessModes:
+                - ReadWriteOnce
+              moverSecurityContext: "{{ podSecurityContext }}"
+      when: podSecurityContext is defined
+
+    - name: Create ReplicationDestination (w/o mSC)
+      kubernetes.core.k8s:
+        state: present
+        definition:
+          apiVersion: volsync.backube/v1alpha1
+          kind: ReplicationDestination
+          metadata:
+            name: test
+            namespace: "{{ namespace }}"
+          spec:
+            rsyncTLS:
+              copyMethod: Snapshot
+              capacity: 1Gi
+              accessModes:
+                - ReadWriteOnce
+      when: podSecurityContext is not defined
+
+    - name: Create source PVC
+      kubernetes.core.k8s:
+        state: present
+        definition:
+          kind: PersistentVolumeClaim
+          apiVersion: v1
+          metadata:
+            name: data-source
+            namespace: "{{ namespace }}"
+          spec:
+            accessModes:
+              - ReadWriteOnce
+            resources:
+              requests:
+                storage: 1Gi
+
+    - name: Write data into the source PVC
+      include_role:
+        name: write_to_pvc
+      vars:
+        data: 'some-data'
+        path: '/.subdir/subdir2/testfile1'
+        file_count: 2
+        pvc_name: 'data-source'
+
+    - name: Write more data into the source PVC at different subdir
+      include_role:
+        name: write_to_pvc
+      vars:
+        data: 'some-more-data'
+        path: '/subdir2/ttestfilehere'
+        file_count: 2
+        pvc_name: 'data-source'
+
+    - name: Write more data into the source PVC
+      include_role:
+        name: write_to_pvc
+      vars:
+        data: 'even-more-data'
+        path: '/.hiddenfile'
+        file_count: 3
+        pvc_name: 'data-source'
+
+    - name: Write many files into the root of the source PVC
+      include_role:
+        name: write_to_pvc
+      vars:
+        data: 'data'
+        path: '/datafilelongname-thisnameisverylong-anditkeepsgoing-andgoing-andgoing-maybethisisenough'
+        file_count: 21000
+        pvc_name: 'data-source'
+
+    - name: Wait for key and address to be ready
+      kubernetes.core.k8s_info:
+        api_version: volsync.backube/v1alpha1
+        kind: ReplicationDestination
+        name: test
+        namespace: "{{ namespace }}"
+      register: res
+      until: >
+        res.resources | length > 0 and
+        res.resources[0].status.rsyncTLS is defined and
+        res.resources[0].status.rsyncTLS.keySecret is defined and
+        res.resources[0].status.rsyncTLS.address is defined
+      delay: 1
+      retries: 300
+
+    - name: Create ReplicationSource (w/ mSC)
+      kubernetes.core.k8s:
+        state: present
+        definition:
+          apiVersion: volsync.backube/v1alpha1
+          kind: ReplicationSource
+          metadata:
+            name: source
+            namespace: "{{ namespace }}"
+          spec:
+            sourcePVC: data-source
+            trigger:
+              schedule: "0 0 1 1 *"
+            rsyncTLS:
+              keySecret: "{{ res.resources[0].status.rsyncTLS.keySecret }}"
+              address: "{{ res.resources[0].status.rsyncTLS.address }}"
+              copyMethod: Snapshot
+              moverSecurityContext: "{{ podSecurityContext }}"
+      when: podSecurityContext is defined
+
+    - name: Create ReplicationSource (w/o mSC)
+      kubernetes.core.k8s:
+        state: present
+        definition:
+          apiVersion: volsync.backube/v1alpha1
+          kind: ReplicationSource
+          metadata:
+            name: source
+            namespace: "{{ namespace }}"
+          spec:
+            sourcePVC: data-source
+            trigger:
+              schedule: "0 0 1 1 *"
+            rsyncTLS:
+              keySecret: "{{ res.resources[0].status.rsyncTLS.keySecret }}"
+              address: "{{ res.resources[0].status.rsyncTLS.address }}"
+              copyMethod: Snapshot
+      when: podSecurityContext is not defined
+
+    - name: Check status of replicationsource
+      kubernetes.core.k8s_info:
+        api_version: volsync.backube/v1alpha1
+        kind: ReplicationSource
+        name: source
+        namespace: "{{ namespace }}"
+      register: res
+      until: >
+        res.resources | length > 0 and
+        res.resources[0].status.lastSyncDuration is defined and
+        res.resources[0].status.lastSyncTime is defined and
+        res.resources[0].status.latestMoverStatus is defined and
+        res.resources[0].status.latestMoverStatus.result == "Successful" and
+        res.resources[0].status.latestMoverStatus.logs is search("sent.*bytes.*received.*bytes.*") and
+        res.resources[0].status.latestMoverStatus.logs is search("rsync completed in.*")
+      delay: 1
+      retries: 900
+
+    - name: Wait for sync to complete
+      kubernetes.core.k8s_info:
+        api_version: volsync.backube/v1alpha1
+        kind: ReplicationDestination
+        name: test
+        namespace: "{{ namespace }}"
+      register: res
+      until: >
+        res.resources | length > 0 and
+        res.resources[0].status.latestImage is defined and
+        res.resources[0].status.latestImage.kind == "VolumeSnapshot" and
+        res.resources[0].status.latestMoverStatus is defined and
+        res.resources[0].status.latestMoverStatus.result == "Successful"
+      delay: 1
+      retries: 900
+
+    - name: Convert latestImage to PVC using VolumePopulator
+      kubernetes.core.k8s:
+        state: present
+        definition:
+          apiVersion: v1
+          kind: PersistentVolumeClaim
+          metadata:
+            name: data-dest
+            namespace: "{{ namespace }}"
+          spec:
+            accessModes:
+              - ReadWriteOnce
+            dataSourceRef:
+              kind: ReplicationDestination
+              apiGroup: volsync.backube
+              name: test
+            resources:
+              requests:
+                storage: 1Gi
+      when: cluster_info.volumepopulator_supported
+
+    - name: Convert latestImage to PVC
+      kubernetes.core.k8s:
+        state: present
+        definition:
+          apiVersion: v1
+          kind: PersistentVolumeClaim
+          metadata:
+            name: data-dest
+            namespace: "{{ namespace }}"
+          spec:
+            accessModes:
+              - ReadWriteOnce
+            dataSource:
+              kind: VolumeSnapshot
+              apiGroup: snapshot.storage.k8s.io
+              name: "{{ res.resources[0].status.latestImage.name }}"
+            resources:
+              requests:
+                storage: 1Gi
+      when: not cluster_info.volumepopulator_supported
+
+    - name: Verify contents of PVC
+      include_role:
+        name: compare_pvc_data
+      vars:
+        pvc1_name: data-source
+        pvc2_name: data-dest
+        timeout: 900