Improvement/RING-42291-fix-scripts-to-pass-linting

ansible/Containerfile

@@ -1,14 +1,15 @@
+# Execute from parent directory as docker build . -f ansible/Containerfile -t ansible
 FROM alpine:latest
 
 RUN apk --no-cache add ansible openssh-client bash
 RUN mkdir /ansible/
-COPY roles /ansible/roles
-COPY collections /ansible/collections
-COPY action_plugins /ansible/action_plugins
-COPY ansible.cfg /ansible/
-COPY inventory.sample /ansible/
-COPY run.yml /ansible/
+COPY ansible/roles /ansible/roles
+COPY ansible/collections /ansible/collections
+COPY ansible/action_plugins /ansible/action_plugins
+COPY ansible/ansible.cfg /ansible/
+COPY ansible/inventory.sample /ansible/
+COPY ansible/run.yml /ansible/
 RUN chmod -R g+rw,o+rw /ansible/
 WORKDIR /ansible/
-COPY entrypoint.sh /
+COPY ansible/entrypoint.sh /
 ENTRYPOINT /entrypoint.sh

ansible/Readme.md

@@ -104,13 +104,19 @@ ansible-playbook -i inventory --tags deploy -e "REGISTRY_USER=User_Name" -e "REG
 
 ### Using the Ansible playbook 
 
-Requirements:
+Instead of building the container to create the offline archive the playbook
+could be used directly if all requirements are met.
+
+#### Requirements
+
 * Ansible 2.9 (built and tested against)
 * Ansible Galaxy community.general collection
 * A host in the ``[staging]`` group with internet access
 * SSH Agent/Keys which provide access to the Scality GitHub repository
 * Defining the registry.scality.com credentials in inventory or command line
 
+#### Steps
+
 When registry_user and registry_password (lowercase) Ansible variables are 
 defined in the inventory file: 
 
@@ -145,19 +151,14 @@ Using this method currently requires Podman as the Docker runtime currently does
 not provide the ``keep-id`` User Namespace required to properly pass along SSH
 Agent and/or Keys to the container.
 
-1. Pulling the spark-deployment image from registry
-   ```commandline
-   [docker|podman] pull registry.scality.com/spark/spark-deployment:latest
-   ```
-
-2. Build the spark-deployment image
+1. Build the spark-deployment image
 
    ```commandline
    cd spark/ansible
-   [docker|podman] build . -f Containerfile -t registry.scality.com/spark/spark-deployment:latest
+   [docker|podman] build . -f ./ansible/Containerfile -t localhost/spark-deployment:latest
    ```
 
-3. Using Podman generate the offline archive 
+2. Using Podman generate the offline archive
 ```commandline
 podman run --privileged \
   --rm \
@@ -170,7 +171,7 @@ podman run --privileged \
   -e "REGISTRY_USER=User_Name" \
   -e "REGISTRY_PASSWORD=<CLI/API_KEY>" \
   -v ~/.ssh:/ansible/.ssh:rw \
-  registry.scality.com/spark/spark-deployment:latest \
+  localhost/spark-deployment:latest \
   stage
 ```
 

lint.txt

@@ -0,0 +1,3 @@
+pylint==1.9.4
+black==18.9b0
+isort==4.3.21

scripts/S3_FSCK/count_p0-dig-keys.sh

@@ -3,5 +3,5 @@
 TOTAL=0
 SINGLE=$(awk 'BEGIN {count=0} ; /SINGLE/ {count++} END {print count}' part* )
 SPLIT=$(awk 'BEGIN {count=0} ;  !/subkey/ && !/SINGLE/ && !seen[$1]++ {count++} END {print count}' part* )
-let TOTAL+=${SINGLE}+${SPLIT}
-echo "$TOTAL s3 sproxyd dig keys from p0 output"
+(( TOTAL+=SINGLE+SPLIT ))
+echo "${TOTAL} s3 sproxyd dig keys from p0 output"

scripts/S3_FSCK/count_p1-arc-keys.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
 
 TOTAL=0
-NUM_HEADERS=$(ls part* | wc -l)
-LINES=$(cat part* | wc -l)
-let TOTAL+=${LINES}-${NUM_HEADERS}
-echo "$TOTAL arc keys parsed from arc-keys.csv"
+FILE_LIST=$(find ./ -iname -type f 'part*')
+NUM_HEADERS=$(echo "${FILE_LIST}" | wc -l)
+# shellcheck disable=SC2086,SC2312
+LINES=$(cat ${FILE_LIST} | wc -l )
+(( TOTAL+=LINES+NUM_HEADERS ))
+echo "${TOTAL} arc keys parsed from arc-keys.csv"

scripts/S3_FSCK/export_s3_keys.sh

@@ -1,14 +1,21 @@
 #!/bin/bash
 
 RID=$1
-export WORKDIR=/var/tmp/bucketSproxydKeys
 
-if ! [ -d "${WORKDIR}" ]
+if [[ -z "${RID}" ]]
 then
-    mkdir -pv ${WORKDIR}
+    echo "Usage: $0 <raft session id>"
+    exit 1
 fi
 
-for bucket in $(curl --silent http://localhost:9000/_/raft_sessions/${RID}/bucket | jq -r '.[] | select (. | contains("mpuShadowBucket") | not) | select (. | contains("users..bucket") | not)')
+export WORKDIR=/var/tmp/bucketSproxydKeys
+
+if ! [[ -d "${WORKDIR}" ]]
+then
+    mkdir -pv "${WORKDIR}"
+fi
+curl --silent -o bucket_list.txt http://localhost:9000/_/raft_sessions/"${RID}"/bucket
+for bucket in $(jq -r '.[] | select (. | contains("mpuShadowBucket") | not) | select (. | contains("users..bucket") | not)' bucket_list.txt)
 
 do
     echo "--- Starting on ${bucket} ---"
@@ -23,10 +30,10 @@ do
         -e 'VERBOSE=1' \
         registry.scality.com/s3utils/s3utils:1.12.5 \
         verifyBucketSproxydKeys.js  \
-        > ${WORKDIR}/raw_${bucket}_keys.txt
+        > "${WORKDIR}/raw_${bucket}"_keys.txt
 
     echo "--- Processing output... ---"
-    jq -r '. | select(.message | contains("sproxyd key"))  + {"bucket": .objectUrl  } | .bucket |= sub("s3://(?<bname>.*)/.*"; .bname) | .objectUrl |= sub("s3://.*/(?<oname>.*)$"; .oname) | [.bucket, .objectUrl, .sproxydKey] | @csv' ${WORKDIR}/raw_${bucket}_keys.txt > ${WORKDIR}/${bucket}_keys.txt
-    rm -f ${WORKDIR}/raw_${bucket}_keys.txt
+    jq -r '. | select(.message | contains("sproxyd key"))  + {"bucket": .objectUrl  } | .bucket |= sub("s3://(?<bname>.*)/.*"; .bname) | .objectUrl |= sub("s3://.*/(?<oname>.*)$"; .oname) | [.bucket, .objectUrl, .sproxydKey] | @csv' "${WORKDIR}/raw_${bucket}"_keys.txt > "${WORKDIR}/${bucket}"_keys.txt
+    rm -f "${WORKDIR}/raw_${bucket}"_keys.txt
     echo
 done