diff --git a/sdlf-cicd/deploy-generic.sh b/sdlf-cicd/deploy-generic.sh index 1b7898bf..c0f7415f 100755 --- a/sdlf-cicd/deploy-generic.sh +++ b/sdlf-cicd/deploy-generic.sh @@ -1,36 +1,156 @@ #!/bin/bash -pflag=false -DIRNAME=$(dirname "$0") +bold=$(tput bold) +underline=$(tput smul) +notunderline=$(tput rmul) +notbold=$(tput sgr0) + +die() { + printf '%s\n' "$1" >&2 + exit 1 +} + +version () { echo "awssdlf/2.7.0"; } usage () { echo " - -h -- Opens up this help message - -p -- Name of the AWS profile to use +Serverless Data Lake Framework (SDLF) is a collection of infrastructure-as-code artifacts to deploy data architectures on AWS. +This script creates a CodeBuild project with the set of permissions required to deploy the specified SDLF constructs. + +Usage: ./deploy-generic.sh [-V | --version] [-h | --help] [-p | --profile ] [-c sdlf-construct1 | --construct sdlf-construct1 ] [-c sdlf-construct...] + +Options + -V, --version -- Print the SDLF version + -h, --help -- Show this help message + -p -- Name of the AWS profile to use + -c -- Name of the SDLF construct that will be used + -- Name to uniquely identify this deployment + + ${underline}-c sdlf${notunderline} can be used as a shorthand for "all SDLF constructs" + ${underline}-c sdlf-stage${notunderline} can be used as a shorthand for "all SDLF Stage constructs" + +Examples + Create a CodeBuild project named sdlf-main able to deploy any SDLF construct: + ${bold}./deploy-generic.sh${notbold} ${underline}sdlf-main${notunderline} ${bold}-c${notbold} ${underline}sdlf${notunderline} + + Create a CodeBuild project named sdlf-data-team able to deploy technical catalogs, data processing workflows and consumption tools: + ${bold}./deploy-generic.sh${notbold} ${underline}sdlf-data-team${notunderline} ${bold}-c${notbold} ${underline}sdlf-dataset${notunderline} ${bold}-c${notbold} ${underline}sdlf-stage${notunderline} ${bold}-c${notbold} ${underline}sdlf-team${notunderline} + +More details and examples on https://sdlf.readthedocs.io/en/latest/constructs/cicd/ "; } -options=':p:h' -while getopts "$options" option + +pflag=false +rflag=false +cflag=false + +DIRNAME=$(dirname "$0") + +while : do - case "$option" in - p ) pflag=true; PROFILE=$OPTARG;; - h ) usage; exit;; - \? ) echo "Unknown option: -$OPTARG" >&2; exit 1;; - : ) echo "Missing option argument for -$OPTARG" >&2; exit 1;; - * ) echo "Unimplemented option: -$OPTARG" >&2; exit 1;; + case $1 in + -h|-\?|--help) + usage + exit + ;; + -V|--version) + version + exit + ;; + -p|--profile) + if [ "$2" ]; then + pflag=true; + PROFILE=$2 + shift + else + die 'ERROR: "--profile" requires a non-empty option argument.' + fi + ;; + --profile=?*) + pflag=true; + PROFILE=${1#*=} # delete everything up to "=" and assign the remainder + ;; + --profile=) # handle the case of an empty --profile= + die 'ERROR: "--profile" requires a non-empty option argument.' + ;; + -r|--region) + if [ "$2" ]; then + rflag=true; + REGION=$2 + shift + else + die 'ERROR: "--region" requires a non-empty option argument.' + fi + ;; + --region=?*) + rflag=true; + REGION=${1#*=} # delete everything up to "=" and assign the remainder + ;; + --region=) # handle the case of an empty --region= + die 'ERROR: "--region" requires a non-empty option argument.' + ;; + -c|--construct) + if [ "$2" ]; then + CONSTRUCTS+=("$2") + shift + else + die 'ERROR: "--construct" requires a non-empty option argument.' + fi + ;; + --construct=?*) + CONSTRUCTS+=("${1#*=}") # delete everything up to "=" and assign the remainder + ;; + --construct=) # handle the case of an empty --profile= + die 'ERROR: "--construct" requires a non-empty option argument.' + ;; + --) # end of all options + shift + break + ;; + -?*) + printf 'WARN: Unknown option (ignored): %s\n' "$1" >&2 + ;; + *) # default case: no more options, so break out of the loop + break esac + + shift done -if ! "$pflag" +if [ -z ${1+x} ]; then die 'ERROR: "./deploy-generic.sh" requires a non-option argument.'; fi + +if "$pflag" then - echo "-p not specified, using default..." >&2 - PROFILE="default" + echo "using AWS profile $PROFILE..." >&2 fi -REGION=$(aws configure get region --profile "$PROFILE") +if "$rflag" +then + echo "using AWS region $REGION..." >&2 +fi + +STACK_NAME="sdlf-cicd-$1" +DEPLOY_CODEBUILD_BOOTSTRAP=true +# it is not expected to have that many of these stacks created, so pagination is not handled for now. +stacks=$(aws cloudformation list-stacks --stack-status-filter CREATE_COMPLETE UPDATE_COMPLETE UPDATE_ROLLBACK_COMPLETE --query "StackSummaries[?starts_with(StackName, 'sdlf-cicd-')].StackName" --output text) +for stack in $stacks +do + if aws cloudformation describe-stack-resource --stack-name "$stack" --logical-resource-id rSdlfBootstrapCodeBuildProject > /dev/null 2>&1 + then + if [ "$stack" != "$STACK_NAME" ] + then + DEPLOY_CODEBUILD_BOOTSTRAP=false + fi + echo "SDLF CodeBuild bootstrap project found in stack $stack." + break + fi +done -STACK_NAME="sdlf-cicd" +echo "CloudFormation stack name: $STACK_NAME" aws cloudformation deploy \ --stack-name "$STACK_NAME" \ --template-file "$DIRNAME"/template-cicd-generic-git.yaml \ + --parameter-overrides \ + pCodebuildBootstrap="$DEPLOY_CODEBUILD_BOOTSTRAP" \ + pCodeBuildSuffix="$1" \ --tags Framework=sdlf \ --capabilities "CAPABILITY_NAMED_IAM" "CAPABILITY_AUTO_EXPAND" \ - --region "$REGION" \ - --profile "$PROFILE" || exit 1 + ${REGION:+--region "$REGION"} \ + ${PROFILE:+--profile "$PROFILE"} || exit 1 diff --git a/sdlf-cicd/template-cicd-generic-git.yaml b/sdlf-cicd/template-cicd-generic-git.yaml index 4ecce9ec..98326fef 100644 --- a/sdlf-cicd/template-cicd-generic-git.yaml +++ b/sdlf-cicd/template-cicd-generic-git.yaml @@ -6,9 +6,18 @@ Parameters: Description: S3 Bucket Prefix if different from default. Must be a valid S3 Bucket name Type: String Default: sdlf + pCodebuildBootstrap: + Description: Deploy the sdlf-cicd-bootstrap CodeBuild project + Type: String + Default: false + pCodeBuildSuffix: + Description: Suffix appended to the CodeBuild project name + Type: String + AllowedPattern: '[A-Za-z0-9\-_]{1,139}' Conditions: UseCustomBucketPrefix: !Not [!Equals [!Ref pCustomBucketPrefix, sdlf]] + DeployCodeBuildBootstrap: !Equals [!Ref pCodebuildBootstrap, true] Resources: rKMSKey: @@ -87,6 +96,7 @@ Resources: rSdlfBootstrapCodeBuildServiceRole: Type: AWS::IAM::Role + Condition: DeployCodeBuildBootstrap Properties: AssumeRolePolicyDocument: Version: "2012-10-17" @@ -165,6 +175,7 @@ Resources: rSdlfBootstrapCodeBuildProject: Type: AWS::CodeBuild::Project + Condition: DeployCodeBuildBootstrap Properties: Name: sdlf-cicd-bootstrap Artifacts: @@ -190,15 +201,6 @@ Resources: aws --version # version 2 build: commands: - - echo Configuring AWS credentials - - echo "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI $AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" - - curl -qL -o aws_credentials.json http://169.254.170.2/$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI > aws_credentials.json - - |- - aws configure set aws_access_key_id `jq -r '.AccessKeyId' aws_credentials.json` - aws configure set aws_secret_access_key `jq -r '.SecretAccessKey' aws_credentials.json` - aws configure set aws_session_token `jq -r '.Token' aws_credentials.json` - aws configure set default.region "$AWS_REGION" - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) - |- # section to remove and replace with CodeBuild source configuration git clone https://github.com/awslabs/aws-serverless-data-lake-framework.git @@ -222,6 +224,7 @@ Resources: --as-target - |- cp sdlf-cicd/manifest-*.yaml . + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) sed -i -e "s/111111111111/$AWS_ACCOUNT_ID/g" -e "s/eu-west-1/$AWS_REGION/g" manifest-sdlf.yaml seedfarmer apply manifest-sdlf.yaml artifacts: @@ -294,7 +297,7 @@ Resources: rDatalakeCodeBuildProject: Type: AWS::CodeBuild::Project Properties: - Name: sdlf-cicd-datalake + Name: !Sub sdlf-cicd-${pCodeBuildSuffix} Artifacts: Type: NO_ARTIFACTS EncryptionKey: !GetAtt rKMSKey.Arn