#!/usr/bin/env bash # # Storage for your qloudpulse data in your own AWS account: a KMS key, an S3 bucket, an RDS for # PostgreSQL instance, and the role qloudpulse uses them through. Everything is encrypted with the key, and deleting the # role or disabling the key ends qloudpulse's access. # # Run it in AWS CloudShell, signed in to the account that should hold the data, in the region it should be kept in. # The "Where your data is kept" page writes your company's values in below; this copy takes them as options instead: # # bash qloudpulse-storage.sh --role-arn --external-id --allow-from [,...] [--region eu-west-1] # # It is safe to run again: what exists keeps its data and gets its settings put again. The database takes about ten # minutes to create the first time. You pay AWS for what it makes: about $30 a month at the default size in eu-west-1. # # Options: # --region where the data is kept (default: CloudShell's region, else eu-west-1) # --instance-class the database size (default db.t4g.small; smaller is refused, reports would time out) # --vpc the VPC for the database (default: the region's default VPC) # --delete remove the role qloudpulse-storage, ending qloudpulse's access; your data stays # --delete --with-data also delete the database (a final snapshot is kept), and empty and delete the bucket # --yes answer --delete's question, for a run with nobody to ask set -euo pipefail # Your company's values, from the page that made this script. QLOUDPULSE_ROLE_ARN="${QLOUDPULSE_ROLE_ARN:-}" EXTERNAL_ID="${EXTERNAL_ID:-}" ALLOW_FROM="${ALLOW_FROM:-}" ROLE=qloudpulse-storage DB=qloudpulse-data KEY_ALIAS=alias/qloudpulse-data SECURITY_GROUP=qloudpulse-data CREATED_BY=qloudpulse-setup REGION="${AWS_REGION:-${AWS_DEFAULT_REGION:-eu-west-1}}" INSTANCE_CLASS=db.t4g.small VPC="" DELETE=false WITH_DATA=false YES=false usage() { awk 'NR > 1 && /^#/ { sub(/^# ?/, ""); print; next } NR > 1 { exit }' "$0" >&2; exit 1; } while [ "$#" -gt 0 ]; do case "$1" in --role-arn) QLOUDPULSE_ROLE_ARN="${2:-}"; shift ;; --external-id) EXTERNAL_ID="${2:-}"; shift ;; --allow-from) ALLOW_FROM="${2:-}"; shift ;; --region) REGION="${2:-}"; shift ;; --instance-class) INSTANCE_CLASS="${2:-}"; shift ;; --vpc) VPC="${2:-}"; shift ;; --delete) DELETE=true ;; --with-data) WITH_DATA=true ;; --yes) YES=true ;; -h|--help) usage ;; *) echo "Unknown option $1." >&2; usage ;; esac shift done if [ "$WITH_DATA" = true ] && [ "$DELETE" != true ]; then echo "--with-data goes with --delete." >&2 exit 1 fi if [[ ! "$REGION" =~ ^[a-z]{2}(-[a-z]+)+-[0-9]$ ]]; then echo "$REGION is not an AWS region." >&2 exit 1 fi export AWS_REGION="$REGION" AWS_DEFAULT_REGION="$REGION" ACCOUNT=$(aws sts get-caller-identity --query Account --output text) PARTITION=$(aws sts get-caller-identity --query Arn --output text | cut -d: -f2) BUCKET="qloudpulse-data-$ACCOUNT-$REGION" TAGS_JSON="[{\"Key\":\"CreatedBy\",\"Value\":\"$CREATED_BY\"}]" made_by_us_role() { [ "$(aws iam list-role-tags --role-name "$ROLE" --query "Tags[?Key=='CreatedBy'].Value | [0]" --output text 2>/dev/null)" = "$CREATED_BY" ] } made_by_us_bucket() { [ "$(aws s3api get-bucket-tagging --bucket "$BUCKET" --query "TagSet[?Key=='CreatedBy'].Value | [0]" --output text 2>/dev/null)" = "$CREATED_BY" ] } made_by_us_db() { local arn arn=$(aws rds describe-db-instances --db-instance-identifier "$DB" --query 'DBInstances[0].DBInstanceArn' --output text 2>/dev/null) || return 1 [ "$(aws rds list-tags-for-resource --resource-name "$arn" --query "TagList[?Key=='CreatedBy'].Value | [0]" --output text)" = "$CREATED_BY" ] } # --- Removing it again ------------------------------------------------------------------------------------------------- if [ "$DELETE" = true ]; then echo "This removes from account $ACCOUNT, region $REGION, what this script made, where it finds it:" echo " - the role $ROLE, which ends qloudpulse's access to your data" if [ "$WITH_DATA" = true ]; then echo " - the database $DB, keeping a final snapshot, and its security group and subnet group" echo " - the bucket $BUCKET with every file and every old version in it" echo "The KMS key $KEY_ALIAS stays, since the final snapshot needs it. Schedule its deletion yourself when you" echo "no longer want the snapshot: that makes every copy encrypted with it unreadable, for good." else echo "Your database, bucket and key stay, with your data in them. Add --with-data to delete them too." fi if [ "$YES" != true ]; then read -r -p "Type delete to go on: " ANSWER || ANSWER="" [ "$ANSWER" = delete ] || { echo "Nothing was removed."; exit 1; } fi if aws iam get-role --role-name "$ROLE" >/dev/null 2>&1; then if made_by_us_role; then for policy in $(aws iam list-role-policies --role-name "$ROLE" --query PolicyNames --output text); do aws iam delete-role-policy --role-name "$ROLE" --policy-name "$policy" done aws iam delete-role --role-name "$ROLE" echo "Deleted the role $ROLE." else echo "The role $ROLE was not made by this script: it stays." >&2 fi fi if [ "$WITH_DATA" = true ]; then if made_by_us_db; then SNAPSHOT="$DB-final-$(date -u +%Y%m%d-%H%M%S)" aws rds modify-db-instance --db-instance-identifier "$DB" --no-deletion-protection --apply-immediately >/dev/null aws rds delete-db-instance --db-instance-identifier "$DB" --final-db-snapshot-identifier "$SNAPSHOT" >/dev/null echo "Deleting the database $DB, with the final snapshot $SNAPSHOT. This takes a few minutes." aws rds wait db-instance-deleted --db-instance-identifier "$DB" aws rds delete-db-subnet-group --db-subnet-group-name "$DB" 2>/dev/null || true SG_ID=$(aws ec2 describe-security-groups --filters "Name=group-name,Values=$SECURITY_GROUP" "Name=tag:CreatedBy,Values=$CREATED_BY" \ --query 'SecurityGroups[0].GroupId' --output text) if [ "$SG_ID" != None ] && ! aws ec2 delete-security-group --group-id "$SG_ID" 2>/dev/null; then echo "The security group $SECURITY_GROUP is still in use for a moment: run this again to remove it." >&2 fi echo "Deleted the database $DB and its network settings." elif aws rds describe-db-instances --db-instance-identifier "$DB" >/dev/null 2>&1; then echo "The database $DB was not made by this script: it stays." >&2 fi if aws s3api head-bucket --bucket "$BUCKET" --expected-bucket-owner "$ACCOUNT" >/dev/null 2>&1; then if made_by_us_bucket; then echo "Emptying the bucket $BUCKET." while :; do BATCH=$(aws s3api list-object-versions --bucket "$BUCKET" --max-items 500 --output json \ --query '{Objects: [Versions[].{Key: Key, VersionId: VersionId}, DeleteMarkers[].{Key: Key, VersionId: VersionId}][] | [?Key != null]}') [ "$(echo "$BATCH" | jq '.Objects | length')" -gt 0 ] || break aws s3api delete-objects --bucket "$BUCKET" --delete "$(echo "$BATCH" | jq -c '. + {Quiet: true}')" >/dev/null done aws s3api delete-bucket --bucket "$BUCKET" echo "Deleted the bucket $BUCKET." else echo "The bucket $BUCKET was not made by this script: it stays." >&2 fi fi fi echo "Done. qloudpulse no longer reaches data in account $ACCOUNT." exit 0 fi # --- The values qloudpulse gives, checked before anything is made ------------------------------------------------------ if [[ ! "$QLOUDPULSE_ROLE_ARN" =~ ^arn:aws[a-z-]*:iam::[0-9]{12}:role/[A-Za-z0-9+=,.@_/-]+$ ]]; then echo "Give --role-arn: the role ARN on the 'Where your data is kept' page." >&2 exit 1 fi if [[ ! "$EXTERNAL_ID" =~ ^[A-Za-z0-9+=,.@:/_-]{8,}$ ]]; then echo "Give --external-id: your company's ExternalId from the same page." >&2 exit 1 fi CIDRS=() IFS=',' read -r -a CIDRS <<<"$ALLOW_FROM" if [ "${#CIDRS[@]}" -eq 0 ]; then echo "Give --allow-from: the addresses qloudpulse connects from, from the same page." >&2 exit 1 fi for cidr in "${CIDRS[@]}"; do if [[ ! "$cidr" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}/(2[4-9]|3[0-2])$ ]]; then echo "$cidr is not an address range of /24 or narrower; the database is opened to qloudpulse's addresses only." >&2 exit 1 fi done case "$INSTANCE_CLASS" in db.t*.micro|db.t*.nano) echo "$INSTANCE_CLASS is too small: reports would time out. Use db.t4g.small or larger." >&2; exit 1 ;; db.*) ;; *) echo "$INSTANCE_CLASS is not an RDS instance class." >&2; exit 1 ;; esac command -v jq >/dev/null || { echo "jq is needed; AWS CloudShell has it." >&2; exit 1; } echo "Setting up qloudpulse storage in account $ACCOUNT, region $REGION." # --- KMS key ----------------------------------------------------------------------------------------------------------- KEY_ID=$(aws kms describe-key --key-id "$KEY_ALIAS" --query KeyMetadata.KeyId --output text 2>/dev/null || true) if [ -z "$KEY_ID" ]; then KEY_ID=$(aws kms create-key --description "Encrypts the data qloudpulse keeps in this account" \ --tags TagKey=CreatedBy,TagValue="$CREATED_BY" --query KeyMetadata.KeyId --output text) aws kms create-alias --alias-name "$KEY_ALIAS" --target-key-id "$KEY_ID" echo "Created the KMS key $KEY_ALIAS." else echo "KMS key $KEY_ALIAS exists." fi aws kms enable-key-rotation --key-id "$KEY_ID" KEY_ARN=$(aws kms describe-key --key-id "$KEY_ID" --query KeyMetadata.Arn --output text) # --- S3 bucket --------------------------------------------------------------------------------------------------------- if aws s3api head-bucket --bucket "$BUCKET" --expected-bucket-owner "$ACCOUNT" >/dev/null 2>&1; then made_by_us_bucket || { echo "The bucket $BUCKET exists and was not made by this script; it is left alone." >&2; exit 1; } echo "Bucket $BUCKET exists." else if [ "$REGION" = us-east-1 ]; then aws s3api create-bucket --bucket "$BUCKET" --object-ownership BucketOwnerEnforced >/dev/null else aws s3api create-bucket --bucket "$BUCKET" --object-ownership BucketOwnerEnforced \ --create-bucket-configuration LocationConstraint="$REGION" >/dev/null fi echo "Created bucket $BUCKET." fi aws s3api put-bucket-tagging --bucket "$BUCKET" --tagging "{\"TagSet\": $TAGS_JSON}" aws s3api put-public-access-block --bucket "$BUCKET" --public-access-block-configuration \ BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true aws s3api put-bucket-encryption --bucket "$BUCKET" --server-side-encryption-configuration "{ \"Rules\": [{\"ApplyServerSideEncryptionByDefault\": {\"SSEAlgorithm\": \"aws:kms\", \"KMSMasterKeyID\": \"$KEY_ARN\"}, \"BucketKeyEnabled\": true}]}" aws s3api put-bucket-versioning --bucket "$BUCKET" --versioning-configuration Status=Enabled aws s3api put-bucket-lifecycle-configuration --bucket "$BUCKET" --lifecycle-configuration '{ "Rules": [{"ID": "qloudpulse-old-versions", "Status": "Enabled", "Filter": {}, "NoncurrentVersionExpiration": {"NoncurrentDays": 30}, "AbortIncompleteMultipartUpload": {"DaysAfterInitiation": 7}}]}' >/dev/null aws s3api put-bucket-policy --bucket "$BUCKET" --policy "{ \"Version\": \"2012-10-17\", \"Statement\": [{\"Sid\": \"TlsOnly\", \"Effect\": \"Deny\", \"Principal\": \"*\", \"Action\": \"s3:*\", \"Resource\": [\"arn:$PARTITION:s3:::$BUCKET\", \"arn:$PARTITION:s3:::$BUCKET/*\"], \"Condition\": {\"Bool\": {\"aws:SecureTransport\": \"false\"}}}]}" echo "Bucket $BUCKET: private, encrypted with $KEY_ALIAS, versioned, TLS only." # --- Network for the database: a security group open to qloudpulse's addresses only ------------------------------------ if [ -z "$VPC" ]; then VPC=$(aws ec2 describe-vpcs --filters Name=is-default,Values=true --query 'Vpcs[0].VpcId' --output text) if [ "$VPC" = None ]; then echo "Region $REGION has no default VPC: give --vpc with a VPC whose subnets reach an internet gateway." >&2 exit 1 fi fi SUBNETS=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC" --query 'Subnets[].SubnetId' --output text) ZONES=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC" --query 'Subnets[].AvailabilityZone' --output text \ | tr '\t' '\n' | sort -u | grep -c . || true) if [ "$ZONES" -lt 2 ]; then echo "VPC $VPC needs subnets in at least two availability zones for RDS." >&2 exit 1 fi SG_ID=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$VPC" "Name=group-name,Values=$SECURITY_GROUP" \ --query 'SecurityGroups[0].GroupId' --output text) if [ "$SG_ID" = None ]; then SG_ID=$(aws ec2 create-security-group --vpc-id "$VPC" --group-name "$SECURITY_GROUP" \ --description "qloudpulse reaches its database for this account from these addresses only" \ --tag-specifications "[{\"ResourceType\":\"security-group\",\"Tags\":$TAGS_JSON}]" --query GroupId --output text) echo "Created security group $SECURITY_GROUP." fi # Put the rules again: remove every 5432 rule, then add one per qloudpulse address. OLD_RULES=$(aws ec2 describe-security-group-rules --filters "Name=group-id,Values=$SG_ID" \ --query 'SecurityGroupRules[?!IsEgress && FromPort==`5432`].SecurityGroupRuleId' --output text) if [ -n "$OLD_RULES" ]; then # shellcheck disable=SC2086 # the ids are separate words aws ec2 revoke-security-group-ingress --group-id "$SG_ID" --security-group-rule-ids $OLD_RULES >/dev/null fi for cidr in "${CIDRS[@]}"; do aws ec2 authorize-security-group-ingress --group-id "$SG_ID" --ip-permissions \ "IpProtocol=tcp,FromPort=5432,ToPort=5432,IpRanges=[{CidrIp=$cidr,Description=qloudpulse}]" >/dev/null done echo "Security group $SECURITY_GROUP: Postgres open to ${CIDRS[*]} only." if aws rds describe-db-subnet-groups --db-subnet-group-name "$DB" >/dev/null 2>&1; then # shellcheck disable=SC2086 aws rds modify-db-subnet-group --db-subnet-group-name "$DB" --subnet-ids $SUBNETS >/dev/null else # shellcheck disable=SC2086 aws rds create-db-subnet-group --db-subnet-group-name "$DB" --subnet-ids $SUBNETS \ --db-subnet-group-description "Subnets for the qloudpulse data database" --tags "$TAGS_JSON" >/dev/null echo "Created DB subnet group $DB." fi # --- RDS for PostgreSQL ------------------------------------------------------------------------------------------------ if aws rds describe-db-instances --db-instance-identifier "$DB" >/dev/null 2>&1; then made_by_us_db || { echo "The database $DB exists and was not made by this script; it is left alone." >&2; exit 1; } aws rds modify-db-instance --db-instance-identifier "$DB" --vpc-security-group-ids "$SG_ID" \ --deletion-protection --backup-retention-period 7 --publicly-accessible --apply-immediately >/dev/null echo "Database $DB exists; its network, backups and deletion protection were put again." else aws rds create-db-instance --db-instance-identifier "$DB" \ --engine postgres --engine-version 17 --db-instance-class "$INSTANCE_CLASS" \ --storage-type gp3 --allocated-storage 20 --max-allocated-storage 100 \ --storage-encrypted --kms-key-id "$KEY_ARN" \ --master-username qloudpulse --manage-master-user-password --master-user-secret-kms-key-id "$KEY_ARN" \ --db-name qloudpulse --db-subnet-group-name "$DB" --vpc-security-group-ids "$SG_ID" --publicly-accessible \ --backup-retention-period 7 --deletion-protection --copy-tags-to-snapshot --auto-minor-version-upgrade \ --tags "$TAGS_JSON" >/dev/null echo "Creating the database $DB ($INSTANCE_CLASS). This takes about ten minutes." fi aws rds wait db-instance-available --db-instance-identifier "$DB" DB_ARN=$(aws rds describe-db-instances --db-instance-identifier "$DB" --query 'DBInstances[0].DBInstanceArn' --output text) ENDPOINT=$(aws rds describe-db-instances --db-instance-identifier "$DB" --query 'DBInstances[0].Endpoint.Address' --output text) SECRET_ARN=$(aws rds describe-db-instances --db-instance-identifier "$DB" --query 'DBInstances[0].MasterUserSecret.SecretArn' --output text) if [[ "$SECRET_ARN" != arn:* ]]; then echo "The database $DB has no password kept by RDS in Secrets Manager, which qloudpulse signs in with." >&2 echo "Turn it on: aws rds modify-db-instance --db-instance-identifier $DB --manage-master-user-password --master-user-secret-kms-key-id $KEY_ARN --apply-immediately" >&2 exit 1 fi # --- The role qloudpulse uses all of it through -------------------------------------------------------------------------- TRUST=$(cat </dev/null 2>&1; then made_by_us_role || { echo "The role $ROLE exists and was not made by this script; it is left alone." >&2; exit 1; } aws iam update-assume-role-policy --role-name "$ROLE" --policy-document "$TRUST" echo "Updated the trust policy of $ROLE." else aws iam create-role --role-name "$ROLE" --max-session-duration 3600 \ --description "Assumed by qloudpulse to keep this company's data in this account's bucket and database." \ --assume-role-policy-document "$TRUST" --tags "$TAGS_JSON" >/dev/null echo "Created the role $ROLE." fi aws iam put-role-policy --role-name "$ROLE" --policy-name "$ROLE-policy" --policy-document "$POLICY" echo echo "Done. On the 'Where your data is kept' page, enter:" echo " Account: $ACCOUNT" echo " Region: $REGION" echo "qloudpulse finds the rest itself: bucket $BUCKET, database $ENDPOINT, role $ROLE."