```bash #!/bin/bash # # Description : This script takes a single file that lists loadable sources on s3 and exports # the compiled text files to the directory in which it is invoked, preserving # the scope/user_id/imei directory structure # # WARNING : DO NOT USE FOR LARGE EXPORTS (ALTHOUGH YOU MAY WISH TO PARALLELISE LARGE EXPORTS # USING THIS SCRIPT, I.E. SPLIT THE FILE LISTING LOADABLE SOURCES INTO MULTIPLE # SMALLER FILES) set -e set -o pipefail usage() { echo "usage: aws-export.sh [ -f LOADABLE_SOURCE_FILE ] [ -o OUTPUT_DIR ]" } while getopts f:o: option do case $option in f) LOADABLE_SRC=${OPTARG};; o) OUTPUT_DIR=${OPTARG};; esac done if [ -z "$LOADABLE_SRC" ]; then echo "########################################" echo "Exporting compiled text files from AWS" echo "########################################" echo "Date : $(date)" echo "Host : $(hostname)" echo "########################################" echo "" echo "" echo "You have not specified a file that lists compiled text with the '-f' flag." echo "Please enter the path to such a file : " read -r LOADABLE_SRC fi if [ -z "$OUTPUT_DIR" ]; then OUTPUT_DIR='./' echo "########################################" echo "Exporting files to current directory" echo "########################################" fi # Set timer to zero SECONDS=0 while read -r COMPILED_TXT; do echo "Copying file $COMPILED_TXT" SUB_DIR=$(echo "$COMPILED_TXT" | awk '{split($0, path, "/"); print path[4]"/"path[5]"/"path[6]"/"}') echo "Target is : $OUTPUT_DIR/$SUB_DIR" aws s3 cp "$COMPILED_TXT" "$OUTPUT_DIR/$SUB_DIR" 2>/dev/null ; \ echo "########################################" done < "$LOADABLE_SRC" DURATION=$SECONDS echo "" echo "" echo "Export took : $DURATION seconds" echo "Files exported to : $OUTPUT_DIR" echo "########################################" ```