#!/bin/bash # # This file is part of the EESSI infrastructure, # see https://github.com/EESSI/infrastructure # # author: Bob Droege (@bedroge) # author: Terje Kvernes (@terjekv) # author: Thomas Roeblitz (@trz42) # # license: GPLv2 # function upload_to_staging_bucket { _file=$1 _bucket=$2 _path=$3 _endpoint_url=$4 _options= if [[ ! -z "${_endpoint_url}" ]]; then _options="--endpoint-url ${_endpoint_url}" fi aws ${_options} s3 cp "${_file}" s3://${_bucket}/${_path} } # This needs expanding etc. function check_file_name { filename=$1 if ( echo ${filename} | grep ^eessi > /dev/null && echo ${filename} | grep -E '(compat|init|software)' > /dev/null ); then return 0 else return 1 fi } function create_metadata_file { _artefact=$1 _url=$2 _repository=$3 _pull_request_number=$4 _pull_request_comment_id=$5 _tmpfile=$(mktemp) jq -n \ --arg un $(whoami) \ --arg ip $(curl -s https://checkip.amazonaws.com) \ --arg hn "$(hostname -f)" \ --arg fn "$(basename ${_artefact})" \ --arg sz "$(du -b "${_artefact}" | awk '{print $1}')" \ --arg ct "$(date -r "${_artefact}")" \ --arg sha256 "$(sha256sum "${_artefact}" | awk '{print $1}')" \ --arg url "${_url}" \ --arg repo "${_repository}" \ --arg pr "${_pull_request_number}" \ --arg pr_comment_id "${_pull_request_comment_id}" \ '{ uploader: {username: $un, ip: $ip, hostname: $hn}, payload: {filename: $fn, size: $sz, ctime: $ct, sha256sum: $sha256, url: $url}, link2pr: {repo: $repo, pr: $pr, pr_comment_id: $pr_comment_id}, }' > "${_tmpfile}" echo "${_tmpfile}" } function display_help { echo "Usage: $0 [OPTIONS] <filenames>" >&2 echo " -a | --artefact-prefix PREFIX - a directory to which the artefact" >&2 echo " shall be uploaded; BASH variable" >&2 echo " expansion will be applied; arg '-l'" >&2 echo " lists variables that are defined at" >&2 echo " the time of expansion" >&2 echo " -e | --endpoint-url URL - endpoint url (needed for non AWS S3)" >&2 echo " -h | --help - display this usage information" >&2 echo " -i | --pr-comment-id - identifier of a PR comment; may be" >&2 echo " used to efficiently determine the PR" >&2 echo " comment to be updated during the" >&2 echo " ingestion procedure" >&2 echo " -l | --list-variables - list variables that are available" >&2 echo " for expansion" >&2 echo " -k | --sign-key SCRIPT_KEY - specify location of the key to be" >&2 echo " used to sign artefacts and metadata" >&2 echo " files [optional; default: don't sign]" >&2 echo " -m | --metadata-prefix PREFIX - a directory to which the metadata" >&2 echo " file shall be uploaded; BASH variable" >&2 echo " expansion will be applied; arg '-l'" >&2 echo " lists variables that are defined at" >&2 echo " the time of expansion" >&2 echo " -n | --bucket-name BUCKET - bucket name (same as BUCKET above)" >&2 echo " -p | --pull-request-number INT - a pull request number (INT); used to" >&2 echo " link the upload to a PR" >&2 echo " -r | --repository FULL_NAME - a repository name ACCOUNT/REPONAME;" >&2 echo " used to link the upload to a PR" >&2 echo " -s | --sign-script SCRIPT_PATH - path to script that is used to sign" >&2 echo " artefacts and metadata files. The" >&2 echo " script is called with two arguments:" >&2 echo " KEY file_to_sign. The KEY is the one" >&2 echo " provided via option --sign-key. The" >&2 echo " latter is determined by this script." >&2 echo " [optional; default: don't sign]" >&2 } if [[ $# -lt 1 ]]; then display_help exit 1 fi # process command line args POSITIONAL_ARGS=() # depends on which service hosts the bucket # minio: https://MINIO_SERVER:MINIO_PORT/{bucket_name}/ # s3aws: https://{bucket_name}.s3.amazonaws.com/ # should be contructable from endpoint_url and bucket_name bucket_base= # default bucket is eessi-staging bucket_name="eessi-staging" # provided via options in the bot's config file app.cfg endpoint_url= # provided via command line arguments pr_comment_id="none" pull_request_number="none" github_repository="EESSI/software-layer" sign_key= sign_script= # provided via options in the bot's config file app.cfg and/or command line argument metadata_prefix= artefact_prefix= # other variables legacy_aws_path= variables="github_repository legacy_aws_path pull_request_number" while [[ $# -gt 0 ]]; do case $1 in -a|--artefact-prefix) artefact_prefix="$2" shift 2 ;; -e|--endpoint-url) endpoint_url="$2" shift 2 ;; -h|--help) display_help exit 0 ;; -l|--list-variables) echo "variables that will be expanded: name (default value)" for var in ${variables} do echo " ${var} (${!var:-unset})" done exit 0 ;; -i|--pr-comment-id) pr_comment_id="$2" shift 2 ;; -k|--sign-key) sign_key=$2 if [[ ! -r "${sign_key}" ]]; then echo "Error: SSH key '${sign_key}' to be used for signing doesn't exist or cannot be read" >&2 exit 1 fi shift 2 ;; -m|--metadata-prefix) metadata_prefix="$2" shift 2 ;; -n|--bucket-name) bucket_name="$2" shift 2 ;; -p|--pull-request-number) pull_request_number="$2" shift 2 ;; -r|--repository) github_repository="$2" shift 2 ;; -s|--sign-script) sign_script=$2 if [[ ! -x "${sign_script}" ]]; then echo "Error: Script '${sign_script}' to be used for signing doesn't exist or is not executable" >&2 exit 1 fi shift 2 ;; -*|--*) echo "Error: Unknown option: $1" >&2 exit 1 ;; *) # No more options POSITIONAL_ARGS+=("$1") # save positional arg shift ;; esac done # restore potentially parsed filename(s) into $* set -- "${POSITIONAL_ARGS[@]}" # ensure that either none or both of $sign_key and $sign_script are defined if [[ -n "${sign_key}" ]] && [[ -n "${sign_script}" ]]; then sign=1 elif [[ -n "${sign_key}" ]]; then sign=0 echo "Error: Signing requires a key (${sign_key}) AND a script (${sign_script}); likely the bot config is incomplete" >&2 exit 1 elif [[ -n "${sign_script}" ]]; then sign=0 echo "Error: Signing requires a key (${sign_key}) AND a script (${sign_script}); likely the bot config is incomplete" >&2 exit 1 else sign=0 fi # infer bucket_base: # if endpoint_url is not set (assume AWS S3 is used), # bucket_base=https://${bucket_name}.s3.amazonaws.com/ # if endpoint_url is set (assume non AWS S3, eg minio, is used), # bucket_base=${endpoint_url}/${bucket_name}/ # check if endpoint_url is not set if [[ -z "${endpoint_url}" ]]; then # assume AWS S3 being used bucket_base=https://${bucket_name}.s3.amazonaws.com else # assume non AWS S3 being used or AWS S3 with bucket not in DNS bucket_base=${endpoint_url}/${bucket_name} fi for file in "$*"; do if [[ -r "${file}" && -f "${file}" && -s "${file}" ]]; then basefile=$( basename ${file} ) if check_file_name ${basefile}; then if tar tf "${file}" | head -n1 > /dev/null; then # 'legacy_aws_path' might be used in artefact_prefix or metadata_prefix # its purpose is to support the old/legacy method to derive the location # where to store the artefact and metadata file export legacy_aws_path=$(basename ${file} | tr -s '-' '/' \ | perl -pe 's/^eessi.//;' | perl -pe 's/\.tar\.gz$//;' ) if [ -z ${artefact_prefix} ]; then aws_path=${legacy_aws_path} else export pull_request_number export github_repository aws_path=$(envsubst <<< "${artefact_prefix}") fi aws_file=$(basename ${file}) # 1st sign artefact, and upload signature if [[ "${sign}" = "1" ]]; then # sign artefact ${sign_script} sign ${sign_key} ${file} # TODO check if signing worked (just check exit code == 0) sig_file=${file}.sig aws_sig_file=${aws_file}.sig # uploading signature echo " store artefact signature at ${aws_path}/${aws_sig_file}" upload_to_staging_bucket \ "${sig_file}" \ "${bucket_name}" \ "${aws_path}/${aws_sig_file}" \ "${endpoint_url}" else echo "no signing method defined; not signing artefact" fi echo Uploading to "${url}" echo " store artefact at ${aws_path}/${aws_file}" upload_to_staging_bucket \ "${file}" \ "${bucket_name}" \ "${aws_path}/${aws_file}" \ "${endpoint_url}" echo "Creating metadata file" url="${bucket_base}/${aws_path}/${aws_file}" echo "create_metadata_file file=${file} \ url=${url} \ github_repository=${github_repository} \ pull_request_number=${pull_request_number} \ pr_comment_id=${pr_comment_id}" metadata_file=$(create_metadata_file "${file}" \ "${url}" \ "${github_repository}" \ "${pull_request_number}" \ "${pr_comment_id}") aws_metadata_file=${aws_file}.meta.txt # TODO check that creating the metadata file succeeded echo "metadata:" cat ${metadata_file} if [ -z ${metadata_prefix} ]; then aws_path=${legacy_aws_path} else export pull_request_number export github_repository aws_path=$(envsubst <<< "${metadata_prefix}") fi # 2nd sign metadata file, and upload signature if [[ "${sign}" = "1" ]]; then # sign metadata file ${sign_script} sign ${sign_key} ${metadata_file} # TODO check if signing worked (just check exit code == 0) sig_metadata_file=${metadata_file}.sig aws_sig_metadata_file=${aws_metadata_file}.sig echo " store metadata signature at ${aws_path}/${aws_sig_metadata_file}" upload_to_staging_bucket \ "${sig_metadata_file}" \ "${bucket_name}" \ "${aws_path}/${aws_sig_metadata_file}" \ "${endpoint_url}" else echo "no signing method defined; not signing metadata file" fi echo " store metadata file at ${aws_path}/${aws_file}.meta.txt" upload_to_staging_bucket \ "${metadata_file}" \ "${bucket_name}" \ "${aws_path}/${aws_file}.meta.txt" \ "${endpoint_url}" else echo "'${file}' is not a tar file." exit 1 fi else echo "${file} does not look like an eessi layer filename!" exit 1 fi else echo "'${file}' is not a readable non zero-sized file." exit 1 fi done