eessi-upload-to-staging

#!/bin/bash
#
# This file is part of the EESSI infrastructure,
# see https://github.com/EESSI/infrastructure
#
# author: Bob Droege (@bedroge)
# author: Terje Kvernes (@terjekv)
# author: Thomas Roeblitz (@trz42)
#
# license: GPLv2
#

function upload_to_staging_bucket
{
  _file=$1
  _bucket=$2
  _path=$3
  _endpoint_url=$4

  _options=
  if [[ ! -z "${_endpoint_url}" ]]; then
    _options="--endpoint-url ${_endpoint_url}"
  fi
  aws ${_options} s3 cp "${_file}" s3://${_bucket}/${_path}
}

# This needs expanding etc.
function check_file_name
{
  filename=$1
  if ( echo ${filename} | grep ^eessi > /dev/null &&
    echo ${filename} | grep -E '(compat|init|software)' > /dev/null ); then
    return 0
  else
    return 1
  fi
}

function create_metadata_file
{
  _artefact=$1
  _url=$2
  _repository=$3
  _pull_request_number=$4
  _pull_request_comment_id=$5

  _tmpfile=$(mktemp)

  jq -n \
    --arg un $(whoami) \
    --arg ip $(curl -s https://checkip.amazonaws.com) \
    --arg hn "$(hostname -f)" \
    --arg fn "$(basename ${_artefact})" \
    --arg sz "$(du -b "${_artefact}" | awk '{print $1}')" \
    --arg ct "$(date -r "${_artefact}")" \
    --arg sha256 "$(sha256sum "${_artefact}" | awk '{print $1}')" \
    --arg url "${_url}" \
    --arg repo "${_repository}" \
    --arg pr "${_pull_request_number}" \
    --arg pr_comment_id "${_pull_request_comment_id}" \
    '{
       uploader: {username: $un, ip: $ip, hostname: $hn},
       payload: {filename: $fn, size: $sz, ctime: $ct, sha256sum: $sha256, url: $url},
       link2pr: {repo: $repo, pr: $pr, pr_comment_id: $pr_comment_id},
     }' > "${_tmpfile}"

  echo "${_tmpfile}"
}

function display_help
{
  echo "Usage: $0 [OPTIONS] <filenames>"                                           >&2
  echo "  -a | --artefact-prefix PREFIX  -  a directory to which the artefact"     >&2
  echo "                                    shall be uploaded; BASH variable"      >&2
  echo "                                    expansion will be applied; arg '-l'"   >&2
  echo "                                    lists variables that are defined at"   >&2
  echo "                                    the time of expansion"                 >&2
  echo "  -e | --endpoint-url URL        -  endpoint url (needed for non AWS S3)"  >&2
  echo "  -h | --help                    -  display this usage information"        >&2
  echo "  -i | --pr-comment-id           -  identifier of a PR comment; may be"    >&2
  echo "                                    used to efficiently determine the PR"  >&2
  echo "                                    comment to be updated during the"      >&2
  echo "                                    ingestion procedure"                   >&2
  echo "  -l | --list-variables          -  list variables that are available"     >&2
  echo "                                    for expansion"                         >&2
  echo "  -k | --sign-key SCRIPT_KEY     -  specify location of the key to be"     >&2
  echo "                                    used to sign artefacts and metadata"   >&2
  echo "                                    files [optional; default: don't sign]" >&2
  echo "  -m | --metadata-prefix PREFIX  -  a directory to which the metadata"     >&2
  echo "                                    file shall be uploaded; BASH variable" >&2
  echo "                                    expansion will be applied; arg '-l'"   >&2
  echo "                                    lists variables that are defined at"   >&2
  echo "                                    the time of expansion"                 >&2
  echo "  -n | --bucket-name BUCKET      -  bucket name (same as BUCKET above)"    >&2
  echo "  -p | --pull-request-number INT -  a pull request number (INT); used to"  >&2
  echo "                                    link the upload to a PR"               >&2
  echo "  -r | --repository FULL_NAME    -  a repository name ACCOUNT/REPONAME;"   >&2
  echo "                                    used to link the upload to a PR"       >&2
  echo "  -s | --sign-script SCRIPT_PATH -  path to script that is used to sign"   >&2
  echo "                                    artefacts and metadata files. The"     >&2
  echo "                                    script is called with two arguments:"  >&2
  echo "                                    KEY file_to_sign. The KEY is the one"  >&2
  echo "                                    provided via option --sign-key. The"   >&2
  echo "                                    latter is determined by this script."  >&2
  echo "                                    [optional; default: don't sign]"       >&2
}

if [[ $# -lt 1 ]]; then
    display_help
    exit 1
fi


# process command line args
POSITIONAL_ARGS=()

# depends on which service hosts the bucket
#   minio: https://MINIO_SERVER:MINIO_PORT/{bucket_name}/
#   s3aws: https://{bucket_name}.s3.amazonaws.com/
# should be contructable from endpoint_url and bucket_name
bucket_base=

# default bucket is eessi-staging
bucket_name="eessi-staging"

# provided via options in the bot's config file app.cfg
endpoint_url=

# provided via command line arguments
pr_comment_id="none"
pull_request_number="none"
github_repository="EESSI/software-layer"
sign_key=
sign_script=

# provided via options in the bot's config file app.cfg and/or command line argument
metadata_prefix=
artefact_prefix=

# other variables
legacy_aws_path=
variables="github_repository legacy_aws_path pull_request_number"

while [[ $# -gt 0 ]]; do
  case $1 in
    -a|--artefact-prefix)
      artefact_prefix="$2"
      shift 2
      ;;
    -e|--endpoint-url)
      endpoint_url="$2"
      shift 2
      ;;
    -h|--help)
      display_help
      exit 0
      ;;
    -l|--list-variables)
      echo "variables that will be expanded: name (default value)"
      for var in ${variables}
      do
        echo "    ${var} (${!var:-unset})"
      done
      exit 0
      ;;
    -i|--pr-comment-id)
      pr_comment_id="$2"
      shift 2
      ;;
    -k|--sign-key)
      sign_key=$2
      if [[ ! -r "${sign_key}" ]]; then
        echo "Error: SSH key '${sign_key}' to be used for signing doesn't exist or cannot be read" >&2
        exit 1
      fi
      shift 2
      ;;
    -m|--metadata-prefix)
      metadata_prefix="$2"
      shift 2
      ;;
    -n|--bucket-name)
      bucket_name="$2"
      shift 2
      ;;
    -p|--pull-request-number)
      pull_request_number="$2"
      shift 2
      ;;
    -r|--repository)
      github_repository="$2"
      shift 2
      ;;
    -s|--sign-script)
      sign_script=$2
      if [[ ! -x "${sign_script}" ]]; then
        echo "Error: Script '${sign_script}' to be used for signing doesn't exist or is not executable" >&2
        exit 1
      fi
      shift 2
      ;;
    -*|--*)
      echo "Error: Unknown option: $1" >&2
      exit 1
      ;;
    *)  # No more options
      POSITIONAL_ARGS+=("$1") # save positional arg
      shift
      ;;
  esac
done

# restore potentially parsed filename(s) into $*
set -- "${POSITIONAL_ARGS[@]}"

# ensure that either none or both of $sign_key and $sign_script are defined
if [[ -n "${sign_key}" ]] && [[ -n "${sign_script}" ]]; then
  sign=1
elif [[ -n "${sign_key}" ]]; then
  sign=0
  echo "Error: Signing requires a key (${sign_key}) AND a script (${sign_script}); likely the bot config is incomplete" >&2
  exit 1
elif [[ -n "${sign_script}" ]]; then
  sign=0
  echo "Error: Signing requires a key (${sign_key}) AND a script (${sign_script}); likely the bot config is incomplete" >&2
  exit 1
else
  sign=0
fi

# infer bucket_base:
#   if endpoint_url is not set (assume AWS S3 is used),
#     bucket_base=https://${bucket_name}.s3.amazonaws.com/
#   if endpoint_url is set (assume non AWS S3, eg minio, is used),
#     bucket_base=${endpoint_url}/${bucket_name}/
# check if endpoint_url is not set
if [[ -z "${endpoint_url}" ]]; then
  # assume AWS S3 being used
  bucket_base=https://${bucket_name}.s3.amazonaws.com
else
  # assume non AWS S3 being used or AWS S3 with bucket not in DNS
  bucket_base=${endpoint_url}/${bucket_name}
fi

for file in "$*"; do
  if [[ -r "${file}" && -f "${file}" &&  -s "${file}" ]]; then
    basefile=$( basename ${file} )
    if check_file_name ${basefile}; then
      if tar tf "${file}" | head -n1 > /dev/null; then
        # 'legacy_aws_path' might be used in artefact_prefix or metadata_prefix
        # its purpose is to support the old/legacy method to derive the location
        # where to store the artefact and metadata file
        export legacy_aws_path=$(basename ${file} | tr -s '-' '/' \
                 | perl -pe 's/^eessi.//;' | perl -pe 's/\.tar\.gz$//;' )
        if [ -z ${artefact_prefix} ]; then
          aws_path=${legacy_aws_path}
        else
          export pull_request_number
          export github_repository
          aws_path=$(envsubst <<< "${artefact_prefix}")
        fi
        aws_file=$(basename ${file})
        # 1st sign artefact, and upload signature
        if [[ "${sign}" = "1" ]]; then
          # sign artefact
          ${sign_script} sign ${sign_key} ${file}
          # TODO check if signing worked (just check exit code == 0)
          sig_file=${file}.sig
          aws_sig_file=${aws_file}.sig

          # uploading signature
          echo "  store artefact signature at ${aws_path}/${aws_sig_file}"
          upload_to_staging_bucket \
                  "${sig_file}" \
                  "${bucket_name}" \
                  "${aws_path}/${aws_sig_file}" \
                  "${endpoint_url}"
        else
          echo "no signing method defined; not signing artefact"
        fi

        echo Uploading to "${url}"
        echo "  store artefact at ${aws_path}/${aws_file}"
        upload_to_staging_bucket \
                "${file}" \
                "${bucket_name}" \
                "${aws_path}/${aws_file}" \
                "${endpoint_url}"

        echo "Creating metadata file"
        url="${bucket_base}/${aws_path}/${aws_file}"
        echo "create_metadata_file file=${file} \
                                   url=${url} \
                                   github_repository=${github_repository} \
                                   pull_request_number=${pull_request_number} \
                                   pr_comment_id=${pr_comment_id}"
        metadata_file=$(create_metadata_file "${file}" \
                                             "${url}" \
                                             "${github_repository}" \
                                             "${pull_request_number}" \
                                             "${pr_comment_id}")
        aws_metadata_file=${aws_file}.meta.txt
        # TODO check that creating the metadata file succeeded
        echo "metadata:"
        cat ${metadata_file}

        if [ -z ${metadata_prefix} ]; then
          aws_path=${legacy_aws_path}
        else
          export pull_request_number
          export github_repository
          aws_path=$(envsubst <<< "${metadata_prefix}")
        fi
        # 2nd sign metadata file, and upload signature
        if [[ "${sign}" = "1" ]]; then
          # sign metadata file
          ${sign_script} sign ${sign_key} ${metadata_file}
          # TODO check if signing worked (just check exit code == 0)
          sig_metadata_file=${metadata_file}.sig
          aws_sig_metadata_file=${aws_metadata_file}.sig

          echo "  store metadata signature at ${aws_path}/${aws_sig_metadata_file}"
          upload_to_staging_bucket \
                  "${sig_metadata_file}" \
                  "${bucket_name}" \
                  "${aws_path}/${aws_sig_metadata_file}" \
                  "${endpoint_url}"
        else
          echo "no signing method defined; not signing metadata file"
        fi
        echo "  store metadata file at ${aws_path}/${aws_file}.meta.txt"
        upload_to_staging_bucket \
                "${metadata_file}" \
                "${bucket_name}" \
                "${aws_path}/${aws_file}.meta.txt" \
                "${endpoint_url}"
      else
        echo "'${file}' is not a tar file."
        exit 1
      fi
    else
      echo "${file} does not look like an eessi layer filename!"
      exit 1
    fi
  else
      echo "'${file}' is not a readable non zero-sized file."
      exit 1
  fi
done