[FAB-4458] Incremental downloads in bootstrap script

When a network failure occurs, bootstrap.sh will restart the download
using the curl -C option which provides resuming capabilities. The
Nexus repo, however, has some limitations which cause curl to return
error codes even when the download succeeds (see comments in the script).
They have been treated in the script. This will allow failed downloads
to be treated automatically, making the documentation leaner. Several
scenarios were tested in RHEL 7.4 and Ubuntu 14.04.

Patch-set #5: Rebase + incremental download of Fabric-CA binaries
+ handling of 404 errors (requested file isn't available)

Change-Id: If8e4da57b2c19431c2d098590ff3bbd87dfa83d6
Signed-off-by: Phillip Viana <phillip.l.viana@gmail.com>
Signed-off-by: Arnaud J Le Hors <lehors@us.ibm.com>

Author: Phillip Viana

scripts/bootstrap.sh

@@ -72,15 +72,78 @@ samplesInstall() {
   fi
 }
 
+# Incrementally downloads the .tar.gz file locally first, only decompressing it
+# after the download is complete. This is slower than binaryDownload() but
+# allows the download to be resumed.
+binaryIncrementalDownload() {
+      local BINARY_FILE=$1
+      local URL=$2
+      curl -f -s -C - ${URL} -o ${BINARY_FILE} || rc=$?
+      # Due to limitations in the current Nexus repo:
+      # curl returns 33 when there's a resume attempt with no more bytes to download
+      # curl returns 2 after finishing a resumed download
+      # with -f curl returns 22 on a 404
+      if [ "$rc" = 22 ]; then
+	  # looks like the requested file doesn't actually exist so stop here
+	  return 22
+      fi
+      if [ -z "$rc" ] || [ $rc -eq 33 ] || [ $rc -eq 2 ]; then
+          # The checksum validates that RC 33 or 2 are not real failures
+          echo "==> File downloaded. Verifying the md5sum..."
+          localMd5sum=$(md5sum ${BINARY_FILE} | awk '{print $1}')
+          remoteMd5sum=$(curl -s ${URL}.md5)
+          if [ "$localMd5sum" == "$remoteMd5sum" ]; then
+              echo "==> Extracting ${BINARY_FILE}..."
+              tar xzf ./${BINARY_FILE} --overwrite
+	      echo "==> Done."
+              rm -f ${BINARY_FILE} ${BINARY_FILE}.md5
+          else
+              echo "Download failed: the local md5sum is different from the remote md5sum. Please try again."
+              rm -f ${BINARY_FILE} ${BINARY_FILE}.md5
+              exit 1
+          fi
+      else
+          echo "Failure downloading binaries (curl RC=$rc). Please try again and the download will resume from where it stopped."
+          exit 1
+      fi
+}
+
+# This will attempt to download the .tar.gz all at once, but will trigger the
+# binaryIncrementalDownload() function upon a failure, allowing for resume
+# if there are network failures.
+binaryDownload() {
+      local BINARY_FILE=$1
+      local URL=$2
+      # Check if a previous failure occurred and the file was partially downloaded
+      if [ -e ${BINARY_FILE} ]; then
+          echo "==> Partial binary file found. Resuming download..."
+          binaryIncrementalDownload ${BINARY_FILE} ${URL}
+      else
+          curl ${URL} | tar xz || rc=$?
+          if [ ! -z "$rc" ]; then
+              echo "==> There was an error downloading the binary file. Switching to incremental download."
+              echo "==> Downloading file..."
+              binaryIncrementalDownload ${BINARY_FILE} ${URL}
+	  else
+	      echo "==> Done."
+          fi
+      fi
+}
+
 binariesInstall() {
   echo "===> Downloading version ${FABRIC_TAG} platform specific fabric binaries"
-  curl https://nexus.hyperledger.org/content/repositories/releases/org/hyperledger/fabric/hyperledger-fabric/${ARCH}-${VERSION}/hyperledger-fabric-${ARCH}-${VERSION}.tar.gz | tar xz
+  binaryDownload ${BINARY_FILE} https://nexus.hyperledger.org/content/repositories/releases/org/hyperledger/fabric/hyperledger-fabric/${ARCH}-${VERSION}/${BINARY_FILE}
+  if [ $? -eq 22 ]; then
+     echo
+     echo "------> ${FABRIC_TAG} platform specific fabric binary is not available to download <----"
+     echo
+   fi
 
   echo "===> Downloading version ${CA_TAG} platform specific fabric-ca-client binary"
-  curl https://nexus.hyperledger.org/content/repositories/releases/org/hyperledger/fabric-ca/hyperledger-fabric-ca/${ARCH}-${VERSION}/hyperledger-fabric-ca-${ARCH}-${VERSION}.tar.gz | tar xz
-  if [ $? != 0 ]; then
+  binaryDownload ${CA_BINARY_FILE} https://nexus.hyperledger.org/content/repositories/releases/org/hyperledger/fabric-ca/hyperledger-fabric-ca/${ARCH}-${CA_VERSION}/${CA_BINARY_FILE}
+  if [ $? -eq 22 ]; then
      echo
-     echo "------> ${CA_TAG} fabric-ca-client binary is not available to download  (Avaialble from 1.1.0-rc1) <----"
+     echo "------> ${CA_TAG} fabric-ca-client binary is not available to download  (Available from 1.1.0-rc1) <----"
      echo
    fi
 }
@@ -118,6 +181,9 @@ if echo $1 | grep -P -q '\d'; then
   fi
 fi
 
+BINARY_FILE=hyperledger-fabric-${ARCH}-${VERSION}.tar.gz
+CA_BINARY_FILE=hyperledger-fabric-ca-${ARCH}-${CA_VERSION}.tar.gz
+
 : ${CA_TAG:="$MARCH-$CA_VERSION"}
 : ${FABRIC_TAG:="$MARCH-$VERSION"}
 : ${THIRDPARTY_TAG:="$MARCH-$THIRDPARTY_IMAGE_VERSION"}