v0.16.34 (#180)

api.py
   * check for presence of clock skew argument rather than check google.auth.__version__
fiss.py
   * meth_acl/config_acl
      - changed map(f(x),iterable) returns into list comprehensions to make py2/3 consistent
   * mop updated to account for change in Terra output directory structure
fccore.py
   * removed version string parser in favor of above check for the clock skew argument

Author: dheiman

changelog.txt

@@ -3,6 +3,12 @@ Change Log for FISSFC: the (Fi)recloud (S)ervice (S)elector
 =======================================================================
 Terms used below:  HL = high level interface, LL = low level interface
 
+v0.16.34 - Hotfixes: in Python 2 map() returns a list, in Python 3, map()
+           returns a generator object - tasks that returned map(f(x),iterable)
+           now return a list comprehension; clock-skew now checks for argument
+           presence at runtime rather than google.auth version; mop HL function
+           updated to account for new output path structure in Terra.
+
 v0.16.33 - HL: added space_size, space_cost; hotfixes: fixed Python 2 compatibility;
            blocked google-auth versions with restrictive clock-skew and enabled
            later versions with modifiable clock-skew, increasing clock-skew

firecloud/__about__.py

@@ -1,2 +1,2 @@
 # Package version
-__version__ = "0.16.33"
+__version__ = "0.16.34"

firecloud/api.py

@@ -21,14 +21,15 @@
 from six.moves.urllib.parse import urlencode, urljoin
 from six import string_types
 
+import inspect
+
 import google.auth
 from google.auth.exceptions import DefaultCredentialsError, RefreshError
 from google.auth.transport.requests import AuthorizedSession, Request
 from google.oauth2 import id_token
 
 from firecloud.errors import FireCloudServerError
 from firecloud.fccore import __fcconfig as fcconfig
-from firecloud.fccore import release_tuple_from_version_string
 from firecloud.__about__ import __version__
 
 FISS_USER_AGENT = "FISS/" + __version__
@@ -49,12 +50,18 @@ def _set_session():
     global __USER_ID
 
     if __SESSION is None:
+        # determine if clock_skew_in_seconds is a parameter for id_token.verify_oauth2_token()
+        try:  # PY3
+            argspec = inspect.getfullargspec(id_token.verify_oauth2_token)
+        except AttributeError:  # PY2
+            argspec = inspect.getargspec(id_token.verify_oauth2_token)
+            
         try:
             __SESSION = AuthorizedSession(google.auth.default(['https://www.googleapis.com/auth/userinfo.profile',
                                                                'https://www.googleapis.com/auth/userinfo.email'])[0])
             health()
             # google.auth 2.1.0 introduced a restrictive clock skew that was unmodifiable until 2.3.2
-            if release_tuple_from_version_string(google.auth.__version__) >= (2,3,2):
+            if 'clock_skew_in_seconds' in argspec.args:
                 __USER_ID = id_token.verify_oauth2_token(__SESSION.credentials.id_token,
                                                          Request(session=__SESSION),
                                                          clock_skew_in_seconds=10)['email']

firecloud/fccore.py

@@ -21,7 +21,6 @@
 import tempfile
 import shutil
 import subprocess
-import re
 from io import IOBase
 from firecloud import __about__
 from google.auth import environment_vars
@@ -220,46 +219,4 @@ def edit_file(name, backup=None):
     current_tolm  = os.stat(name).st_mtime
     return current_tolm != previous_tolm
 
-
-# From PEP-440:
-# https://peps.python.org/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions
-VERSION_PATTERN = r"""
-    v?
-    (?:
-        (?:(?P<epoch>[0-9]+)!)?                           # epoch
-        (?P<release>[0-9]+(?:\.[0-9]+)*)                  # release segment
-        (?P<pre>                                          # pre-release
-            [-_\.]?
-            (?P<pre_l>(a|b|c|rc|alpha|beta|pre|preview))
-            [-_\.]?
-            (?P<pre_n>[0-9]+)?
-        )?
-        (?P<post>                                         # post release
-            (?:-(?P<post_n1>[0-9]+))
-            |
-            (?:
-                [-_\.]?
-                (?P<post_l>post|rev|r)
-                [-_\.]?
-                (?P<post_n2>[0-9]+)?
-            )
-        )?
-        (?P<dev>                                          # dev release
-            [-_\.]?
-            (?P<dev_l>dev)
-            [-_\.]?
-            (?P<dev_n>[0-9]+)?
-        )?
-    )
-    (?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
-"""
-
-version_regex = re.compile(
-    r"^\s*" + VERSION_PATTERN + r"\s*$",
-    re.VERBOSE | re.IGNORECASE,
-)
-
-def release_tuple_from_version_string(version_string):
-    return tuple(int(val) for val in version_regex.match(version_string).group('release').split('.'))
-
 # }}}

firecloud/fiss.py

@@ -476,7 +476,7 @@ def meth_acl(args):
                                                     args.snapshot_id)
     fapi._check_response_code(r, 200)
     acls = sorted(r.json(), key=lambda k: k['user'])
-    return map(lambda acl: '{0}\t{1}'.format(acl['user'], acl['role']), acls)
+    return ['{0}\t{1}'.format(acl['user'], acl['role']) for acl in acls]
 
 @fiss_cmd
 def meth_set_acl(args):
@@ -668,7 +668,7 @@ def config_acl(args):
                                                     args.snapshot_id)
     fapi._check_response_code(r, 200)
     acls = sorted(r.json(), key=lambda k: k['user'])
-    return map(lambda acl: '{0}\t{1}'.format(acl['user'], acl['role']), acls)
+    return ['{0}\t{1}'.format(acl['user'], acl['role']) for acl in acls]
 
 @fiss_cmd
 def config_set_acl(args):
@@ -1374,10 +1374,17 @@ def list_blob_gen(bucket_name):
 
         # Check to see if bucket file path contain the user's submission id
         # to ensure deletion of files in the submission directories only.
-        # Splits the bucket file: "gs://bucket_Id/submission_id/file_path", by the '/' symbol
-        # and stores values in a 5 length array: ['gs:', '' , 'bucket_Id', submission_id, file_path]
-        # to extract the submission id from the 4th element (index 3) of the array
-        bucket_files = set(bucket_file for bucket_file in bucket_file_sizes if bucket_file.split('/', 4)[3] in submission_ids)
+        # Splits the bucket file: gs://<bucket>/<submission_id>/<file_path> or
+        #                         gs://<bucket>/submissions/<submission_id>/<file_path>, by the '/' symbol
+        # and stores values in a 6 length array: ['gs:', '' , <bucket>, <submission_id>, <workflow_name>, <file_path>] or
+        #                                        ['gs:', '' , <bucket>, 'submissions', <submission_id>, <file_path>]
+        # to extract the submission id from the 4th or 5th element (index 3 or 4) of the array
+        bucket_files = set()
+        for bucket_file in bucket_file_sizes:
+            for sub_id in bucket_file.split('/', 5)[3:5]:
+                if sub_id in submission_ids:
+                    bucket_files.add(bucket_file)
+                    break
 
     except Exception as e:
         eprint("Error retrieving files from bucket:" +
@@ -1428,12 +1435,17 @@ def can_delete(f):
             return False
         if filename == "rc":
             return False
+        if filename == "memory_retry_rc":
+            return False
         # Don't delete tool's exec.sh or script
         if filename in ('exec.sh', 'script'):
             return False
         # keep stdout, stderr, and output
         if filename in ('stderr', 'stdout', 'output'):
             return False
+        # Don't delete utility scripts
+        if filename in ('gcs_localization.sh', 'gcs_delocalization.sh', 'gcs_transfer.sh'):
+            return False
         # Only delete specified unreferenced files
         if args.include:
             for glob in args.include: