Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: update CMR search utility to replace deprecated scrolling #46

Merged
merged 1 commit into from
Oct 17, 2024
Merged

fix: update CMR search utility to replace deprecated scrolling #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions IS2view/utilities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env python
u"""
utilities.py
Written by Tyler Sutterley (08/2024)
Written by Tyler Sutterley (10/2024)
Download and management utilities

PYTHON DEPENDENCIES:
Expand All @@ -11,6 +11,8 @@
https://s3fs.readthedocs.io/en/latest/

UPDATE HISTORY:
Updated 10/2024: update CMR search utility to replace deprecated scrolling
https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html
Updated 08/2024: generalize hash function to use any available algorithm
Updated 05/2024: add wrapper to importlib for optional dependencies
Updated 11/2023: updated ssl context to fix deprecation error
Expand Down Expand Up @@ -1186,7 +1188,6 @@ def cmr(
CMR_KEYS.append(f'?provider={provider}')
CMR_KEYS.append('&sort_key[]=start_date')
CMR_KEYS.append('&sort_key[]=producer_granule_id')
CMR_KEYS.append('&scroll=true')
CMR_KEYS.append(f'&page_size={cmr_page_size}')
# append product string
CMR_KEYS.append(f'&short_name={product}')
Expand All @@ -1209,21 +1210,22 @@ def cmr(
# output list of granule names and urls
producer_granule_ids = []
granule_urls = []
cmr_scroll_id = None
cmr_search_after = None
while True:
req = urllib2.Request(cmr_query_url)
if cmr_scroll_id:
req.add_header('cmr-scroll-id', cmr_scroll_id)
# add CMR search after header
if cmr_search_after:
req.add_header('CMR-Search-After', cmr_search_after)
logging.debug(f'CMR-Search-After: {cmr_search_after}')
response = opener.open(req)
# get scroll id for next iteration
if not cmr_scroll_id:
headers = {k.lower():v for k, v in dict(response.info()).items()}
cmr_scroll_id = headers['cmr-scroll-id']
# get search after index for next iteration
headers = {k.lower():v for k,v in dict(response.info()).items()}
cmr_search_after = headers.get('cmr-search-after')
# read the CMR search as JSON
search_page = json.loads(response.read().decode('utf-8'))
ids, urls = cmr_filter_json(search_page,
endpoint=endpoint, request_type=request_type)
if not urls:
if not urls or cmr_search_after is None:
break
# extend lists
producer_granule_ids.extend(ids)
Expand Down
6 changes: 3 additions & 3 deletions doc/make.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ if "%SPHINXBUILD%" == "" (
set SOURCEDIR=source
set BUILDDIR=build

if "%1" == "" goto help

%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
Expand All @@ -21,10 +19,12 @@ if errorlevel 9009 (
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
echo.https://www.sphinx-doc.org/
exit /b 1
)

if "%1" == "" goto help

%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end

Expand Down