diff --git a/CHANGELOG.md b/CHANGELOG.md
index d3ebce9b465ede51e4d72ab57a3143c15b51d808..d1b89169beb4fa9c80b18e5d0bc7b7020916371e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -43,6 +43,8 @@ release.
### Fixed
+- Fixed bugs in downloadIsisData script [#5024](https://github.com/USGS-Astrogeology/ISIS3/issues/5024)
+
## [7.1.0] - 2022-07-27
### Changed
diff --git a/README.md b/README.md
index ae6e1733d730e21aef6ced8ed0de876551e83c38..13324ead57ddc34d18dd58e817dc899499d70957 100644
--- a/README.md
+++ b/README.md
@@ -389,7 +389,7 @@ If you plan to work with data from all missions, then the download will require
the outdated rsync download information can be found [here](https://github.com/USGS-Astrogeology/ISIS3/wiki/Outdated-ISIS-Data-Information)
-The ISIS Data Area is hosted on a combination of AWS S3 buckets and public http servers e.g. NAIF, Jaxa, ESA and not through conda channels like the ISIS binaries. This requires using the `downloadIsisData.py` script from within a terminal window within your Unix distribution, or from within WSL if running Windows 10. Downloading all mission data requires over 520 GB of disk space. If you want to acquire only certain mission data [click here](#Mission-Specific-Data-Downloads). To download all ISIS data files, continue reading.
+The ISIS Data Area is hosted on a combination of AWS S3 buckets and public http servers e.g. NAIF, Jaxa, ESA and not through conda channels like the ISIS binaries. This requires using the `downloadIsisData` script from within a terminal window within your Unix distribution, or from within WSL if running Windows 10. Downloading all mission data requires over 520 GB of disk space. If you want to acquire only certain mission data [click here](#Mission-Specific-Data-Downloads). To download all ISIS data files, continue reading.
To download all ISIS data, use the following command:
@@ -513,11 +513,12 @@ You can download the script and config file from the repo:
conda install -c conda-forge rclone
# download the script and rclone config file
-curl https://github.com/USGS-Astrogeology/ISIS3/raw/dev/isis/scripts/downloadIsisData.py -o downloadIsisData.py
+curl https://raw.githubusercontent.com/USGS-Astrogeology/ISIS3/dev/isis/scripts/downloadIsisData -o downloadIsisData.py
-curl https://github.com/USGS-Astrogeology/ISIS3/raw/dev/isis/config/rclone.conf -o rclone.conf
+curl https://raw.githubusercontent.com/USGS-Astrogeology/ISIS3/dev/isis/config/rclone.conf -o rclone.conf
# run the script as normal, using --config to point to where you downloaded the config file
-python downloadIsisData.py sync <mission> $ISISDATA --config rclone.conf
-
+python downloadIsisData <mission> $ISISDATA --config rclone.conf
```
+
+> The script does not support python2, sometimes you need to explicitly use python3 with `python3 downloadIsisData <mission> $ISISDATA --config rclone.conf`
diff --git a/isis/CMakeLists.txt b/isis/CMakeLists.txt
index 040eabe499703824e426510a75560dfa3d6b1951..e8b252a038d612bd20af0258b127dd15df3c01b3 100644
--- a/isis/CMakeLists.txt
+++ b/isis/CMakeLists.txt
@@ -610,7 +610,7 @@ install(FILES ${CMAKE_BINARY_DIR}/version DESTINATION ${CMAKE_INSTA
install(DIRECTORY ${CMAKE_SOURCE_DIR}/scripts DESTINATION ${CMAKE_INSTALL_PREFIX})
install(PROGRAMS ${CMAKE_BINARY_DIR}/lib/Camera.plugin DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/)
install(PROGRAMS ${CMAKE_SOURCE_DIR}/scripts/downloadIsisData DESTINATION ${CMAKE_INSTALL_PREFIX}/bin/)
-install(PROGRAMS ${CMAKE_SOURCE_DIR}/config/rclone.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/etc/isis/)
+install(FILES ${CMAKE_SOURCE_DIR}/config/rclone.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/etc/isis/)
# Trigger all post-install behavior.
# - The only way to run commands post-install in CMake is to add a subdirectory at
diff --git a/isis/config/rclone.conf b/isis/config/rclone.conf
index 0b86075c79b803b521bfdce41c0aff8a770712d3..50421d3f792e4cf580d9015f0ec3b8106965f990 100644
--- a/isis/config/rclone.conf
+++ b/isis/config/rclone.conf
@@ -18,19 +18,19 @@ url = http://www.darts.isas.jaxa.jp/
[apollo15_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/apollo15
+remote = asc_s3:asc-isisdata/usgs_data/apollo15
[apollo16_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/apollo16
+remote = asc_s3:asc-isisdata/usgs_data/apollo16
[apollo17_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/apollo17
+remote = asc_s3:asc-isisdata/usgs_data/apollo17
[lro_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/lro
+remote = asc_s3:asc-isisdata/usgs_data/lro
[tgo_naifKernels]
type = alias
@@ -38,7 +38,7 @@ remote = esa:/data/SPICE/ExoMars2016/kernels/
[tgo_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/tgo
+remote = asc_s3:asc-isisdata/usgs_data/tgo
[dawn_naifKernels]
type = alias
@@ -46,7 +46,7 @@ remote = naif:/pub/naif/DAWN/kernels
[dawn_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/dawn
+remote = asc_s3:asc-isisdata/usgs_data/dawn
[cassini_naifKernels]
type = alias
@@ -54,7 +54,7 @@ remote = naif:/pub/naif/CASSINI/kernels
[cassini_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/cassini
+remote = asc_s3:asc-isisdata/usgs_data/cassini
[hayabusa2_naifKernels]
type = alias
@@ -62,7 +62,7 @@ remote = jaxa:/pub/hayabusa2/spice_bundle/spice_kernels
[hayabusa2_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/hayabusa2
+remote = asc_s3:asc-isisdata/usgs_data/hayabusa2
[juno_naifKernels]
type = alias
@@ -70,7 +70,7 @@ remote = naif:/pub/naif/pds/data/jno-j_e_ss-spice-6-v1.0/jnosp_1000/data/
[juno_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/juno
+remote = asc_s3:asc-isisdata/usgs_data/juno
[odyssey_naifKernels]
type = alias
@@ -78,7 +78,7 @@ remote = naif:/pub/naif/M01/kernels
[odyssey_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/odyssey
+remote = asc_s3:asc-isisdata/usgs_data/odyssey
[mro_naifKernels]
type = alias
@@ -86,7 +86,7 @@ remote = naif:/pub/naif/MRO/kernels
[mro_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/mro
+remote = asc_s3:asc-isisdata/usgs_data/mro
[mex_naifKernels]
type = alias
@@ -94,11 +94,11 @@ remote = naif:/pub/naif/MEX/kernels
[mex_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/mex
+remote = asc_s3:asc-isisdata/usgs_data/mex
[legacybase_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/legacy_base
+remote = asc_s3:asc-isisdata/usgs_data/legacy_base
[hayabusa_naifKernels]
type = alias
@@ -106,7 +106,7 @@ remote = jaxa:/pub/spice/HAYABUSA/kernels
[hayabusa_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/hayabusa
+remote = asc_s3:asc-isisdata/usgs_data/hayabusa
[chandrayaan1_naifKernels]
type = alias
@@ -114,7 +114,7 @@ remote = esa:/data/SPICE/CHANDRAYAAN-1/kernels
[chandrayaan1_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/chandrayaan1
+remote = asc_s3:asc-isisdata/usgs_data/chandrayaan1
[clementine1_naifKernels]
type = alias
@@ -122,7 +122,7 @@ remote = naif:/pub/naif/pds/data/clem1-l-spice-6-v1.0/clsp_1000/data/
[clementine1_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/clementine1
+remote = asc_s3:asc-isisdata/usgs_data/clementine1
[kaguya_naifKernels]
type = alias
@@ -130,7 +130,7 @@ remote = jaxa:/pub/pds3/sln-l-spice-6-v1.0/slnsp_1000/data/
[kaguya_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/kaguya
+remote = asc_s3:asc-isisdata/usgs_data/kaguya
[mariner10_naifKernels]
type = alias
@@ -138,7 +138,7 @@ remote = naif:/pub/naif/M10/kernels
[mariner10_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/mariner10
+remote = asc_s3:asc-isisdata/usgs_data/mariner10
[messenger_naifKernels]
type = alias
@@ -146,7 +146,7 @@ remote = naif:/pub/naif/pds/data/mess-e_v_h-spice-6-v1.0/messsp_1000/data/
[messenger_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/messenger
+remote = asc_s3:asc-isisdata/usgs_data/messenger
[mgs_naifKernels]
type = alias
@@ -154,7 +154,7 @@ remote = naif:/pub/naif/pds/data/mgs-m-spice-6-v1.0/mgsp_1000/data/
[mgs_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/mgs
+remote = asc_s3:asc-isisdata/usgs_data/mgs
[near_naifKernels]
type = alias
@@ -162,7 +162,7 @@ remote = naif:/pub/naif/pds/data/near-a-spice-6-v1.0/nearsp_1000/data/
[near_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/near
+remote = asc_s3:asc-isisdata/usgs_data/near
[newhorizons_naifKernels]
type = alias
@@ -170,7 +170,7 @@ remote = naif:/pub/naif/pds/data/nh-j_p_ss-spice-6-v1.0/nhsp_1000/data/
[newhorizons_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/newhorizons
+remote = asc_s3:asc-isisdata/usgs_data/newhorizons
[osirisrex_naifKernels]
type = alias
@@ -178,11 +178,11 @@ remote = naif:/pub/naif/pds/pds4/orex/orex_spice/spice_kernels/
[osirisrex_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/osirisrex/
+remote = asc_s3:asc-isisdata/usgs_data/osirisrex/
[rolo_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/rolo
+remote = asc_s3:asc-isisdata/usgs_data/rolo
[rosetta_naifKernels]
type = alias
@@ -190,7 +190,7 @@ remote = naif:/pub/naif/ROSETTA/kernels
[rosetta_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/rosetta
+remote = asc_s3:asc-isisdata/usgs_data/rosetta
[smart1_naifKernels]
type = alias
@@ -198,7 +198,7 @@ remote = esa:/data/SPICE/SMART-1/kernels
[smart1_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/smart1
+remote = asc_s3:asc-isisdata/usgs_data/smart1
[viking1_naifKernels]
type = alias
@@ -206,7 +206,7 @@ remote = naif:/pub/naif/VIKING/kernels
[viking1_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/viking1
+remote = asc_s3:asc-isisdata/usgs_data/viking1
[viking2_naifKernels]
type = alias
@@ -214,7 +214,7 @@ remote = naif:/pub/naif/VIKING/kernels
[viking2_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/viking2
+remote = asc_s3:asc-isisdata/usgs_data/viking2
[voyager1_naifKernels]
type = alias
@@ -222,7 +222,7 @@ remote = naif:/pub/naif/VOYAGER/kernels
[voyager1_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/voyager1
+remote = asc_s3:asc-isisdata/usgs_data/voyager1
[voyager2_naifKernels]
type = alias
@@ -230,9 +230,9 @@ remote = naif:/pub/naif/VOYAGER/kernels
[voyager2_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/voyager2
+remote = asc_s3:asc-isisdata/usgs_data/voyager2
[base_usgs]
type = alias
-remote = asc_s3:asc-isisdata/isis_data/base
+remote = asc_s3:asc-isisdata/usgs_data/base
diff --git a/isis/scripts/downloadIsisData b/isis/scripts/downloadIsisData
index b3a4ca1ee2f01a8973fe587a54f8ead1cecd5ed3..07b773dcb5000a8c83fb5e58a0c801f9616178dc 100755
--- a/isis/scripts/downloadIsisData
+++ b/isis/scripts/downloadIsisData
@@ -2,7 +2,6 @@
from multiprocessing import ProcessError
from multiprocessing.dummy import Process
-from pprint import pprint
import logging as log
import os
import json
@@ -11,22 +10,22 @@ import subprocess
import tempfile
from shutil import which
from os import path
-
+from collections import OrderedDict
def find_conf():
from pathlib import Path
local_path = Path("rclone.conf")
- install_path = Path(os.environ.get("CONDA_PREFIX")) / "etc" / "isis" / local_path
- repo_path = Path(os.path.dirname(__file__)) / '..' / 'config' / 'rclone.conf'
-
+ # this should be installed in scripts folder, so config is one directory up in etc
+ install_path = Path(os.path.dirname(__file__)) / '..' / "etc" / "isis" / 'rclone.conf'
+ repo_path = Path(os.path.dirname(__file__)) / '..' / 'config' / 'rclone.conf'
if local_path.exists():
- return str(local_path)
+ return str(local_path)
elif repo_path.exists():
return str(repo_path)
elif install_path.exists():
return str(install_path)
else:
- return ""
+ raise RuntimeError("No config found, try passing it in explicitly with --config <path>")
# set log level to debug
@@ -102,7 +101,9 @@ def create_rclone_arguments(destination, mission_name, dry_run=False, ntransfers
"""
log.debug(f"Creating RClone command for {mission_name}")
mission_dir_name, source_type = mission_name.replace(":", "").split("_")
-
+ if (mission_dir_name == "legacybase"):
+ # We still want things to go into base
+ mission_dir_name = "base"
log.debug(f"Mission_dir_name: {mission_dir_name}, source_type: {source_type}")
destination = os.path.join(destination, str(mission_dir_name).replace(":",""))
@@ -111,11 +112,10 @@ def create_rclone_arguments(destination, mission_name, dry_run=False, ntransfers
if not os.path.exists(destination) and dry_run==False:
log.debug("{destination} does not exist, making directory")
- print(destination)
os.makedirs(destination)
extra_args=[f"{mission_name}",f"{destination}", "--progress", f"--checkers={ntransfers}", f"--transfers={ntransfers}", "--track-renames", f"--log-level={log.getLevelName(log.getLogger().getEffectiveLevel())}"]
-
+
if dry_run:
extra_args.append("--dry-run")
@@ -139,25 +139,25 @@ def main(mission, dest, cfg_path, dry_run, ntransfers):
log.debug(f"Using config: {cfg_path}")
result = rclone("listremotes", config=cfg_path)
config_sources = result.get('out').decode("utf-8").split("\n")
- if config_sources == ['']:
+ if config_sources == ['']:
log.error("Remote sources came back empty. Get more info by re-running with verbose flag.")
quit(-1)
log.debug(f"Remote Sources: {config_sources}")
- supported_missions = {}
- for source in config_sources:
+ supported_missions = OrderedDict({})
+ for source in sorted(config_sources, key=lambda x: x.split("_")[-1]):
parsed_name = source.split("_")
# If it is a mission, it should be in the format <mission_nam>_<source_type>
- if len(parsed_name) == 2 and parsed_name[1] in ["usgs:", "naifKernels:"]:
+ if len(parsed_name) == 2 and parsed_name[1] in ["usgs:", "naifKernels:"]:
remotes_mission_name = parsed_name[0]
supported_missions[remotes_mission_name] = supported_missions.get(remotes_mission_name, []) + [source]
log.debug(f"Supported missions:\n {supported_missions.keys()}")
log.debug(f"Complete Dictionary:\n {json.dumps(supported_missions, indent=2)}")
- if not mission in supported_missions.keys():
+ if not mission in supported_missions.keys() and mission.upper() not in ("ALL", "LEGACYBASE"):
raise LookupError(f"{mission} is not in the list of supported missions: {supported_missions.keys()}")
-
+
if mission == "legacybase":
args = create_rclone_arguments(dest, "legacybase_usgs:", dry_run, ntransfers)
rclone(command="sync", extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)
@@ -170,16 +170,15 @@ def main(mission, dest, cfg_path, dry_run, ntransfers):
else:
for remote in supported_missions[mission]:
args = create_rclone_arguments(dest, remote, dry_run, ntransfers)
- rclone(command="sync", extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)
+ rclone(command="sync", extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)
-if __name__ == '__main__':
+if __name__ == '__main__':
helpString = (
'''This will allow for a user to download isis data directly to their machine from the USGS S3 buckets as well as public end points\n\n'''
- '''To use the download ISIS Data script you must supply 3 parameters with an optional 4th.\n\n'''
- '''<rclone command> <Mission name> <destination to copy to> <--dry-run (optional)> \n'''
- '''Example of how to run this program:\n'''
- '''python downloadIsisData.py tgo ~/isisData/tgo\n\n'''
+ '''<Mission name> <destination to copy to> <--dry-run (optional)> \n\n'''
+ '''Example of how to run this program:\n\n'''
+ ''' downloadIsisData tgo ~/isisData/\n\n'''
'''NOTE: if you would like to download the data for every mission supply the value ALL for the <Mission name> parameter''')
parser = argparse.ArgumentParser(description = helpString, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('mission', help='mission for files to be downloaded')
@@ -206,3 +205,5 @@ if __name__ == '__main__':
log.basicConfig(**log_kwargs)
main(args.mission, args.dest, os.path.expanduser(args.config), args.dry_run, args.num_transfers)
+
+