Skip to content
Snippets Groups Projects
Unverified Commit c14a746b authored by Kelvin Rodriguez's avatar Kelvin Rodriguez Committed by GitHub
Browse files

downloadIsisData Script fixes (#5025)

* fixes for downloadIsisData

* added changelog line

* no more conda_prefix

* tweaked do string

* removed prints

* made usgs sources download second

* isis_data to usgs_data

* update readme with correc paths to .conf and script
parent 96e715d9
No related branches found
No related tags found
No related merge requests found
......@@ -43,6 +43,8 @@ release.
### Fixed
- Fixed bugs in downloadIsisData script [#5024](https://github.com/USGS-Astrogeology/ISIS3/issues/5024)
## [7.1.0] - 2022-07-27
### Changed
......
......@@ -389,7 +389,7 @@ If you plan to work with data from all missions, then the download will require
the outdated rsync download information can be found [here](https://github.com/USGS-Astrogeology/ISIS3/wiki/Outdated-ISIS-Data-Information)
The ISIS Data Area is hosted on a combination of AWS S3 buckets and public http servers e.g. NAIF, Jaxa, ESA and not through conda channels like the ISIS binaries. This requires using the `downloadIsisData.py` script from within a terminal window within your Unix distribution, or from within WSL if running Windows 10. Downloading all mission data requires over 520 GB of disk space. If you want to acquire only certain mission data [click here](#Mission-Specific-Data-Downloads). To download all ISIS data files, continue reading.
The ISIS Data Area is hosted on a combination of AWS S3 buckets and public http servers e.g. NAIF, Jaxa, ESA and not through conda channels like the ISIS binaries. This requires using the `downloadIsisData` script from within a terminal window within your Unix distribution, or from within WSL if running Windows 10. Downloading all mission data requires over 520 GB of disk space. If you want to acquire only certain mission data [click here](#Mission-Specific-Data-Downloads). To download all ISIS data files, continue reading.
To download all ISIS data, use the following command:
......@@ -513,11 +513,12 @@ You can download the script and config file from the repo:
conda install -c conda-forge rclone
# download the script and rclone config file
curl https://github.com/USGS-Astrogeology/ISIS3/raw/dev/isis/scripts/downloadIsisData.py -o downloadIsisData.py
curl https://raw.githubusercontent.com/USGS-Astrogeology/ISIS3/dev/isis/scripts/downloadIsisData -o downloadIsisData.py
curl https://github.com/USGS-Astrogeology/ISIS3/raw/dev/isis/config/rclone.conf -o rclone.conf
curl https://raw.githubusercontent.com/USGS-Astrogeology/ISIS3/dev/isis/config/rclone.conf -o rclone.conf
# run the script as normal, using --config to point to where you downloaded the config file
python downloadIsisData.py sync <mission> $ISISDATA --config rclone.conf
python downloadIsisData <mission> $ISISDATA --config rclone.conf
```
> The script does not support python2, sometimes you need to explicitly use python3 with `python3 downloadIsisData <mission> $ISISDATA --config rclone.conf`
......@@ -610,7 +610,7 @@ install(FILES ${CMAKE_BINARY_DIR}/version DESTINATION ${CMAKE_INSTA
install(DIRECTORY ${CMAKE_SOURCE_DIR}/scripts DESTINATION ${CMAKE_INSTALL_PREFIX})
install(PROGRAMS ${CMAKE_BINARY_DIR}/lib/Camera.plugin DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/)
install(PROGRAMS ${CMAKE_SOURCE_DIR}/scripts/downloadIsisData DESTINATION ${CMAKE_INSTALL_PREFIX}/bin/)
install(PROGRAMS ${CMAKE_SOURCE_DIR}/config/rclone.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/etc/isis/)
install(FILES ${CMAKE_SOURCE_DIR}/config/rclone.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/etc/isis/)
# Trigger all post-install behavior.
# - The only way to run commands post-install in CMake is to add a subdirectory at
......
......@@ -18,19 +18,19 @@ url = http://www.darts.isas.jaxa.jp/
[apollo15_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/apollo15
remote = asc_s3:asc-isisdata/usgs_data/apollo15
[apollo16_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/apollo16
remote = asc_s3:asc-isisdata/usgs_data/apollo16
[apollo17_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/apollo17
remote = asc_s3:asc-isisdata/usgs_data/apollo17
[lro_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/lro
remote = asc_s3:asc-isisdata/usgs_data/lro
[tgo_naifKernels]
type = alias
......@@ -38,7 +38,7 @@ remote = esa:/data/SPICE/ExoMars2016/kernels/
[tgo_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/tgo
remote = asc_s3:asc-isisdata/usgs_data/tgo
[dawn_naifKernels]
type = alias
......@@ -46,7 +46,7 @@ remote = naif:/pub/naif/DAWN/kernels
[dawn_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/dawn
remote = asc_s3:asc-isisdata/usgs_data/dawn
[cassini_naifKernels]
type = alias
......@@ -54,7 +54,7 @@ remote = naif:/pub/naif/CASSINI/kernels
[cassini_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/cassini
remote = asc_s3:asc-isisdata/usgs_data/cassini
[hayabusa2_naifKernels]
type = alias
......@@ -62,7 +62,7 @@ remote = jaxa:/pub/hayabusa2/spice_bundle/spice_kernels
[hayabusa2_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/hayabusa2
remote = asc_s3:asc-isisdata/usgs_data/hayabusa2
[juno_naifKernels]
type = alias
......@@ -70,7 +70,7 @@ remote = naif:/pub/naif/pds/data/jno-j_e_ss-spice-6-v1.0/jnosp_1000/data/
[juno_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/juno
remote = asc_s3:asc-isisdata/usgs_data/juno
[odyssey_naifKernels]
type = alias
......@@ -78,7 +78,7 @@ remote = naif:/pub/naif/M01/kernels
[odyssey_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/odyssey
remote = asc_s3:asc-isisdata/usgs_data/odyssey
[mro_naifKernels]
type = alias
......@@ -86,7 +86,7 @@ remote = naif:/pub/naif/MRO/kernels
[mro_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/mro
remote = asc_s3:asc-isisdata/usgs_data/mro
[mex_naifKernels]
type = alias
......@@ -94,11 +94,11 @@ remote = naif:/pub/naif/MEX/kernels
[mex_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/mex
remote = asc_s3:asc-isisdata/usgs_data/mex
[legacybase_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/legacy_base
remote = asc_s3:asc-isisdata/usgs_data/legacy_base
[hayabusa_naifKernels]
type = alias
......@@ -106,7 +106,7 @@ remote = jaxa:/pub/spice/HAYABUSA/kernels
[hayabusa_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/hayabusa
remote = asc_s3:asc-isisdata/usgs_data/hayabusa
[chandrayaan1_naifKernels]
type = alias
......@@ -114,7 +114,7 @@ remote = esa:/data/SPICE/CHANDRAYAAN-1/kernels
[chandrayaan1_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/chandrayaan1
remote = asc_s3:asc-isisdata/usgs_data/chandrayaan1
[clementine1_naifKernels]
type = alias
......@@ -122,7 +122,7 @@ remote = naif:/pub/naif/pds/data/clem1-l-spice-6-v1.0/clsp_1000/data/
[clementine1_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/clementine1
remote = asc_s3:asc-isisdata/usgs_data/clementine1
[kaguya_naifKernels]
type = alias
......@@ -130,7 +130,7 @@ remote = jaxa:/pub/pds3/sln-l-spice-6-v1.0/slnsp_1000/data/
[kaguya_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/kaguya
remote = asc_s3:asc-isisdata/usgs_data/kaguya
[mariner10_naifKernels]
type = alias
......@@ -138,7 +138,7 @@ remote = naif:/pub/naif/M10/kernels
[mariner10_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/mariner10
remote = asc_s3:asc-isisdata/usgs_data/mariner10
[messenger_naifKernels]
type = alias
......@@ -146,7 +146,7 @@ remote = naif:/pub/naif/pds/data/mess-e_v_h-spice-6-v1.0/messsp_1000/data/
[messenger_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/messenger
remote = asc_s3:asc-isisdata/usgs_data/messenger
[mgs_naifKernels]
type = alias
......@@ -154,7 +154,7 @@ remote = naif:/pub/naif/pds/data/mgs-m-spice-6-v1.0/mgsp_1000/data/
[mgs_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/mgs
remote = asc_s3:asc-isisdata/usgs_data/mgs
[near_naifKernels]
type = alias
......@@ -162,7 +162,7 @@ remote = naif:/pub/naif/pds/data/near-a-spice-6-v1.0/nearsp_1000/data/
[near_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/near
remote = asc_s3:asc-isisdata/usgs_data/near
[newhorizons_naifKernels]
type = alias
......@@ -170,7 +170,7 @@ remote = naif:/pub/naif/pds/data/nh-j_p_ss-spice-6-v1.0/nhsp_1000/data/
[newhorizons_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/newhorizons
remote = asc_s3:asc-isisdata/usgs_data/newhorizons
[osirisrex_naifKernels]
type = alias
......@@ -178,11 +178,11 @@ remote = naif:/pub/naif/pds/pds4/orex/orex_spice/spice_kernels/
[osirisrex_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/osirisrex/
remote = asc_s3:asc-isisdata/usgs_data/osirisrex/
[rolo_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/rolo
remote = asc_s3:asc-isisdata/usgs_data/rolo
[rosetta_naifKernels]
type = alias
......@@ -190,7 +190,7 @@ remote = naif:/pub/naif/ROSETTA/kernels
[rosetta_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/rosetta
remote = asc_s3:asc-isisdata/usgs_data/rosetta
[smart1_naifKernels]
type = alias
......@@ -198,7 +198,7 @@ remote = esa:/data/SPICE/SMART-1/kernels
[smart1_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/smart1
remote = asc_s3:asc-isisdata/usgs_data/smart1
[viking1_naifKernels]
type = alias
......@@ -206,7 +206,7 @@ remote = naif:/pub/naif/VIKING/kernels
[viking1_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/viking1
remote = asc_s3:asc-isisdata/usgs_data/viking1
[viking2_naifKernels]
type = alias
......@@ -214,7 +214,7 @@ remote = naif:/pub/naif/VIKING/kernels
[viking2_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/viking2
remote = asc_s3:asc-isisdata/usgs_data/viking2
[voyager1_naifKernels]
type = alias
......@@ -222,7 +222,7 @@ remote = naif:/pub/naif/VOYAGER/kernels
[voyager1_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/voyager1
remote = asc_s3:asc-isisdata/usgs_data/voyager1
[voyager2_naifKernels]
type = alias
......@@ -230,9 +230,9 @@ remote = naif:/pub/naif/VOYAGER/kernels
[voyager2_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/voyager2
remote = asc_s3:asc-isisdata/usgs_data/voyager2
[base_usgs]
type = alias
remote = asc_s3:asc-isisdata/isis_data/base
remote = asc_s3:asc-isisdata/usgs_data/base
......@@ -2,7 +2,6 @@
from multiprocessing import ProcessError
from multiprocessing.dummy import Process
from pprint import pprint
import logging as log
import os
import json
......@@ -11,14 +10,14 @@ import subprocess
import tempfile
from shutil import which
from os import path
from collections import OrderedDict
def find_conf():
from pathlib import Path
local_path = Path("rclone.conf")
install_path = Path(os.environ.get("CONDA_PREFIX")) / "etc" / "isis" / local_path
# this should be installed in scripts folder, so config is one directory up in etc
install_path = Path(os.path.dirname(__file__)) / '..' / "etc" / "isis" / 'rclone.conf'
repo_path = Path(os.path.dirname(__file__)) / '..' / 'config' / 'rclone.conf'
if local_path.exists():
return str(local_path)
elif repo_path.exists():
......@@ -26,7 +25,7 @@ def find_conf():
elif install_path.exists():
return str(install_path)
else:
return ""
raise RuntimeError("No config found, try passing it in explicitly with --config <path>")
# set log level to debug
......@@ -102,7 +101,9 @@ def create_rclone_arguments(destination, mission_name, dry_run=False, ntransfers
"""
log.debug(f"Creating RClone command for {mission_name}")
mission_dir_name, source_type = mission_name.replace(":", "").split("_")
if (mission_dir_name == "legacybase"):
# We still want things to go into base
mission_dir_name = "base"
log.debug(f"Mission_dir_name: {mission_dir_name}, source_type: {source_type}")
destination = os.path.join(destination, str(mission_dir_name).replace(":",""))
......@@ -111,7 +112,6 @@ def create_rclone_arguments(destination, mission_name, dry_run=False, ntransfers
if not os.path.exists(destination) and dry_run==False:
log.debug("{destination} does not exist, making directory")
print(destination)
os.makedirs(destination)
extra_args=[f"{mission_name}",f"{destination}", "--progress", f"--checkers={ntransfers}", f"--transfers={ntransfers}", "--track-renames", f"--log-level={log.getLevelName(log.getLogger().getEffectiveLevel())}"]
......@@ -144,8 +144,8 @@ def main(mission, dest, cfg_path, dry_run, ntransfers):
quit(-1)
log.debug(f"Remote Sources: {config_sources}")
supported_missions = {}
for source in config_sources:
supported_missions = OrderedDict({})
for source in sorted(config_sources, key=lambda x: x.split("_")[-1]):
parsed_name = source.split("_")
# If it is a mission, it should be in the format <mission_nam>_<source_type>
if len(parsed_name) == 2 and parsed_name[1] in ["usgs:", "naifKernels:"]:
......@@ -155,7 +155,7 @@ def main(mission, dest, cfg_path, dry_run, ntransfers):
log.debug(f"Supported missions:\n {supported_missions.keys()}")
log.debug(f"Complete Dictionary:\n {json.dumps(supported_missions, indent=2)}")
if not mission in supported_missions.keys():
if not mission in supported_missions.keys() and mission.upper() not in ("ALL", "LEGACYBASE"):
raise LookupError(f"{mission} is not in the list of supported missions: {supported_missions.keys()}")
if mission == "legacybase":
......@@ -176,10 +176,9 @@ def main(mission, dest, cfg_path, dry_run, ntransfers):
if __name__ == '__main__':
helpString = (
'''This will allow for a user to download isis data directly to their machine from the USGS S3 buckets as well as public end points\n\n'''
'''To use the download ISIS Data script you must supply 3 parameters with an optional 4th.\n\n'''
'''<rclone command> <Mission name> <destination to copy to> <--dry-run (optional)> \n'''
'''Example of how to run this program:\n'''
'''python downloadIsisData.py tgo ~/isisData/tgo\n\n'''
'''<Mission name> <destination to copy to> <--dry-run (optional)> \n\n'''
'''Example of how to run this program:\n\n'''
''' downloadIsisData tgo ~/isisData/\n\n'''
'''NOTE: if you would like to download the data for every mission supply the value ALL for the <Mission name> parameter''')
parser = argparse.ArgumentParser(description = helpString, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('mission', help='mission for files to be downloaded')
......@@ -206,3 +205,5 @@ if __name__ == '__main__':
log.basicConfig(**log_kwargs)
main(args.mission, args.dest, os.path.expanduser(args.config), args.dry_run, args.num_transfers)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment