From 70ec72a4de94990b63b7ade96d8ad8cdcc3a4fcc Mon Sep 17 00:00:00 2001 From: Amy Stamile <74275278+amystamile-usgs@users.noreply.github.com> Date: Thu, 17 Aug 2023 10:45:12 -0700 Subject: [PATCH] fixes rclone recopying existing kernels (#5255) * fixes rclone duplication * updated changelog * Updated script * Addressed PR feedback --- CHANGELOG.md | 1 + isis/config/rclone.conf | 246 ++++++++++++++++++++++++---------- isis/scripts/downloadIsisData | 52 +++---- 3 files changed, 197 insertions(+), 102 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac0de72e4a..bc91829f99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ release. ### Fixed - Updated History constructor to check for invalid BLOB before copying History BLOB to output cube [#4966](https://github.com/DOI-USGS/ISIS3/issues/4966) - Updated photomet MinnaertEmpirical model to support photemplate-style PVL format [#3621](https://github.com/DOI-USGS/ISIS3/issues/3621) +- Fixed downloadIsisData script copying existing files with every rclone download [#5245](https://github.com/DOI-USGS/ISIS3/issues/5245) - Fixed gaussstretch segmentation fault error and refactored gaussstretch files/tests to use gtest [#5240](https://github.com/DOI-USGS/ISIS3/issues/5240) - Fix matrix inversion errors in findfeatures due to bad FASTGEOM matrix transforms using a more robust implementation to detect these errors and throw exceptions. Images with these errors are captured and logged to the TONOTMATCHED file. Fixes [#4639](https://github.com/DOI-USGS/ISIS3/issues/4639) - Fixed findfeatures use of projected mosaics with correct check for TargetName in the Mapping labels. [#4772](https://github.com/DOI-USGS/ISIS3/issues/4772) diff --git a/isis/config/rclone.conf b/isis/config/rclone.conf index a40bfbfc8c..df0bc9cd7c 100644 --- a/isis/config/rclone.conf +++ b/isis/config/rclone.conf @@ -18,258 +18,364 @@ url = http://www.darts.isas.jaxa.jp/ [sbn] type =http -url =https://sbnarchive.psi.edu/pds3/ +url = https://sbnarchive.psi.edu/pds3/ -[control_usgs] +[base] +type = alias +remote = asc_s3:asc-isisdata/usgs_data/base/ + +[legacybase] +type = alias +remote = asc_s3:asc-isisdata/usgs_data/legacy_base/ + +[control] type = alias remote = asc_s3:asc-isisdata/usgs_data/control/ -[apollo15_usgs] +[apollo15] type = alias remote = asc_s3:asc-isisdata/usgs_data/apollo15/ -[apollo16_usgs] +[apollo16] type = alias remote = asc_s3:asc-isisdata/usgs_data/apollo16/ -[apollo17_usgs] +[apollo17] type = alias remote = asc_s3:asc-isisdata/usgs_data/apollo17/ -[lro_usgs] +[lro] type = alias remote = asc_s3:asc-isisdata/usgs_data/lro/ [tgo_naifKernels] -type = alias -remote = esa:/data/SPICE/ExoMars2016/kernels/ +type = combine +upstreams = kernels=esa:/data/SPICE/ExoMars2016/kernels/ [tgo_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/tgo/ +[tgo] +type = union +upstreams = tgo_usgs: tgo_naifKernels: + [dawn_naifKernels] -type = alias -remote = naif:/pub/naif/DAWN/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/DAWN/kernels/ [dawn_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/dawn/ [dawn_pck] -type =alias -remote = sbn:dawn/fc/DWNCHSPG_2/GEOMETRY/ +type = combine +upstreams = pck=sbn:dawn/fc/DWNCHSPG_2/GEOMETRY/ + +[dawn_kernel_combine] +type = combine +upstreams = kernels=dawn_pck: + +[dawn] +type = union +upstreams = dawn_kernel_combine: dawn_naifKernels: dawn_usgs: [cassini_naifKernels] -type = alias -remote = naif:/pub/naif/CASSINI/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/CASSINI/kernels/ [cassini_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/cassini/ +[cassini] +type = union +upstreams = cassini_usgs: cassini_naifKernels: + [hayabusa2_naifKernels] -type = alias -remote = jaxa:/pub/hayabusa2/spice_bundle/spice_kernels/ +type = combine +upstreams = kernels=jaxa:/pub/hayabusa2/spice_bundle/spice_kernels/ [hayabusa2_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/hayabusa2/ +[hayabusa2] +type = union +upstreams = hayabusa2_usgs: hayabusa2_naifKernels: + [galileo_naifKernels] -type = alias -remote = naif:/pub/naif/GLL/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/GLL/kernels/ [galileo_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/galileo/ +[galileo] +type = union +upstreams = galileo_usgs: galileo_naifKernels: + [juno_naifKernels] -type = alias -remote = naif:/pub/naif/pds/data/jno-j_e_ss-spice-6-v1.0/jnosp_1000/data/ +type = combine +upstreams = kernels=naif:/pub/naif/JUNO/kernels/ [juno_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/juno/ -[lo_usgs] +[juno] +type = union +upstreams = juno_usgs: juno_naifKernels: + +[lo] type = alias remote = asc_s3:asc-isisdata/usgs_data/lo/ [odyssey_naifKernels] -type = alias -remote = naif:/pub/naif/M01/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/M01/kernels/ [odyssey_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/odyssey/ +[odyssey] +type = union +upstreams = odyssey_usgs: odyssey_naifKernels: + [mro_naifKernels] -type = alias -remote = naif:/pub/naif/MRO/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/MRO/kernels/ [mro_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/mro/ +[mro] +type = union +upstreams = mro_usgs: mro_naifKernels: + [mex_naifKernels] -type = alias -remote = naif:/pub/naif/MEX/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/MEX/kernels/ [mex_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/mex/ -[legacybase_usgs] -type = alias -remote = asc_s3:asc-isisdata/usgs_data/legacy_base/ +[mex] +type = union +upstreams = mex_usgs: mex_naifKernels: [hayabusa_naifKernels] -type = alias -remote = jaxa:/pub/spice/HAYABUSA/kernels/ +type = combine +upstreams = kernels=jaxa:/pub/spice/HAYABUSA/kernels/ [hayabusa_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/hayabusa/ +[hayabusa] +type = union +upstreams = hayabusa_usgs: hayabusa_naifKernels: + [chandrayaan1_naifKernels] -type = alias -remote = esa:/data/SPICE/CHANDRAYAAN-1/kernels/ +type = combine +upstreams = kernels=esa:/data/SPICE/CHANDRAYAAN-1/kernels/ [chandrayaan1_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/chandrayaan1/ +[chandrayaan1] +type = union +upstreams = chandrayaan1_usgs: chandrayaan1_naifKernels: + [clementine1_naifKernels] -type = alias -remote = naif:/pub/naif/pds/data/clem1-l-spice-6-v1.0/clsp_1000/data/ +type = combine +upstreams = naif:/pub/naif/CLEMENTINE/kernels/ [clementine1_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/clementine1/ +[clementine1] +type = union +upstreams = clementine1_usgs: clementine1_naifKernels: + [kaguya_naifKernels] -type = alias -remote = jaxa:/pub/pds3/sln-l-spice-6-v1.0/slnsp_1000/data/ +type = combine +upstreams = kernels=jaxa:/pub/spice/SELENE/kernels/ [kaguya_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/kaguya/ +[kaguya] +type = union +upstreams = kaguya_usgs: kaguya_naifKernels: + [mariner10_naifKernels] -type = alias -remote = naif:/pub/naif/M10/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/M10/kernels/ [mariner10_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/mariner10/ -[mer_usgs] +[mariner10] +type = union +upstreams = mariner10_usgs: mariner10_naifKernels: + +[mer] type = alias remote = asc_s3:asc-isisdata/usgs_data/mer/ [messenger_naifKernels] -type = alias -remote = naif:/pub/naif/pds/data/mess-e_v_h-spice-6-v1.0/messsp_1000/data/ +type = combine +upstreams = kernels=naif:/pub/naif/pds/data/mess-e_v_h-spice-6-v1.0/messsp_1000/data/kernels/ [messenger_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/messenger/ +[messenger] +type = union +upstreams = messenger_usgs: messenger_naifKernels: + [mgs_naifKernels] -type = alias -remote = naif:/pub/naif/pds/data/mgs-m-spice-6-v1.0/mgsp_1000/data/ +type = combine +upstreams = kernels=naif:/pub/naif/pds/data/mgs-m-spice-6-v1.0/mgsp_1000/data/kernels/ [mgs_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/mgs/ +[mgs] +type = union +upstreams = mgs_usgs: mgs_naifKernels: + [near_naifKernels] -type = alias -remote = naif:/pub/naif/pds/data/near-a-spice-6-v1.0/nearsp_1000/data/ +type = combine +upstreams = kernels=naif:/pub/naif/pds/data/near-a-spice-6-v1.0/nearsp_1000/data/ [near_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/near/ +[near] +type = union +upstreams = near_usgs: near_naifKernels: + [newhorizons_naifKernels] -type = alias -remote = naif:/pub/naif/pds/data/nh-j_p_ss-spice-6-v1.0/nhsp_1000/data/ +type = combine +upstreams = kernels=naif:/pub/naif/NEWHORIZONS/kernels/ [newhorizons_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/newhorizons/ +[newhorizons] +type = union +upstreams = newhorizons_usgs: newhorizons_naifKernels: + [osirisrex_naifKernels] -type = alias -remote = naif:/pub/naif/pds/pds4/orex/orex_spice/spice_kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/pds/pds4/orex/orex_spice/spice_kernels/ [osirisrex_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/osirisrex/ -[rolo_usgs] +[osirisrex] +type = union +upstreams = osirisrex_usgs: osirisrex_naifKernels: + +[rolo] type = alias remote = asc_s3:asc-isisdata/usgs_data/rolo/ [rosetta_naifKernels] -type = alias -remote = naif:/pub/naif/ROSETTA/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/ROSETTA/kernels/ [rosetta_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/rosetta/ +[rosetta] +type = union +upstreams = rosetta_usgs: rosetta_naifKernels: + [smart1_naifKernels] -type = alias -remote = esa:/data/SPICE/SMART-1/kernels/ +type = combine +upstreams = kernels=esa:/data/SPICE/SMART-1/kernels/ [smart1_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/smart1/ +[smart1] +type = union +upstreams = smart1_usgs: smart1_naifKernels: + [viking1_naifKernels] -type = alias -remote = naif:/pub/naif/VIKING/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/VIKING/kernels/ [viking1_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/viking1/ +[viking1] +type = union +upstreams = viking1_usgs: viking1_naifKernels: + [viking2_naifKernels] -type = alias -remote = naif:/pub/naif/VIKING/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/VIKING/kernels/ [viking2_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/viking2/ +[viking2] +type = union +upstreams = viking2_usgs: viking2_naifKernels: + [voyager1_naifKernels] -type = alias -remote = naif:/pub/naif/VOYAGER/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/VOYAGER/kernels/ [voyager1_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/voyager1/ +[voyager1] +type = union +upstreams = voyager1_usgs: voyager1_naifKernels: + [voyager2_naifKernels] -type = alias -remote = naif:/pub/naif/VOYAGER/kernels/ +type = combine +upstreams = kernels=naif:/pub/naif/VOYAGER/kernels/ [voyager2_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/voyager2/ -[base_usgs] -type = alias -remote = asc_s3:asc-isisdata/usgs_data/base/ +[voyager2] +type = union +upstreams = voyager2_usgs: voyager2_naifKernels: [msl_usgs] type = alias remote = asc_s3:asc-isisdata/usgs_data/msl/ [msl_naifKernels] -type = alias -remote = naif:/pub/naif/MSL/kernels/ - +type = combine +upstreams = kernels=naif:/pub/naif/MSL/kernels/ +[msl] +type = union +upstreams = msl_usgs: msl_naifKernels: \ No newline at end of file diff --git a/isis/scripts/downloadIsisData b/isis/scripts/downloadIsisData index 6f23cce7e4..6f3e00849f 100755 --- a/isis/scripts/downloadIsisData +++ b/isis/scripts/downloadIsisData @@ -123,17 +123,14 @@ def create_rclone_arguments(destination, mission_name, parsedArgs, rclone_kwargs set of kernels that will be downloaded """ log.debug(f"Creating RClone command for {mission_name}") - mission_dir_name, source_type = mission_name.replace(":", "").split("_") + + mission_dir_name = mission_name - if (mission_dir_name == "legacybase"): + if (mission_name == "legacybase"): # We still want things to go into base mission_dir_name = "base" - - log.debug(f"Mission_dir_name: {mission_dir_name}, source_type: {source_type}") - destination = os.path.join(destination, str(mission_dir_name).replace(":","")) - if source_type == "naifKernels": - destination = os.path.join(destination, "kernels") + destination = os.path.join(destination, str(mission_dir_name)) if args.filter: filters = [f"- {arg}" for arg in args.filters] @@ -153,7 +150,7 @@ def create_rclone_arguments(destination, mission_name, parsedArgs, rclone_kwargs filter_list.append("- *") filter_args = [f'--filter={item}' for item in filter_list] - extra_args = [f"{mission_name}", + extra_args = [f"{mission_name}:", f"{destination}", "--progress", f"--checkers={parsedArgs.num_transfers}", @@ -189,34 +186,25 @@ def main(mission, dest, cfg_path, parsedArgs, kwargs): quit(-1) log.debug(f"Remote Sources: {config_sources}") - supported_missions = OrderedDict({}) - for source in sorted(config_sources, key=lambda x: x.split("_")[-1]): - parsed_name = source.split("_") - # If it is a mission, it should be in the format _ - if len(parsed_name) == 2 and parsed_name[1] in ["usgs:", "naifKernels:"]: - remotes_mission_name = parsed_name[0] - supported_missions[remotes_mission_name] = supported_missions.get(remotes_mission_name, []) + [source] + supported_missions = [] + for source in config_sources: + if not any(x in source for x in ["_", "esa", "naif", "jaxa", "sbn"]): + if source != "": + supported_missions.append(source.replace(":", "")) - log.debug(f"Supported missions:\n {supported_missions.keys()}") - log.debug(f"Complete Dictionary:\n {json.dumps(supported_missions, indent=2)}") + log.debug(f"Supported missions:\n {supported_missions}") - if not mission in supported_missions.keys() and mission.upper() not in ("ALL", "LEGACYBASE"): - raise LookupError(f"{mission} is not in the list of supported missions: {supported_missions.keys()}") + if not mission in supported_missions and mission.upper() not in ("ALL", "LEGACYBASE"): + raise LookupError(f"{mission} is not in the list of supported missions: {supported_missions}") - if mission == "legacybase": - args = create_rclone_arguments(dest, "legacybase_usgs:", parsedArgs, kwargs) - rclone(command=parsedArgs.command, extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False) - elif(mission.upper() == "ALL"): - supported_missions.pop("legacybase") - for mission, remotes in supported_missions.items(): - for remote in remotes: - args = create_rclone_arguments(dest, remote, parsedArgs, kwargs) - rclone(command=parsedArgs.command, extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False) - else: - for remote in supported_missions[mission]: - args = create_rclone_arguments(dest, remote, parsedArgs, kwargs) + if(mission.upper() == "ALL"): + supported_missions.remove("legacybase") + for mission in supported_missions: + args = create_rclone_arguments(dest, mission, parsedArgs, kwargs) rclone(command=parsedArgs.command, extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False) - + else: + args = create_rclone_arguments(dest, mission, parsedArgs, kwargs) + rclone(command=parsedArgs.command, extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False) if __name__ == '__main__': usageString = "usage: downloadIsisData mission dest [-h] [-v] [-n NUM_TRANSFERS] [--config CONFIG] [rclone flags]" -- GitLab