From 70b03e90663a83caf728df885b4cc542c8bd41cb Mon Sep 17 00:00:00 2001
From: Robert Butora <robert.butora@inaf.it>
Date: Fri, 20 Dec 2024 17:17:28 +0200
Subject: [PATCH] vlkb-obscore: 1. config *_surveys -> *_datasets ; 2.
implements ingestion of given extension HDU (--extnum)
---
.../engine/src/common/include/fitsfiles.hpp | 2 +-
.../engine/src/common/src/fitsfiles.cpp | 4 +--
.../engine/src/vlkb-obscore/src/config.hpp | 4 +--
.../engine/src/vlkb-obscore/src/database.hpp | 3 +-
.../vlkb-obscore/src/database/database.cpp | 6 ++--
.../engine/src/vlkb-obscore/src/main.cpp | 33 +++++++++++++++----
6 files changed, 37 insertions(+), 15 deletions(-)
diff --git a/data-access/engine/src/common/include/fitsfiles.hpp b/data-access/engine/src/common/include/fitsfiles.hpp
index 5fc90b0..717069d 100644
--- a/data-access/engine/src/common/include/fitsfiles.hpp
+++ b/data-access/engine/src/common/include/fitsfiles.hpp
@@ -39,7 +39,7 @@ namespace fitsfiles
key_values_by_type key_values;
};
- std::vector<Hdu> fname2hdrstr(std::string filename, unsigned int maxHduPos, const keys_by_type *keys = nullptr);
+ std::vector<Hdu> fname2hdrstr(std::string filename, unsigned int maxHduPos, unsigned int minHduPos = 1, const keys_by_type *keys = nullptr);
// for services
diff --git a/data-access/engine/src/common/src/fitsfiles.cpp b/data-access/engine/src/common/src/fitsfiles.cpp
index df4c9d5..e58683b 100644
--- a/data-access/engine/src/common/src/fitsfiles.cpp
+++ b/data-access/engine/src/common/src/fitsfiles.cpp
@@ -359,7 +359,7 @@ string fitsfiles::append_card_if_not_in_header(string header, const vector<fits_
// deprecated
vector<fitsfiles::Hdu> fitsfiles::fname2hdrstr(
- string filename, unsigned int maxHduPos,
+ string filename, unsigned int maxHduPos, unsigned int minHduPos,
const keys_by_type* keys)
{
LOG_trace(__func__);
@@ -369,7 +369,7 @@ vector<fitsfiles::Hdu> fitsfiles::fname2hdrstr(
// FIXME enhance fits::header to avoid open the file at each cycle
unsigned int hdupos;
- for(hdupos = 1; hdupos <= maxHduPos; hdupos++)
+ for(hdupos = minHduPos; hdupos <= maxHduPos; hdupos++)
{
fits::header hdr(filename, hdupos);
// FIXME catch run-time except if not IMAGE_HDU -> how to deal with this?
diff --git a/data-access/engine/src/vlkb-obscore/src/config.hpp b/data-access/engine/src/vlkb-obscore/src/config.hpp
index 40fe226..0d57397 100644
--- a/data-access/engine/src/vlkb-obscore/src/config.hpp
+++ b/data-access/engine/src/vlkb-obscore/src/config.hpp
@@ -28,7 +28,7 @@ class config
private:
std::string value(std::string key) {return m_settings.at(key);}
- const std::string fits_dir{"fits_path_surveys"};
+ const std::string fits_dir{"fits_path_datasets"};
const std::string pg_uri{"pg_uri"};
const std::string pg_schema{"pg_schema"};
@@ -48,7 +48,7 @@ class config
std::map<const std::string, std::string> m_settings
{
- {fits_dir, "/srv/surveys"},
+ {fits_dir, "/srv/datasets"},
{pg_uri, empty_string},
{pg_schema, empty_string},
diff --git a/data-access/engine/src/vlkb-obscore/src/database.hpp b/data-access/engine/src/vlkb-obscore/src/database.hpp
index d7a5f8e..0d1ae42 100644
--- a/data-access/engine/src/vlkb-obscore/src/database.hpp
+++ b/data-access/engine/src/vlkb-obscore/src/database.hpp
@@ -15,7 +15,8 @@ namespace database
const std::string obscore_access_format,
const std::string remote_fitsdir,
const std::string db_uri, const std::string db_schema,
- const std::string fitsdir, int max_hdupos);
+ const std::string fitsdir,
+ int max_hdupos, int min_hdupos = 1);
void dbModifyGroups(int sid, const std::string groups,
const std::string obscore_publisher,
diff --git a/data-access/engine/src/vlkb-obscore/src/database/database.cpp b/data-access/engine/src/vlkb-obscore/src/database/database.cpp
index cbf1a0c..a25b6ea 100644
--- a/data-access/engine/src/vlkb-obscore/src/database/database.cpp
+++ b/data-access/engine/src/vlkb-obscore/src/database/database.cpp
@@ -257,7 +257,8 @@ void database::dbAddSurvey(int sid, const string groups,
const string obscore_publisher,
const string obscore_access_format,
const string obscore_access_url,
- const string db_uri, const string db_schema, const string fitsdir, int max_hdupos)
+ const string db_uri, const string db_schema, const string fitsdir,
+ int max_hdupos, int min_hdupos)
{
LOG_trace(__func__);
@@ -313,7 +314,8 @@ void database::dbAddSurvey(int sid, const string groups,
// 4. set optional values which are available (in header or in metadata)
try
{
- const std::vector<fitsfiles::Hdu> all_hdu{fitsfiles::fname2hdrstr(pathname, max_hdupos,&in_keys)};
+ int min_hdupos = 1;
+ const std::vector<fitsfiles::Hdu> all_hdu{fitsfiles::fname2hdrstr(pathname, max_hdupos, min_hdupos, &in_keys)};
for(fitsfiles::Hdu hdu : all_hdu)
{
diff --git a/data-access/engine/src/vlkb-obscore/src/main.cpp b/data-access/engine/src/vlkb-obscore/src/main.cpp
index 6bfd5c0..9746dc3 100644
--- a/data-access/engine/src/vlkb-obscore/src/main.cpp
+++ b/data-access/engine/src/vlkb-obscore/src/main.cpp
@@ -237,7 +237,14 @@ int cmd_dbFiles(int argc, char * argv[])
int cmd_dbAdd(int argc, char * argv[])
{
- const int max_hdupos = 1; // FIXME add as param to the command
+ const string usage_str{"Usage: dbadd <SID> [SID_to]\n"
+ " dbadd <SID> [--extnum N]\n"
+ "inserts survey metadata to database\n"
+ "works on Primary HDU unless 'extnum' given:\n"
+ "--extnum N where N=1..n (1 is the first extension after the Primary HDU)"};
+
+ int max_hdupos = 1;
+ int min_hdupos = 1;
const string groups = ""; // NOTE: uses separate cmd 'dbmodgroups' to add groups to enable access to PRIVATE surveys
int sid_from, sid_to;
int rc = 0;
@@ -251,16 +258,28 @@ int cmd_dbAdd(int argc, char * argv[])
case 3:
sid_from = std::stoi(argv[1]);
- sid_to = std::stoi(argv[2]);
+ sid_to = std::stoi(argv[2]);
+ break;
+
+ case 4:
+ sid_from = std::stoi(argv[1]);
+ if(0 == string{"--extnum"}.compare(string{argv[2]}))
+ {
+ min_hdupos = max_hdupos = 1 + std::stoi(argv[3]);
+ }
+ else
+ {
+ cout << "--extnum expected\n" << usage_str << endl;
+ return vlkb::EXIT_WITH_USAGE;
+ }
break;
default:
- cout << "Usage: dbadd <SID-from> [SID-to]" << endl
- << "inserts survey(s) metadata to database" << endl;
- rc = vlkb::EXIT_WITH_USAGE;
+ cout << usage_str << endl;
+ return vlkb::EXIT_WITH_USAGE;
}
- OUT_STREAM << "This will add all data into database, (even if already exists!), confirm with 'yes':" << endl;
+ OUT_STREAM << "This will add all data into database. Confirm with 'yes':" << endl;
string answer;
std::cin >> answer;
if(answer.compare("yes") != 0) return vlkb::EXIT_WITH_ERROR;
@@ -271,7 +290,7 @@ int cmd_dbAdd(int argc, char * argv[])
{
database::dbAddSurvey(i, groups,
vlkb::conf.getObsCorePublisher(), vlkb::conf.getObsCoreAccessFormat(), vlkb::conf.getObscoreAccessUrl(),
- vlkb::conf.getDbUri(WITH_PASSWORD), vlkb::conf.getDbSchema(), vlkb::conf.getFitsDir(), max_hdupos);
+ vlkb::conf.getDbUri(WITH_PASSWORD), vlkb::conf.getDbSchema(), vlkb::conf.getFitsDir(), max_hdupos, min_hdupos);
}
catch(exception& e)
{
--
GitLab