vlkb-obscore: 1. config *_surveys -> *_datasets ; 2. implements ingestion of...

vlkb-obscore: 1. config *_surveys -> *_datasets ; 2. implements ingestion of given extension HDU (--extnum)

data-access/engine/src/common/include/fitsfiles.hpp

@@ -39,7 +39,7 @@ namespace fitsfiles
       key_values_by_type key_values;
    };
 
-   std::vector<Hdu> fname2hdrstr(std::string filename, unsigned int maxHduPos, const keys_by_type *keys = nullptr);
+   std::vector<Hdu> fname2hdrstr(std::string filename, unsigned int maxHduPos, unsigned int minHduPos = 1, const keys_by_type *keys = nullptr);
 
 
    // for services

data-access/engine/src/common/src/fitsfiles.cpp

@@ -359,7 +359,7 @@ string fitsfiles::append_card_if_not_in_header(string header, const vector<fits_
 // deprecated
 
 vector<fitsfiles::Hdu> fitsfiles::fname2hdrstr(
-      string filename, unsigned int maxHduPos,
+      string filename, unsigned int maxHduPos, unsigned int minHduPos,
       const keys_by_type* keys)
 {
    LOG_trace(__func__);
@@ -369,7 +369,7 @@ vector<fitsfiles::Hdu> fitsfiles::fname2hdrstr(
 
    // FIXME enhance fits::header to avoid open the file at each cycle
    unsigned int hdupos;
-   for(hdupos = 1; hdupos <= maxHduPos; hdupos++)
+   for(hdupos = minHduPos; hdupos <= maxHduPos; hdupos++)
    {
       fits::header hdr(filename, hdupos);
       // FIXME catch run-time except if not IMAGE_HDU -> how to deal with this?

data-access/engine/src/vlkb-obscore/src/config.hpp

@@ -28,7 +28,7 @@ class config
    private:
       std::string value(std::string key) {return m_settings.at(key);}
 
-      const std::string fits_dir{"fits_path_surveys"};
+      const std::string fits_dir{"fits_path_datasets"};
 
       const std::string pg_uri{"pg_uri"};
       const std::string pg_schema{"pg_schema"};
@@ -48,7 +48,7 @@ class config
 
       std::map<const std::string, std::string> m_settings 
       {
-         {fits_dir, "/srv/surveys"},
+         {fits_dir, "/srv/datasets"},
 
             {pg_uri,    empty_string},
             {pg_schema, empty_string},

data-access/engine/src/vlkb-obscore/src/database.hpp

@@ -15,7 +15,8 @@ namespace database
          const std::string obscore_access_format,
          const std::string remote_fitsdir,
          const std::string db_uri, const std::string db_schema,
-         const std::string fitsdir, int max_hdupos);
+         const std::string fitsdir,
+			int max_hdupos, int min_hdupos = 1);
 
    void dbModifyGroups(int sid, const std::string groups,
          const std::string obscore_publisher,

data-access/engine/src/vlkb-obscore/src/database/database.cpp

@@ -257,7 +257,8 @@ void database::dbAddSurvey(int sid, const string groups,
       const string obscore_publisher,
       const string obscore_access_format,
       const string obscore_access_url,
-      const string db_uri, const string db_schema, const string fitsdir, int max_hdupos)
+      const string db_uri, const string db_schema, const string fitsdir,
+		int max_hdupos, int min_hdupos)
 {
    LOG_trace(__func__);
 
@@ -313,7 +314,8 @@ void database::dbAddSurvey(int sid, const string groups,
       // 4. set optional values which are available (in header or in metadata)
       try
       {
-         const std::vector<fitsfiles::Hdu> all_hdu{fitsfiles::fname2hdrstr(pathname, max_hdupos,&in_keys)};
+			int min_hdupos = 1;
+         const std::vector<fitsfiles::Hdu> all_hdu{fitsfiles::fname2hdrstr(pathname, max_hdupos, min_hdupos, &in_keys)};
 
          for(fitsfiles::Hdu hdu : all_hdu)
          {

data-access/engine/src/vlkb-obscore/src/main.cpp

@@ -237,7 +237,14 @@ int cmd_dbFiles(int argc, char * argv[])
 
 int cmd_dbAdd(int argc, char * argv[])
 {
-   const int max_hdupos = 1; // FIXME add as param to the command
+   const string usage_str{"Usage: dbadd <SID> [SID_to]\n"
+              	           "       dbadd <SID> [--extnum N]\n"
+             	           "inserts survey metadata to database\n"
+             	           "works on Primary HDU unless 'extnum' given:\n"
+             	           "--extnum N where N=1..n (1 is the first extension after the Primary HDU)"};
+ 
+   int max_hdupos = 1;
+   int min_hdupos = 1;
    const string groups = ""; // NOTE: uses separate cmd 'dbmodgroups' to add groups to enable access to PRIVATE surveys
    int sid_from, sid_to;
    int rc = 0;
@@ -254,13 +261,25 @@ int cmd_dbAdd(int argc, char * argv[])
         	sid_to   = std::stoi(argv[2]);
          break;
 
+      case 4:
+         sid_from = std::stoi(argv[1]);
+			if(0 == string{"--extnum"}.compare(string{argv[2]}))
+			{
+         	min_hdupos = max_hdupos = 1 + std::stoi(argv[3]);
+			}
+			else
+			{
+				cout << "--extnum expected\n" << usage_str << endl;
+         	return vlkb::EXIT_WITH_USAGE;
+			}
+         break;
+
       default:
-         cout << "Usage: dbadd <SID-from> [SID-to]" << endl
-            << "inserts survey(s) metadata to database" << endl;
-         rc = vlkb::EXIT_WITH_USAGE;
+			cout << usage_str << endl;
+         return vlkb::EXIT_WITH_USAGE;
    }
 
-   OUT_STREAM << "This will add all data into database, (even if already exists!), confirm with 'yes':" << endl;
+   OUT_STREAM << "This will add all data into database. Confirm with 'yes':" << endl;
    string answer;
    std::cin >> answer;
    if(answer.compare("yes") != 0) return vlkb::EXIT_WITH_ERROR; 
@@ -271,7 +290,7 @@ int cmd_dbAdd(int argc, char * argv[])
       {
          database::dbAddSurvey(i, groups,
                vlkb::conf.getObsCorePublisher(), vlkb::conf.getObsCoreAccessFormat(), vlkb::conf.getObscoreAccessUrl(),
-               vlkb::conf.getDbUri(WITH_PASSWORD), vlkb::conf.getDbSchema(), vlkb::conf.getFitsDir(), max_hdupos);
+               vlkb::conf.getDbUri(WITH_PASSWORD), vlkb::conf.getDbSchema(), vlkb::conf.getFitsDir(), max_hdupos, min_hdupos);
       }
       catch(exception& e)
       {