diff --git a/data-access/engine/src/vlkb-obscore/src/database/SqlSchema_INSERT.cpp b/data-access/engine/src/vlkb-obscore/src/database/SqlSchema_INSERT.cpp
index f3ec20191a358e888dd030947e8ae336c0949409..c9d2f68ca4e3c6138812d0335d5d93e2b5ef1047 100644
--- a/data-access/engine/src/vlkb-obscore/src/database/SqlSchema_INSERT.cpp
+++ b/data-access/engine/src/vlkb-obscore/src/database/SqlSchema_INSERT.cpp
@@ -137,7 +137,7 @@ string region_stcs(vector<point2d> vert)
string to_sqlstring(string str) {return "\'" + str + "\'";}
// FIXME replace this
-string asSqlString(fitsfiles::key_values_by_type key_values, string key)
+string to_sql_value(fitsfiles::key_values_by_type key_values, string key)
{
if(key_values.strValues.count(key)>0)
{
@@ -157,7 +157,7 @@ string asSqlString(fitsfiles::key_values_by_type key_values, string key)
}
}
-
+/*
string get_wavelen(int precision, fitsfiles::key_values_by_type key_values)
{
const double UNIT_CONVERT = 0.001; // FIXME header-card unit [mm] but ObsCore em_ is [m]
@@ -183,21 +183,19 @@ string get_wavelen(int precision, fitsfiles::key_values_by_type key_values)
{
return to_string(precision, UNIT_CONVERT * key_values.doubleValues["WAVELEN"]);
}
-}
+}*/
-// trim from start (in place)
+/*/ trim from start (in place)
inline void ltrim(std::string &s) {
- s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) {
+ s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](char ch) {
return !std::isspace(ch);
}));
}
// trim from end (in place)
inline void rtrim(std::string &s) {
- s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) {
- return !std::isspace(ch);
- }).base(), s.end());
+ s.erase(std::find_if(s.rbegin(), s.rend(), [](char ch) {return !std::isspace(ch);} ).base(), s.end());
}
// trim from both ends (in place)
@@ -205,233 +203,119 @@ inline void trim(std::string &s) {
rtrim(s);
ltrim(s);
}
-
+*/
// returns band in wavelen [m]
double parse_transition(string trans)
{
// Surveys::survTransition "num unit" <-- for images
-
- trim(trans);
-
- if(!trans.empty())
- {
- string::size_type pos = trans.find(' ');
- std::string num = trans.substr(0, pos);
- std::string unit = trans.substr(pos+1);
-
- LOG_STREAM << "parse trans: " << num << " " << unit << endl;
-
- const double speed_of_light = 299792458.0; // [m/s] in vacuum
-
- double value = stod(num);
- if(unit.compare("um") == 0) return value * 1e-6;
- else if(unit.compare("mm") == 0) return value * 1e-3;
- else if(unit.compare("m") == 0) return value;
- else if(unit.compare("GHz") == 0) return (speed_of_light / (value * 1e9));
- else if(unit.compare("MHz") == 0) return (speed_of_light / (value * 1e6));
- else if(unit.compare("kHz") == 0) return (speed_of_light / (value * 1e3));
- else if(unit.compare("Hz") == 0) return (speed_of_light / (value));
- else
- return 0.0; // ignore
- }
+ string::size_type pos = trans.find(' ');
+ std::string num = trans.substr(0, pos);
+ std::string unit = trans.substr(pos+1);
+
+ LOG_STREAM << "parse trans: " << num << " " << unit << endl;
+
+ const double speed_of_light = 299792458.0; // [m/s] in vacuum
+
+ double value = stod(num);
+ if(unit.compare("um") == 0) return value * 1e-6;
+ else if(unit.compare("mm") == 0) return value * 1e-3;
+ else if(unit.compare("m") == 0) return value;
+ else if(unit.compare("GHz") == 0) return (speed_of_light / (value * 1e9));
+ else if(unit.compare("MHz") == 0) return (speed_of_light / (value * 1e6));
+ else if(unit.compare("kHz") == 0) return (speed_of_light / (value * 1e3));
+ else if(unit.compare("Hz") == 0) return (speed_of_light / (value));
else
- return 0.0;
+ throw invalid_argument("parse_transition failed");
}
//----------------------------------------------------------------------
-// public API
+// public API: ObsCore row
//----------------------------------------------------------------------
-void SqlSchema_INSERT::appendRow(/*const int hid, const int sid,*/
- const string& obscore_publisher,
- const string& obscore_access_format,
- const string& obscore_access_url,
- const Survey& surv, const string& authGroups,
- /*const*/ fitsfiles::Hdu& hdu,
- //n ObsCoreKeys ocKeys,
- const std::string& filename, const uintmax_t filesize)
-{
- LOG_trace(__func__);
+void SqlSchema_INSERT::appendRow(
+ const int calib_level,
+ const string& obs_collection,
+ const string& obs_id,
+ const string& obs_publisher_did,
- // header based computations
-
-#if 0
- const int specSystem = 1;// LSRK
- vector<struct Bounds> galBounds =
- legacy::calcBounds(hdu.m_header, "GALACTIC", specSystem);
- vector<struct Bounds> icrsBounds =
- legacy::calcBounds(hdu.m_header, "ICRS", specSystem);
- struct Vertices galVerts = legacy::calcVertices(hdu.m_header, "GALACTIC");
- struct Vertices icrsVerts = legacy::calcVertices(hdu.m_header, "ICRS");
-#else
- const string VELOLSRK{"System=VELO,StdOfRest=LSRK,Unit=km/s"};
- const string WAVELSRK{"System=WAVE,StdOfRest=LSRK,Unit=m"};
- // FIXME StdOfRest <--- SPECSYS
+ const string& access_url,
+ const string& access_format,
+ const uintmax_t access_estsize,
- vector<struct Bounds> galBounds = calc_bounds(hdu.m_header, "GALACTIC", VELOLSRK);
- vector<struct Bounds> icrsBounds = calc_bounds(hdu.m_header, "ICRS", WAVELSRK);
-
- vector<point2d> galVerts = calc_skyvertices(hdu.m_header, "GALACTIC");
- vector<point2d> icrsVerts = calc_skyvertices(hdu.m_header, "ICRS");
-
- reorder_vertices(galVerts);
- reorder_vertices(icrsVerts);
-#endif
-
- // construct publisherDID
-
- bool scramble = false;
- const std::string pubdid(dataset_id::create(surv.storagePath,filename,hdu.m_hdunum, scramble));
-
- ////////////////////////////////////////////////////////////////////////
-
- // set table values
- /*
- headersRow[headersColId::header_id] = to_string(hid);
- headersRow[headersColId::filename] = to_sqlstring(filename);
- headersRow[headersColId::hdunum] = to_string(hdu.m_hdunum);
- headersRow[headersColId::pubdid] = to_sqlstring(pubdid);
- headersRow[headersColId::header] = "$$" + hdu.m_header + "$$";
- headersRow[headersColId::survey_id] = to_string(sid);
-
- // cubebounds - GALACTIC
-
- boundsgalRow[bgheader_id] = to_string(hid);
- boundsgalRow[lfrom] = to_string(galBounds[0].low);//.x_from);
- boundsgalRow[lto] = to_string(galBounds[0].up);//.x_to);
- boundsgalRow[lunit] = to_sqlstring(galBounds[0].unit);//x_unit);
- boundsgalRow[bfrom] = to_string(galBounds[1].low);//y_from);
- boundsgalRow[bto] = to_string(galBounds[1].up);//y_to);
- boundsgalRow[bunit] = to_sqlstring(galBounds[1].unit);//y_unit);
- if(galBounds.size() >2)
- {
- boundsgalRow[vfrom] = to_string(galBounds[2].low);//vel_from);
- boundsgalRow[vto] = to_string(galBounds[2].up);//vel_to);
- boundsgalRow[vunit] = to_sqlstring(galBounds[2].unit);//vel_unit);
- }
- else
- {
- boundsgalRow[vfrom] = "NULL";
- boundsgalRow[vto] = "NULL";
- boundsgalRow[vunit] = "NULL";
- }
-
- // cubebounds - ICRS
-
- boundsicrsRow[biheader_id] = to_string(hid);
- boundsicrsRow[rfrom] = to_string(icrsBounds[0].low);//x_from);
- boundsicrsRow[rto] = to_string(icrsBounds[0].up);//x_to);
- boundsicrsRow[runit] = to_sqlstring(icrsBounds[0].unit);//x_unit);
- boundsicrsRow[dfrom] = to_string(icrsBounds[1].low);//y_from);
- boundsicrsRow[dto] = to_string(icrsBounds[1].up);//y_to);
- boundsicrsRow[dunit] = to_sqlstring(icrsBounds[1].unit);//y_unit);
-
- // vertices - GALACTIC
-
- verticesgalRow[vgheader_id] = to_string(hid);
- verticesgalRow[p1lon] = to_string(galVerts[0].lon);
- verticesgalRow[p1lat] = to_string(galVerts[0].lat);
- verticesgalRow[p2lon] = to_string(galVerts[1].lon);
- verticesgalRow[p2lat] = to_string(galVerts[1].lat);
- verticesgalRow[p3lon] = to_string(galVerts[2].lon);
- verticesgalRow[p3lat] = to_string(galVerts[2].lat);
- verticesgalRow[p4lon] = to_string(galVerts[3].lon);
- verticesgalRow[p4lat] = to_string(galVerts[3].lat);
+ const fitsfiles::Hdu& hdu,
+ const Survey& surv,
+ const string& auth_groups)
+{
+ LOG_trace(__func__);
- // vertices - ICRS
+ // mandatory
- verticesicrsRow[viheader_id] = to_string(hid);
- verticesicrsRow[p1ra] = to_string(icrsVerts[0].lon);
- verticesicrsRow[p1dec] = to_string(icrsVerts[0].lat);
- verticesicrsRow[p2ra] = to_string(icrsVerts[1].lon);
- verticesicrsRow[p2dec] = to_string(icrsVerts[1].lat);
- verticesicrsRow[p3ra] = to_string(icrsVerts[2].lon);
- verticesicrsRow[p3dec] = to_string(icrsVerts[2].lat);
- verticesicrsRow[p4ra] = to_string(icrsVerts[3].lon);
- verticesicrsRow[p4dec] = to_string(icrsVerts[3].lat);
- */
+ obscoreRow[SqlSchema::obscoreColId::calib_level] = to_string(calib_level);
+ obscoreRow[SqlSchema::obscoreColId::obs_collection] = to_sqlstring(obs_collection);
+ obscoreRow[SqlSchema::obscoreColId::obs_id] = to_sqlstring(obs_id);
+ obscoreRow[SqlSchema::obscoreColId::obs_publisher_id] = to_sqlstring(obs_publisher_did);
- // obscore - values from Surveys table
+ // optional
- string keyFacility = surv.fitskeyFacilityName;
- string keyInstrument = surv.fitskeyInstrumentName;
+ obscoreRow[SqlSchema::obscoreColId::access_url] = to_sqlstring(access_url);
+ obscoreRow[SqlSchema::obscoreColId::access_format] = to_sqlstring(access_format);
+ obscoreRow[SqlSchema::obscoreColId::access_estsize] = to_string(access_estsize);
- obscoreRow[dataproduct_type] = to_sqlstring(surv.dataproductType);
- obscoreRow[calib_level] = to_string(surv.calibLevel);
- obscoreRow[obs_collection] = to_sqlstring(surv.getObsCollection());
obscoreRow[o_ucd] = to_sqlstring(surv.oUcd);
- obscoreRow[policy] = to_sqlstring(authPolicyToSQLEnum(surv.authPolicy)) + "::auth_policy";
-
-
- // obscore - values derived from filename
-
- obscoreRow[obs_id] = to_sqlstring(filename.substr(0, filename.find_last_of(".")));
- obscoreRow[obs_publisher_id] = obscore_publisher.empty() ? to_sqlstring(pubdid) : to_sqlstring(obscore_publisher + "?" + pubdid);
- obscoreRow[access_url] = obscore_access_url.empty() ? "" : to_sqlstring(
- obscore_access_url + "/" + surv.storagePath + "/" + filename);
- obscoreRow[access_format] = to_sqlstring(obscore_access_format);
- obscoreRow[access_estsize] = to_string(filesize/1024); // [KB]
-
-
- // obscore - values taken from header cards
-
- obscoreRow[facility_name] = asSqlString(hdu.key_values, keyFacility);
- obscoreRow[instrument_name] = asSqlString(hdu.key_values, keyInstrument);
- obscoreRow[target_name] = asSqlString(hdu.key_values, "OBJECT");
- obscoreRow[s_xel1] = asSqlString(hdu.key_values, "NAXIS1");
- obscoreRow[s_xel2] = asSqlString(hdu.key_values, "NAXIS2");
- obscoreRow[s_resolution] = "NULL"; // double FIXME CDELT+CUNIT->arcsec ? which axis?
+ obscoreRow[dataproduct_type] = to_sqlstring(surv.dataproductType);
+ obscoreRow[facility_name] = to_sql_value(hdu.key_values, surv.fitskeyFacilityName);
+ obscoreRow[instrument_name] = to_sql_value(hdu.key_values, surv.fitskeyInstrumentName);
+ obscoreRow[target_name] = to_sql_value(hdu.key_values, "OBJECT");
- // obscore - vals computed from header
+ // sky & spectrum
-#define max(a,b) (a>b ? a : b)
- /* double ds_ra = (icrsBounds.x_from + icrsBounds.x_to)/2.0;
- double ds_dec = (icrsBounds.y_from + icrsBounds.y_to)/2.0;
- double ds_fov = max( fabs(icrsBounds.x_from - icrsBounds.x_to),
- fabs(icrsBounds.y_from - icrsBounds.y_to) );
- */
+ //const string VELOLSRK{"System=VELO,StdOfRest=LSRK,Unit=km/s"};
+ //vector<struct Bounds> galBounds = calc_bounds(hdu.m_header, "GALACTIC", VELOLSRK);
+ const string WAVELSRK{"System=WAVE,StdOfRest=LSRK,Unit=m"};
+ vector<struct Bounds> icrsBounds = calc_bounds(hdu.m_header, "ICRS", WAVELSRK);
double ds_ra = (icrsBounds[0].low + icrsBounds[0].up)/2.0;
double ds_dec = (icrsBounds[1].low + icrsBounds[1].up)/2.0;
- double ds_fov = max( fabs(icrsBounds[0].low - icrsBounds[0].up),
- fabs(icrsBounds[1].low - icrsBounds[1].up) );
-
-
- obscoreRow[s_ra] = to_string(ds_ra);
- obscoreRow[s_dec] = to_string(ds_dec);
- obscoreRow[s_fov] = to_string(ds_fov);
+ double ra_len = fabs(icrsBounds[0].low - icrsBounds[0].up);
+ double dec_len = fabs(icrsBounds[1].low - icrsBounds[1].up);
+ double ds_fov = (ra_len > dec_len) ? ra_len : dec_len;
+ obscoreRow[s_ra] = to_string(ds_ra);
+ obscoreRow[s_dec] = to_string(ds_dec);
+ obscoreRow[s_fov] = to_string(ds_fov);
+ obscoreRow[s_xel1] = to_sql_value(hdu.key_values, "NAXIS1");
+ obscoreRow[s_xel2] = to_sql_value(hdu.key_values, "NAXIS2");
+ obscoreRow[s_resolution] = "NULL"; // double FIXME CDELT+CUNIT->arcsec ? which axis?
+ vector<point2d> icrsVerts = calc_skyvertices(hdu.m_header, "ICRS");
+ reorder_vertices(icrsVerts);
obscoreRow[s_region] = to_sqlstring(region_stcs(icrsVerts));
- // polygon_region & coordinates are IA2-extensions
- obscoreRow[polygon_region_galactic] = to_sqlstring(region_spoly(galVerts));
- obscoreRow[polygon_region] = to_sqlstring(region_spoly(icrsVerts));
- obscoreRow[coordinates] = to_sqlstring("(" + obscoreRow[s_ra] + "," + obscoreRow[s_dec] + ")");
-
-
- // obscore - spectral axis
-
const int EM_PRECISION{10}; // nanometers
-
- if(icrsBounds.size() >= 3) // 3D cubes
+ if(icrsBounds.size() >= 3) // 3D cubes
{
- //obscoreRow[em_min] = to_string(galBounds[2].low);
- //obscoreRow[em_max] = to_string(galBounds[2].up);
+ //obscoreRow[em_min] = to_string(EM_PRECISION, galBounds[2].low);
+ //obscoreRow[em_max] = to_string(EM_PRECISION, galBounds[2].up);
obscoreRow[em_min] = to_string(EM_PRECISION, icrsBounds[2].low);
obscoreRow[em_max] = to_string(EM_PRECISION, icrsBounds[2].up);
- obscoreRow[em_res_power] = "NULL";//asSqlString(hdu.key_values, "CDELT3");
- obscoreRow[em_xel] = asSqlString(hdu.key_values, "NAXIS3");
+ obscoreRow[em_res_power] = "NULL";//to_sql_value(hdu.key_values, "CDELT3");
+ obscoreRow[em_xel] = to_sql_value(hdu.key_values, "NAXIS3");
}
else if(icrsBounds.size() == 2) // 2D images
{
- // surv.restFrequency [Hz] <-- is empty for 2D images: take from survTransition ?
// surv.survTransition "num unit" <-- for images
-
- double val = parse_transition(surv.survTransition);
- obscoreRow[em_min] = ((val==0.0) ? "NULL" : to_string(val)); //get_wavelen(EM_PRECISION, hdu.key_values);
+ try
+ {
+ double val{parse_transition(surv.survTransition)};
+ obscoreRow[em_min] = to_string(val); //get_wavelen(EM_PRECISION, hdu.key_values);
+ }
+ catch(const std::exception& ex)
+ {
+ LOG_STREAM << "parse_transition failed for " << surv.survTransition << endl;
+ obscoreRow[em_min] = "NULL";
+ }
obscoreRow[em_max] = obscoreRow[em_min];
obscoreRow[em_res_power] = "NULL";
obscoreRow[em_xel] = "NULL";
@@ -444,38 +328,42 @@ void SqlSchema_INSERT::appendRow(/*const int hid, const int sid,*/
obscoreRow[em_xel] = "NULL";
}
-
- // obscore - time
-
- string DATEOBS;
- string DATEEND;
-
- if(hdu.key_values.strValues["DATE-OBS"].size() > 0) DATEOBS = hdu.key_values.strValues["DATE-OBS"];
- if(hdu.key_values.strValues["DATE-END"].size() > 0) DATEEND = hdu.key_values.strValues["DATE-END"];
-
- obscore::ObsCoreTime time =
- obscore::calcObsCoreTime(hdu.key_values.strValues["DATE-OBS"], hdu.key_values.strValues["DATE-END"]);
-
- obscoreRow[t_min] = time.t_min_str; //to_string(time.t_min);
- obscoreRow[t_max] = time.t_max_str; //to_string(time.t_max);
+ // time
+
+ /* FIXME review
+ string DATEOBS;
+ string DATEEND;
+ if(hdu.key_values.strValues["DATE-OBS"].size() > 0) DATEOBS = hdu.key_values.strValues["DATE-OBS"];
+ if(hdu.key_values.strValues["DATE-END"].size() > 0) DATEEND = hdu.key_values.strValues["DATE-END"];
+ obscore::ObsCoreTime time = obscore::calcObsCoreTime(
+ hdu.key_values.strValues["DATE-OBS"],
+ hdu.key_values.strValues["DATE-END"]);*/
+ obscoreRow[t_min] = "NULL";//time.t_min_str; //to_string(time.t_min);
+ obscoreRow[t_max] = "NULL";//time.t_max_str; //to_string(time.t_max);
obscoreRow[t_exptime] = "NULL";// FIXME MALT gives 'nan' to_string(time.t_exptime);
obscoreRow[t_resolution] = "NULL";// HiGAL SQL error '-nan'; to_string(time.t_resolution);
obscoreRow[t_xel] = "NULL";// HiGAL Mosaic gives SQL error: 'too big integer'; to_string(time.t_xel);
-
- // obscore - stokes params
+ // polarization
obscoreRow[pol_states] = "NULL";
obscoreRow[pol_xel] = "NULL";
- // obscore - misc
+ // VLKB extensions
+ // GALACTIC coords
+ vector<point2d> galVerts = calc_skyvertices(hdu.m_header, "GALACTIC");
+ reorder_vertices(galVerts);
+ obscoreRow[polygon_region_galactic] = to_sqlstring(region_spoly(galVerts));
+ // pgSphere-types
+ obscoreRow[polygon_region] = to_sqlstring(region_spoly(icrsVerts));
+ obscoreRow[coordinates] = to_sqlstring("(" + obscoreRow[s_ra] + "," + obscoreRow[s_dec] + ")");
+ // authorization
+ obscoreRow[policy] = to_sqlstring(authPolicyToSQLEnum(surv.authPolicy)) + "::auth_policy";
+ obscoreRow[obscoreColId::groups] = to_sqlstring("{" + auth_groups + "}");
+ // ?
obscoreRow[proposal_id] = "NULL";
- obscoreRow[obscoreColId::groups] = to_sqlstring("{" + authGroups + "}");
-
- // end: all data set
-
SqlSchema::appendRow();
}
diff --git a/data-access/engine/src/vlkb-obscore/src/database/SqlSchema_INSERT.hpp b/data-access/engine/src/vlkb-obscore/src/database/SqlSchema_INSERT.hpp
index 1a9e915c0f40fc066ee2738aaa18fb2d3de4fcf1..e42963b7190c1f194dfda565739cfc47a68c46a0 100644
--- a/data-access/engine/src/vlkb-obscore/src/database/SqlSchema_INSERT.hpp
+++ b/data-access/engine/src/vlkb-obscore/src/database/SqlSchema_INSERT.hpp
@@ -17,14 +17,30 @@ class SqlSchema_INSERT : public SqlSchema
{
public:
- void appendRow(/*const int hid, const int sid,*/
- const std::string& obscore_publisher,
- const std::string& obscore_access_format,
- const std::string& obscore_access_url,
- const Survey& surv, const std::string& authGroups,
- /*const*/ fitsfiles::Hdu& hdu,
-// ObsCoreKeys ocKeys,
- const std::string& filename, const uintmax_t filesize);
+ // void appendRow(/*const int hid, const int sid,*/
+ // const std::string& obscore_publisher,
+ // const std::string& obscore_access_format,
+ // const std::string& obscore_access_url,
+ // const Survey& surv, const std::string& authGroups,
+ // /*const*/ fitsfiles::Hdu& hdu,
+ //// ObsCoreKeys ocKeys,
+ // const std::string& filename, const uintmax_t filesize);
+
+ void appendRow(
+ // mandatory
+ const int calib_level,
+ const std::string& obs_collection,
+ const std::string& obs_id,
+ const std::string& obs_publisher_did,
+ // optional
+ const std::string& access_url,
+ const std::string& access_format,
+ const uintmax_t access_estsize,
+ // optional
+ const fitsfiles::Hdu& hdu, // header
+ const Survey& surv, // metadata
+ // optional
+ const std::string& auth_groups); // security
std::vector<std::string> getINSERT(void);
diff --git a/data-access/engine/src/vlkb-obscore/src/database/database.cpp b/data-access/engine/src/vlkb-obscore/src/database/database.cpp
index b884a1be0a90bc5acd142424b01705443f5089c5..7f08d6c51fc2de0739fc124a292030d151cabcf1 100644
--- a/data-access/engine/src/vlkb-obscore/src/database/database.cpp
+++ b/data-access/engine/src/vlkb-obscore/src/database/database.cpp
@@ -248,12 +248,11 @@ string database::dbListFiles(int sid, const string db_uri, const string db_schem
-
-// add survey
-// - get pattern from Surveys table
-// - list files by pattern
-// - for each file's HDU calc table-data
-// - add table-data as one row to each table
+/* FIXME implement add-survey optionally without need of csv-metadata with params:
+ * const string storage_filter, // identify files
+ * const int calib_level, // must provide calib-level for them
+ * const string obs_collection, // give it a name
+ */
void database::dbAddSurvey(int sid, const string groups,
const string obscore_publisher,
const string obscore_access_format,
@@ -262,62 +261,68 @@ void database::dbAddSurvey(int sid, const string groups,
{
LOG_trace(__func__);
+ // 1, load optional metadata (from Db or csv-file) if exist
DbConn db(db_uri, db_schema);
-
Survey surv = db.querySurveyAttributes(sid);
- string storageFilter = surv.getStorageFilter();
- std::vector<string> pathnames =
- fitsfiles::globVector(fitsdir + "/" + storageFilter);
- // if there are no files for sid-survey
+ // 2, storage-filter identifies the files: collect them
+ string storageFilter = surv.getStorageFilter();
+ std::vector<string> pathnames = fitsfiles::globVector(fitsdir + "/" + storageFilter);
if(pathnames.size() == 0)
{
LOG_STREAM << "Found " << pathnames.size() << " files in " << fitsdir + "/" + storageFilter << endl;
return;
}
- // key-values for obscore-table
- const fitsfiles::keys_by_type in_keys{
- ObsCoreKeys::add_str_keys(ObsCoreKeys::strKeys, std::set<std::string>{surv.fitskeyFacilityName, surv.fitskeyInstrumentName}),
- ObsCoreKeys::uintKeys,
- ObsCoreKeys::doubleKeys};
- // read all filtered pathnames and create 'INSERT INTO headers ...' SQL-statement
+ // 3,a, set obscore mandatory fields: collection-level (the same for all files in the collection)
+ const int calib_level = surv.calibLevel;
+ const string obs_collection = surv.getObsCollection();
+ // generate 'INSERT INTO obscore ...' SQL-statement
+ const fitsfiles::keys_by_type in_keys
+ {
+ ObsCoreKeys::add_str_keys(
+ ObsCoreKeys::strKeys, std::set<std::string>{surv.fitskeyFacilityName, surv.fitskeyInstrumentName}),
+ ObsCoreKeys::uintKeys,
+ ObsCoreKeys::doubleKeys
+ };
SqlSchema_INSERT cmdInsert;
-
- //int max_hid = db.queryMaxHid();
- //LOG_STREAM << "max_hid: " << max_hid << endl;
-
+ const string access_format{obscore_access_format};
for(unsigned int i=0; i < pathnames.size(); i++)
{
+ // do optional obscore::access_* fields
string pathname = pathnames.at(i);
- uintmax_t filesize(fitsfiles::fileSize(pathname));
-
+ const uintmax_t access_estsize{fitsfiles::fileSize(pathname)/1024};
+ // FIXME fitsfiles::globVector() should return rel-pathnames already
string c_pathname = pathname;
string filename = basename((char*)c_pathname.c_str());
+ string rel_pathname{surv.storagePath + "/" + filename};
+ const string access_url{obscore_access_url.empty()
+ ? nullptr : obscore_access_url + "/" + rel_pathname};
- try
- {
- // read all HDU's
+ // 3,b, set obscore mandatory fields: file-dependent
+ const string obs_id = rel_pathname.substr(0, rel_pathname.find_last_of("."));
+ const string obs_publisher_did = obscore_publisher + "?" + rel_pathname;
- std::vector<fitsfiles::Hdu> allHdus = fitsfiles::fname2hdrstr(
- pathname, max_hdupos,
- &in_keys);
- for(unsigned int i=0; i<allHdus.size(); i++)
- {
- fitsfiles::Hdu hdu = allHdus.at(i);
+ // 4. set optional values which are available (in header or in metadata)
+ try
+ {
+ const std::vector<fitsfiles::Hdu> all_hdu{fitsfiles::fname2hdrstr(pathname, max_hdupos,&in_keys)};
+ for(fitsfiles::Hdu hdu : all_hdu)
+ {
cmdInsert.appendRow(
- //0,//++max_hid,
- //sid,
- obscore_publisher,
- obscore_access_format,
- obscore_access_url,
- surv, groups, hdu, filename, filesize);
+ calib_level,
+ obs_collection,
+ obs_id,
+ obs_publisher_did,
+ access_url, access_format, access_estsize,
+ hdu, surv,
+ groups);
}
}
catch (std::exception const &e)