Skip to content
Snippets Groups Projects
Commit 1f569fdf authored by Robert Butora's avatar Robert Butora
Browse files

vlkb-obscore: inserts em_min/_max from metadata-csv (rest)frequency column

parent a18a0e49
Branches
Tags
No related merge requests found
Show whitespace changes
Inline Side-by-side
data-access/engine/src/vlkb-obscore/src/database/SqlSchema_INSERT.cpp
+ 107
− 219
View file @ 1f569fdf
......@@ -137,7 +137,7 @@ string region_stcs(vector<point2d> vert)
string to_sqlstring(string str) {return "\'" + str + "\'";}
// FIXME replace this
string asSqlString(fitsfiles::key_values_by_type key_values, string key)
string to_sql_value(fitsfiles::key_values_by_type key_values, string key)
{
if(key_values.strValues.count(key)>0)
{
......@@ -157,7 +157,7 @@ string asSqlString(fitsfiles::key_values_by_type key_values, string key)
}
}
/*
string get_wavelen(int precision, fitsfiles::key_values_by_type key_values)
{
const double UNIT_CONVERT = 0.001; // FIXME header-card unit [mm] but ObsCore em_ is [m]
......@@ -183,21 +183,19 @@ string get_wavelen(int precision, fitsfiles::key_values_by_type key_values)
{
return to_string(precision, UNIT_CONVERT * key_values.doubleValues["WAVELEN"]);
}
}
}*/
// trim from start (in place)
/*/ trim from start (in place)
inline void ltrim(std::string &s) {
s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) {
s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](char ch) {
return !std::isspace(ch);
}));
}
// trim from end (in place)
inline void rtrim(std::string &s) {
s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) {
return !std::isspace(ch);
}).base(), s.end());
s.erase(std::find_if(s.rbegin(), s.rend(), [](char ch) {return !std::isspace(ch);} ).base(), s.end());
}
// trim from both ends (in place)
......@@ -205,17 +203,12 @@ inline void trim(std::string &s) {
rtrim(s);
ltrim(s);
}
*/
// returns band in wavelen [m]
double parse_transition(string trans)
{
// Surveys::survTransition "num unit" <-- for images
trim(trans);
if(!trans.empty())
{
string::size_type pos = trans.find(' ');
std::string num = trans.substr(0, pos);
std::string unit = trans.substr(pos+1);
......@@ -233,205 +226,96 @@ double parse_transition(string trans)
else if(unit.compare("kHz") == 0) return (speed_of_light / (value * 1e3));
else if(unit.compare("Hz") == 0) return (speed_of_light / (value));
else
return 0.0; // ignore
}
else
return 0.0;
throw invalid_argument("parse_transition failed");
}
//----------------------------------------------------------------------
// public API
// public API: ObsCore row
//----------------------------------------------------------------------
void SqlSchema_INSERT::appendRow(/*const int hid, const int sid,*/
const string& obscore_publisher,
const string& obscore_access_format,
const string& obscore_access_url,
const Survey& surv, const string& authGroups,
/*const*/ fitsfiles::Hdu& hdu,
//n ObsCoreKeys ocKeys,
const std::string& filename, const uintmax_t filesize)
{
LOG_trace(__func__);
void SqlSchema_INSERT::appendRow(
const int calib_level,
const string& obs_collection,
const string& obs_id,
const string& obs_publisher_did,
// header based computations
#if 0
const int specSystem = 1;// LSRK
vector<struct Bounds> galBounds =
legacy::calcBounds(hdu.m_header, "GALACTIC", specSystem);
vector<struct Bounds> icrsBounds =
legacy::calcBounds(hdu.m_header, "ICRS", specSystem);
struct Vertices galVerts = legacy::calcVertices(hdu.m_header, "GALACTIC");
struct Vertices icrsVerts = legacy::calcVertices(hdu.m_header, "ICRS");
#else
const string VELOLSRK{"System=VELO,StdOfRest=LSRK,Unit=km/s"};
const string WAVELSRK{"System=WAVE,StdOfRest=LSRK,Unit=m"};
// FIXME StdOfRest <--- SPECSYS
const string& access_url,
const string& access_format,
const uintmax_t access_estsize,
vector<struct Bounds> galBounds = calc_bounds(hdu.m_header, "GALACTIC", VELOLSRK);
vector<struct Bounds> icrsBounds = calc_bounds(hdu.m_header, "ICRS", WAVELSRK);
const fitsfiles::Hdu& hdu,
const Survey& surv,
vector<point2d> galVerts = calc_skyvertices(hdu.m_header, "GALACTIC");
vector<point2d> icrsVerts = calc_skyvertices(hdu.m_header, "ICRS");
reorder_vertices(galVerts);
reorder_vertices(icrsVerts);
#endif
// construct publisherDID
bool scramble = false;
const std::string pubdid(dataset_id::create(surv.storagePath,filename,hdu.m_hdunum, scramble));
////////////////////////////////////////////////////////////////////////
// set table values
/*
headersRow[headersColId::header_id] = to_string(hid);
headersRow[headersColId::filename] = to_sqlstring(filename);
headersRow[headersColId::hdunum] = to_string(hdu.m_hdunum);
headersRow[headersColId::pubdid] = to_sqlstring(pubdid);
headersRow[headersColId::header] = "$$" + hdu.m_header + "$$";
headersRow[headersColId::survey_id] = to_string(sid);
// cubebounds - GALACTIC
boundsgalRow[bgheader_id] = to_string(hid);
boundsgalRow[lfrom] = to_string(galBounds[0].low);//.x_from);
boundsgalRow[lto] = to_string(galBounds[0].up);//.x_to);
boundsgalRow[lunit] = to_sqlstring(galBounds[0].unit);//x_unit);
boundsgalRow[bfrom] = to_string(galBounds[1].low);//y_from);
boundsgalRow[bto] = to_string(galBounds[1].up);//y_to);
boundsgalRow[bunit] = to_sqlstring(galBounds[1].unit);//y_unit);
if(galBounds.size() >2)
{
boundsgalRow[vfrom] = to_string(galBounds[2].low);//vel_from);
boundsgalRow[vto] = to_string(galBounds[2].up);//vel_to);
boundsgalRow[vunit] = to_sqlstring(galBounds[2].unit);//vel_unit);
}
else
const string& auth_groups)
{
boundsgalRow[vfrom] = "NULL";
boundsgalRow[vto] = "NULL";
boundsgalRow[vunit] = "NULL";
}
// cubebounds - ICRS
boundsicrsRow[biheader_id] = to_string(hid);
boundsicrsRow[rfrom] = to_string(icrsBounds[0].low);//x_from);
boundsicrsRow[rto] = to_string(icrsBounds[0].up);//x_to);
boundsicrsRow[runit] = to_sqlstring(icrsBounds[0].unit);//x_unit);
boundsicrsRow[dfrom] = to_string(icrsBounds[1].low);//y_from);
boundsicrsRow[dto] = to_string(icrsBounds[1].up);//y_to);
boundsicrsRow[dunit] = to_sqlstring(icrsBounds[1].unit);//y_unit);
// vertices - GALACTIC
verticesgalRow[vgheader_id] = to_string(hid);
verticesgalRow[p1lon] = to_string(galVerts[0].lon);
verticesgalRow[p1lat] = to_string(galVerts[0].lat);
verticesgalRow[p2lon] = to_string(galVerts[1].lon);
verticesgalRow[p2lat] = to_string(galVerts[1].lat);
verticesgalRow[p3lon] = to_string(galVerts[2].lon);
verticesgalRow[p3lat] = to_string(galVerts[2].lat);
verticesgalRow[p4lon] = to_string(galVerts[3].lon);
verticesgalRow[p4lat] = to_string(galVerts[3].lat);
// vertices - ICRS
verticesicrsRow[viheader_id] = to_string(hid);
verticesicrsRow[p1ra] = to_string(icrsVerts[0].lon);
verticesicrsRow[p1dec] = to_string(icrsVerts[0].lat);
verticesicrsRow[p2ra] = to_string(icrsVerts[1].lon);
verticesicrsRow[p2dec] = to_string(icrsVerts[1].lat);
verticesicrsRow[p3ra] = to_string(icrsVerts[2].lon);
verticesicrsRow[p3dec] = to_string(icrsVerts[2].lat);
verticesicrsRow[p4ra] = to_string(icrsVerts[3].lon);
verticesicrsRow[p4dec] = to_string(icrsVerts[3].lat);
*/
// obscore - values from Surveys table
string keyFacility = surv.fitskeyFacilityName;
string keyInstrument = surv.fitskeyInstrumentName;
obscoreRow[dataproduct_type] = to_sqlstring(surv.dataproductType);
obscoreRow[calib_level] = to_string(surv.calibLevel);
obscoreRow[obs_collection] = to_sqlstring(surv.getObsCollection());
obscoreRow[o_ucd] = to_sqlstring(surv.oUcd);
obscoreRow[policy] = to_sqlstring(authPolicyToSQLEnum(surv.authPolicy)) + "::auth_policy";
LOG_trace(__func__);
// obscore - values derived from filename
// mandatory
obscoreRow[obs_id] = to_sqlstring(filename.substr(0, filename.find_last_of(".")));
obscoreRow[obs_publisher_id] = obscore_publisher.empty() ? to_sqlstring(pubdid) : to_sqlstring(obscore_publisher + "?" + pubdid);
obscoreRow[access_url] = obscore_access_url.empty() ? "" : to_sqlstring(
obscore_access_url + "/" + surv.storagePath + "/" + filename);
obscoreRow[access_format] = to_sqlstring(obscore_access_format);
obscoreRow[access_estsize] = to_string(filesize/1024); // [KB]
obscoreRow[SqlSchema::obscoreColId::calib_level] = to_string(calib_level);
obscoreRow[SqlSchema::obscoreColId::obs_collection] = to_sqlstring(obs_collection);
obscoreRow[SqlSchema::obscoreColId::obs_id] = to_sqlstring(obs_id);
obscoreRow[SqlSchema::obscoreColId::obs_publisher_id] = to_sqlstring(obs_publisher_did);
// optional
// obscore - values taken from header cards
obscoreRow[SqlSchema::obscoreColId::access_url] = to_sqlstring(access_url);
obscoreRow[SqlSchema::obscoreColId::access_format] = to_sqlstring(access_format);
obscoreRow[SqlSchema::obscoreColId::access_estsize] = to_string(access_estsize);
obscoreRow[facility_name] = asSqlString(hdu.key_values, keyFacility);
obscoreRow[instrument_name] = asSqlString(hdu.key_values, keyInstrument);
obscoreRow[target_name] = asSqlString(hdu.key_values, "OBJECT");
obscoreRow[s_xel1] = asSqlString(hdu.key_values, "NAXIS1");
obscoreRow[s_xel2] = asSqlString(hdu.key_values, "NAXIS2");
obscoreRow[s_resolution] = "NULL"; // double FIXME CDELT+CUNIT->arcsec ? which axis?
obscoreRow[o_ucd] = to_sqlstring(surv.oUcd);
obscoreRow[dataproduct_type] = to_sqlstring(surv.dataproductType);
obscoreRow[facility_name] = to_sql_value(hdu.key_values, surv.fitskeyFacilityName);
obscoreRow[instrument_name] = to_sql_value(hdu.key_values, surv.fitskeyInstrumentName);
obscoreRow[target_name] = to_sql_value(hdu.key_values, "OBJECT");
// obscore - vals computed from header
// sky & spectrum
#define max(a,b) (a>b ? a : b)
/* double ds_ra = (icrsBounds.x_from + icrsBounds.x_to)/2.0;
double ds_dec = (icrsBounds.y_from + icrsBounds.y_to)/2.0;
double ds_fov = max( fabs(icrsBounds.x_from - icrsBounds.x_to),
fabs(icrsBounds.y_from - icrsBounds.y_to) );
*/
//const string VELOLSRK{"System=VELO,StdOfRest=LSRK,Unit=km/s"};
//vector<struct Bounds> galBounds = calc_bounds(hdu.m_header, "GALACTIC", VELOLSRK);
const string WAVELSRK{"System=WAVE,StdOfRest=LSRK,Unit=m"};
vector<struct Bounds> icrsBounds = calc_bounds(hdu.m_header, "ICRS", WAVELSRK);
double ds_ra = (icrsBounds[0].low + icrsBounds[0].up)/2.0;
double ds_dec = (icrsBounds[1].low + icrsBounds[1].up)/2.0;
double ds_fov = max( fabs(icrsBounds[0].low - icrsBounds[0].up),
fabs(icrsBounds[1].low - icrsBounds[1].up) );
double ra_len = fabs(icrsBounds[0].low - icrsBounds[0].up);
double dec_len = fabs(icrsBounds[1].low - icrsBounds[1].up);
double ds_fov = (ra_len > dec_len) ? ra_len : dec_len;
obscoreRow[s_ra] = to_string(ds_ra);
obscoreRow[s_dec] = to_string(ds_dec);
obscoreRow[s_fov] = to_string(ds_fov);
obscoreRow[s_xel1] = to_sql_value(hdu.key_values, "NAXIS1");
obscoreRow[s_xel2] = to_sql_value(hdu.key_values, "NAXIS2");
obscoreRow[s_resolution] = "NULL"; // double FIXME CDELT+CUNIT->arcsec ? which axis?
vector<point2d> icrsVerts = calc_skyvertices(hdu.m_header, "ICRS");
reorder_vertices(icrsVerts);
obscoreRow[s_region] = to_sqlstring(region_stcs(icrsVerts));
// polygon_region & coordinates are IA2-extensions
obscoreRow[polygon_region_galactic] = to_sqlstring(region_spoly(galVerts));
obscoreRow[polygon_region] = to_sqlstring(region_spoly(icrsVerts));
obscoreRow[coordinates] = to_sqlstring("(" + obscoreRow[s_ra] + "," + obscoreRow[s_dec] + ")");
// obscore - spectral axis
const int EM_PRECISION{10}; // nanometers
if(icrsBounds.size() >= 3) // 3D cubes
{
//obscoreRow[em_min] = to_string(galBounds[2].low);
//obscoreRow[em_max] = to_string(galBounds[2].up);
//obscoreRow[em_min] = to_string(EM_PRECISION, galBounds[2].low);
//obscoreRow[em_max] = to_string(EM_PRECISION, galBounds[2].up);
obscoreRow[em_min] = to_string(EM_PRECISION, icrsBounds[2].low);
obscoreRow[em_max] = to_string(EM_PRECISION, icrsBounds[2].up);
obscoreRow[em_res_power] = "NULL";//asSqlString(hdu.key_values, "CDELT3");
obscoreRow[em_xel] = asSqlString(hdu.key_values, "NAXIS3");
obscoreRow[em_res_power] = "NULL";//to_sql_value(hdu.key_values, "CDELT3");
obscoreRow[em_xel] = to_sql_value(hdu.key_values, "NAXIS3");
}
else if(icrsBounds.size() == 2) // 2D images
{
// surv.restFrequency [Hz] <-- is empty for 2D images: take from survTransition ?
// surv.survTransition "num unit" <-- for images
double val = parse_transition(surv.survTransition);
obscoreRow[em_min] = ((val==0.0) ? "NULL" : to_string(val)); //get_wavelen(EM_PRECISION, hdu.key_values);
try
{
double val{parse_transition(surv.survTransition)};
obscoreRow[em_min] = to_string(val); //get_wavelen(EM_PRECISION, hdu.key_values);
}
catch(const std::exception& ex)
{
LOG_STREAM << "parse_transition failed for " << surv.survTransition << endl;
obscoreRow[em_min] = "NULL";
}
obscoreRow[em_max] = obscoreRow[em_min];
obscoreRow[em_res_power] = "NULL";
obscoreRow[em_xel] = "NULL";
......@@ -444,38 +328,42 @@ void SqlSchema_INSERT::appendRow(/*const int hid, const int sid,*/
obscoreRow[em_xel] = "NULL";
}
// time
// obscore - time
/* FIXME review
string DATEOBS;
string DATEEND;
if(hdu.key_values.strValues["DATE-OBS"].size() > 0) DATEOBS = hdu.key_values.strValues["DATE-OBS"];
if(hdu.key_values.strValues["DATE-END"].size() > 0) DATEEND = hdu.key_values.strValues["DATE-END"];
obscore::ObsCoreTime time =
obscore::calcObsCoreTime(hdu.key_values.strValues["DATE-OBS"], hdu.key_values.strValues["DATE-END"]);
obscoreRow[t_min] = time.t_min_str; //to_string(time.t_min);
obscoreRow[t_max] = time.t_max_str; //to_string(time.t_max);
obscore::ObsCoreTime time = obscore::calcObsCoreTime(
hdu.key_values.strValues["DATE-OBS"],
hdu.key_values.strValues["DATE-END"]);*/
obscoreRow[t_min] = "NULL";//time.t_min_str; //to_string(time.t_min);
obscoreRow[t_max] = "NULL";//time.t_max_str; //to_string(time.t_max);
obscoreRow[t_exptime] = "NULL";// FIXME MALT gives 'nan' to_string(time.t_exptime);
obscoreRow[t_resolution] = "NULL";// HiGAL SQL error '-nan'; to_string(time.t_resolution);
obscoreRow[t_xel] = "NULL";// HiGAL Mosaic gives SQL error: 'too big integer'; to_string(time.t_xel);
// obscore - stokes params
// polarization
obscoreRow[pol_states] = "NULL";
obscoreRow[pol_xel] = "NULL";
// obscore - misc
// VLKB extensions
// GALACTIC coords
vector<point2d> galVerts = calc_skyvertices(hdu.m_header, "GALACTIC");
reorder_vertices(galVerts);
obscoreRow[polygon_region_galactic] = to_sqlstring(region_spoly(galVerts));
// pgSphere-types
obscoreRow[polygon_region] = to_sqlstring(region_spoly(icrsVerts));
obscoreRow[coordinates] = to_sqlstring("(" + obscoreRow[s_ra] + "," + obscoreRow[s_dec] + ")");
// authorization
obscoreRow[policy] = to_sqlstring(authPolicyToSQLEnum(surv.authPolicy)) + "::auth_policy";
obscoreRow[obscoreColId::groups] = to_sqlstring("{" + auth_groups + "}");
// ?
obscoreRow[proposal_id] = "NULL";
obscoreRow[obscoreColId::groups] = to_sqlstring("{" + authGroups + "}");
// end: all data set
SqlSchema::appendRow();
}
......
data-access/engine/src/vlkb-obscore/src/database/SqlSchema_INSERT.hpp
+ 24
− 8
View file @ 1f569fdf
......@@ -17,14 +17,30 @@ class SqlSchema_INSERT : public SqlSchema
{
public:
void appendRow(/*const int hid, const int sid,*/
const std::string& obscore_publisher,
const std::string& obscore_access_format,
const std::string& obscore_access_url,
const Survey& surv, const std::string& authGroups,
/*const*/ fitsfiles::Hdu& hdu,
// ObsCoreKeys ocKeys,
const std::string& filename, const uintmax_t filesize);
// void appendRow(/*const int hid, const int sid,*/
// const std::string& obscore_publisher,
// const std::string& obscore_access_format,
// const std::string& obscore_access_url,
// const Survey& surv, const std::string& authGroups,
// /*const*/ fitsfiles::Hdu& hdu,
//// ObsCoreKeys ocKeys,
// const std::string& filename, const uintmax_t filesize);
void appendRow(
// mandatory
const int calib_level,
const std::string& obs_collection,
const std::string& obs_id,
const std::string& obs_publisher_did,
// optional
const std::string& access_url,
const std::string& access_format,
const uintmax_t access_estsize,
// optional
const fitsfiles::Hdu& hdu, // header
const Survey& surv, // metadata
// optional
const std::string& auth_groups); // security
std::vector<std::string> getINSERT(void);
......
data-access/engine/src/vlkb-obscore/src/database/database.cpp
+ 43
− 38
View file @ 1f569fdf
......@@ -248,12 +248,11 @@ string database::dbListFiles(int sid, const string db_uri, const string db_schem
// add survey
// - get pattern from Surveys table
// - list files by pattern
// - for each file's HDU calc table-data
// - add table-data as one row to each table
/* FIXME implement add-survey optionally without need of csv-metadata with params:
* const string storage_filter, // identify files
* const int calib_level, // must provide calib-level for them
* const string obs_collection, // give it a name
*/
void database::dbAddSurvey(int sid, const string groups,
const string obscore_publisher,
const string obscore_access_format,
......@@ -262,62 +261,68 @@ void database::dbAddSurvey(int sid, const string groups,
{
LOG_trace(__func__);
// 1, load optional metadata (from Db or csv-file) if exist
DbConn db(db_uri, db_schema);
Survey surv = db.querySurveyAttributes(sid);
string storageFilter = surv.getStorageFilter();
std::vector<string> pathnames =
fitsfiles::globVector(fitsdir + "/" + storageFilter);
// if there are no files for sid-survey
// 2, storage-filter identifies the files: collect them
string storageFilter = surv.getStorageFilter();
std::vector<string> pathnames = fitsfiles::globVector(fitsdir + "/" + storageFilter);
if(pathnames.size() == 0)
{
LOG_STREAM << "Found " << pathnames.size() << " files in " << fitsdir + "/" + storageFilter << endl;
return;
}
// key-values for obscore-table
const fitsfiles::keys_by_type in_keys{
ObsCoreKeys::add_str_keys(ObsCoreKeys::strKeys, std::set<std::string>{surv.fitskeyFacilityName, surv.fitskeyInstrumentName}),
ObsCoreKeys::uintKeys,
ObsCoreKeys::doubleKeys};
// read all filtered pathnames and create 'INSERT INTO headers ...' SQL-statement
// 3,a, set obscore mandatory fields: collection-level (the same for all files in the collection)
const int calib_level = surv.calibLevel;
const string obs_collection = surv.getObsCollection();
// generate 'INSERT INTO obscore ...' SQL-statement
const fitsfiles::keys_by_type in_keys
{
ObsCoreKeys::add_str_keys(
ObsCoreKeys::strKeys, std::set<std::string>{surv.fitskeyFacilityName, surv.fitskeyInstrumentName}),
ObsCoreKeys::uintKeys,
ObsCoreKeys::doubleKeys
};
SqlSchema_INSERT cmdInsert;
//int max_hid = db.queryMaxHid();
//LOG_STREAM << "max_hid: " << max_hid << endl;
const string access_format{obscore_access_format};
for(unsigned int i=0; i < pathnames.size(); i++)
{
// do optional obscore::access_* fields
string pathname = pathnames.at(i);
uintmax_t filesize(fitsfiles::fileSize(pathname));
const uintmax_t access_estsize{fitsfiles::fileSize(pathname)/1024};
// FIXME fitsfiles::globVector() should return rel-pathnames already
string c_pathname = pathname;
string filename = basename((char*)c_pathname.c_str());
string rel_pathname{surv.storagePath + "/" + filename};
const string access_url{obscore_access_url.empty()
? nullptr : obscore_access_url + "/" + rel_pathname};
try
{
// read all HDU's
// 3,b, set obscore mandatory fields: file-dependent
const string obs_id = rel_pathname.substr(0, rel_pathname.find_last_of("."));
const string obs_publisher_did = obscore_publisher + "?" + rel_pathname;
std::vector<fitsfiles::Hdu> allHdus = fitsfiles::fname2hdrstr(
pathname, max_hdupos,
&in_keys);
for(unsigned int i=0; i<allHdus.size(); i++)
// 4. set optional values which are available (in header or in metadata)
try
{
fitsfiles::Hdu hdu = allHdus.at(i);
const std::vector<fitsfiles::Hdu> all_hdu{fitsfiles::fname2hdrstr(pathname, max_hdupos,&in_keys)};
for(fitsfiles::Hdu hdu : all_hdu)
{
cmdInsert.appendRow(
//0,//++max_hid,
//sid,
obscore_publisher,
obscore_access_format,
obscore_access_url,
surv, groups, hdu, filename, filesize);
calib_level,
obs_collection,
obs_id,
obs_publisher_did,
access_url, access_format, access_estsize,
hdu, surv,
groups);
}
}
catch (std::exception const &e)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment