GXP supfile integration and tests (#409)

* Updated stateAsJson to read both .sup and camera state jsons

* Added string sanitize function

* Moved statefile reader to utilities

* Added .sup filepath to linescan fixture

* Added test for model creation from sup file state

* Added sup file test data

* Added tests for sanitize and readFileInString

* Update hello.json

* Update UtilitiesTests.cpp

* Adds ability to modify GXP .sup Camera State using usgscsm_cam_test (#406)

* Adds ability to modify sup file with camera state or isd.

* Updated tests and documentation

* Added build artifact for usgscsm_cam_test executable

* Updated artifact target

* Added build artifact for usgscsm_cam_test executable (#408)

* Added build artifact for usgscsm_cam_test executable

* Updated artifact target

* Added test for supfile save and load

Co-authored-by: Amy Stamile <astamile@contractor.usgs.gov>
Co-authored-by: Amy Stamile <74275278+amystamile-usgs@users.noreply.github.com>

.appveyor.yml

@@ -30,6 +30,9 @@ artifacts:
     name: usgscsm.dll
   - path: build\ale\Release\ale.dll
     name: ale.dll
+  - path: build\Release\usgscsm_cam_test.exe
+    name: usgscsm_cam_test
+
 
 on_success:
 - cd ../

bin/usgscsm_cam_test.cc

@@ -3,24 +3,29 @@
 // Functionality:
 //--------------
 //
-// - Load a CSM model in ISD format or model state format, via:
+// - Load a CSM model in ISD format, model state, or GXP .sup file format, via:
 //   --model <model file>
 //
 // - Save the model state if invoked with:
 //   --output-model-state <model state .json file>
 //
+// - Modify GXP .sup file's model state with inputed model:
+//   --modify-sup-file <GXP .sup file>
+//
 // - Test projecting rays from the camera to ground, then back,
 //   and compare with the original pixel values.
 
 #include <UsgsAstroPlugin.h>
 #include <RasterGM.h>
 #include <UsgsAstroLsSensorModel.h>
+#include <Utilities.h>
 
 #include <iostream>
 #include <fstream>
 
 struct Options {
   std::string model;              // the .json file in isd or model state format
+  std::string modify_sup_file;    // the .sup file needing a modified model state
   std::string output_model_state; // the output model state in .json format
   int sample_rate;
   double subpixel_offset, height_above_datum, desired_precision;
@@ -90,6 +95,7 @@ bool parseOptions(int argc, char **argv, Options & opt) {
     parsed_options["desired-precision"] = "0.001"; // set default value
 
   // Collect all other option values. If not set, the values will default to 0.
+  opt.modify_sup_file    = parsed_options["modify-sup-file"];
   opt.output_model_state = parsed_options["output-model-state"];
   opt.sample_rate        = sample_rate_double;
   opt.subpixel_offset    = atof(parsed_options["subpixel-offset"].c_str());
@@ -99,27 +105,6 @@ bool parseOptions(int argc, char **argv, Options & opt) {
   return true;
 }
 
-// Read a file's content in a single string
-bool readFileInString(std::string const& filename, std::string & str) {
-
-  str.clear(); // clear the output
-
-  std::ifstream ifs(filename.c_str());
-  if (!ifs.is_open()) {
-    std::cout << "Cannot open file: " << filename << std::endl;
-    return false;
-  }
-  
-  ifs.seekg(0, std::ios::end);   
-  str.reserve(ifs.tellg());
-  ifs.seekg(0, std::ios::beg);
-  str.assign((std::istreambuf_iterator<char>(ifs)),
-             std::istreambuf_iterator<char>());
-  ifs.close();
-
-  return true;
-}
-
 // Sort the errors and print some stats
 void printErrors(std::vector<double> & errors, double desired_precision,
                  double max_achieved_precision) {
@@ -162,6 +147,7 @@ bool loadCsmCameraModel(std::string const& model_file,
   if (!readFileInString(model_file, model_state))
     return false;
 
+
   // Check if loading the model worked
   bool success = false;
 
@@ -219,6 +205,31 @@ bool loadCsmCameraModel(std::string const& model_file,
   return true;
 }
 
+bool updateSupModel(std::string& sup_string, std::string model) {
+  // grab the state JSON out of the original sup file to determine length of string to replace
+  std::string sup_state = stateAsJson(sup_string).dump().c_str();
+
+  // add back in the BEL characters in json state for GXP to read .sup file
+  std::replace(model.begin(), model.end(), '\n', '\a');
+
+  // update the .sup file SENSOR_STATE_LENGTH with new state length
+  std::string sensor_length = "SENSOR_STATE_LENGTH ";
+  size_t start_len_pos = sup_string.find(sensor_length) + sensor_length.length();
+  size_t end_len_pos = sup_string.find("SENSOR_STATE ") - 1;
+  sup_string.replace(start_len_pos, end_len_pos - start_len_pos, std::to_string(model.length()));
+
+  //replace the camera state in .sup file at the state model name which start with "USGS_ASTRO"
+  size_t start_pos = sup_string.find("USGS_ASTRO");
+  std::size_t end_pos = sup_string.find_last_of("}");
+
+  if(start_pos == std::string::npos)
+    return false;
+
+  sup_string.replace(start_pos, (end_pos - start_pos) + 1, model);
+  return true;
+}
+
+
 int main(int argc, char **argv) {
 
   Options opt;
@@ -239,6 +250,19 @@ int main(int argc, char **argv) {
     ofs.close();
   }
 
+  if (opt.modify_sup_file != "") {
+    std::string sup_string;
+    readFileInString(opt.modify_sup_file, sup_string);
+
+    if (!updateSupModel(sup_string, model->getModelState()))
+      return 1;
+
+    std::cout << "Updating model state for sup file: " << opt.modify_sup_file << "\n";
+    std::ofstream ofs_sup(opt.modify_sup_file.c_str());
+    ofs_sup << sup_string << "\n";
+    ofs_sup.close();
+  }
+
   if (opt.sample_rate > 0) {
     csm::ImageVector image_size = model->getImageSize();
     std::cout << "\n";

docs/source/tools/usgscsm_cam_test.rst

@@ -3,8 +3,8 @@ usgscsm_cam_test
 
 This program is shipped with the USGSCSM library in the ``bin`` directory.
 It can be used for performing several operations involving CSM camera
-models, such as loading a camera model, whether in the original ISD format
-or its model state representation, exporting the model state, computing
+models, such as loading a camera model, whether in the original ISD format,
+model state representation, or GXP .sup file exporting the model state, computing
 projections from pixels in the camera to the ground and back, and
 then verifying that the original pixels are obtained.
 
@@ -17,11 +17,15 @@ Example (perform per-pixel operations)::
     usgscsm_cam_test --model camera.json --sample-rate 100 \
        --height-above-datum 320.3 --subpixel-offset 0.57
 
+Example (modify a GXP .sup with new model state)::
+
+   usgscsm_cam_test --model camera.json --modify-sup-file gxp_file.sup
+
 Command line options
 ~~~~~~~~~~~~~~~~~~~~
 
 --model <string (default: "")>
-    Input CSM model (in ISD or model state format).
+    Input CSM model (in ISD, model state, or GXP .sup file format).
 
 --output-model-state <string (default: "")>
     If specified, save the model state to this file.
@@ -43,5 +47,9 @@ Command line options
     Use this value for operations (ground-to-image and image-to-ground)
     which need a precision value. Measured in pixels.
 
+--modify-sup-file (default: "")>
+    Input GXP .sup file to be modified by inputted CSM model. This will override
+    the existing .sup file's SENSOR_STATE.
+
 --help <no value>
     Print the usage message.

include/usgscsm/Utilities.h

@@ -184,6 +184,12 @@ std::vector<double> getSensorOrientations(nlohmann::json isd,
 double getWavelength(nlohmann::json isd, csm::WarningList *list = nullptr);
 nlohmann::json stateAsJson(std::string modelState);
 
+// Read the contents of the file out as a string
+bool readFileInString(std::string const& filename, std::string & str);
+
+// Removes special characters from a string
+void sanitize(std::string &input);
+
 // Apply transforms to orientations and vectors
 void applyRotationToQuatVec(ale::Rotation const& r, std::vector<double> & quaternions);
 void applyRotationTranslationToXyzVec(ale::Rotation const& r, ale::Vec3d const& t,

src/Utilities.cpp

@@ -7,6 +7,8 @@
 #include <stdexcept>
 #include <utility>
 #include <ctime>
+#include <iostream>
+#include <fstream>
 
 #include "ale/Distortion.h"
 
@@ -1408,12 +1410,42 @@ double getWavelength(json isd, csm::WarningList *list) {
 }
 
 json stateAsJson(std::string modelState) {
-  std::size_t found = modelState.find_first_of("\n");
+  // Remove special characters from string
+  sanitize(modelState);
 
-  if (found == std::string::npos) {
-    found = 0;
+  std::size_t foundFirst = modelState.find_first_of("{");
+  std::size_t foundLast = modelState.find_last_of("}");
+
+  if (foundFirst == std::string::npos) {
+    foundFirst = 0;
+  }
+  return json::parse(modelState.begin() + foundFirst, modelState.begin() + foundLast + 1);
+}
+
+void sanitize(std::string &input){
+  // Replaces characters from the string that are not printable with newlines
+  std::replace_if(input.begin(), input.end(), [](int c){return !::isprint(c);}, '\n');
+}
+
+// Read a file's content in a single string
+bool readFileInString(std::string const& filename, std::string & str) {
+
+  str.clear(); // clear the output
+
+  std::ifstream ifs(filename.c_str());
+  if (!ifs.is_open()) {
+    std::cout << "Cannot open file: " << filename << std::endl;
+    return false;
   }
-  return json::parse(modelState.begin() + found, modelState.end());
+
+  ifs.seekg(0, std::ios::end);
+  str.reserve(ifs.tellg());
+  ifs.seekg(0, std::ios::beg);
+  str.assign((std::istreambuf_iterator<char>(ifs)),
+             std::istreambuf_iterator<char>());
+  ifs.close();
+
+  return true;
 }
 
 // Apply a rotation to a vector of quaternions.

tests/CMakeLists.txt

@@ -33,4 +33,14 @@ add_test(NAME test_usgscsm_cam_test_load_state
     COMMAND usgscsm_cam_test --model model_state.json --output-model-state model_state2.json
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests)
 
+
+# 3. Save model state from .sup file
+add_test(NAME test_usgscsm_cam_test_save_sup_state
+    COMMAND usgscsm_cam_test --model data/gxp_model_file.sup --output-model-state gxp_model_state.json
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests)
+# 4. Load model state
+add_test(NAME test_usgscsm_cam_test_load_sup_state
+    COMMAND usgscsm_cam_test --model gxp_model_state.json --output-model-state gxp_model_state2.json
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests)
+
 gtest_discover_tests(runCSMCameraModelTests WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests)

tests/Fixtures.h

@@ -248,6 +248,7 @@ class ConstVelocityLineScanSensorModel : public ::testing::Test {
   csm::Isd isd;
   std::shared_ptr<csm::Model> model;
   UsgsAstroLsSensorModel *sensorModel;
+  std::string supFile = "data/gxp_model_file.sup";
 
   void SetUp() override {
     sensorModel = NULL;

tests/PluginTests.cpp

@@ -161,6 +161,15 @@ TEST_F(ConstVelLineScanIsdTest, ConstructInvalidCamera) {
 
 }
 
+TEST_F(ConstVelocityLineScanSensorModel, ConstructibleFromSupState) {
+  UsgsAstroPlugin testPlugin;
+  std::string modelState;
+  EXPECT_TRUE(readFileInString(supFile, modelState));
+  sanitize(modelState);
+  EXPECT_TRUE(testPlugin.canModelBeConstructedFromState(
+      "USGS_ASTRO_LINE_SCANNER_SENSOR_MODEL", modelState));
+}
+
 TEST_F(SarIsdTest, Constructible) {
   UsgsAstroPlugin testPlugin;
   csm::WarningList warnings;

tests/UtilitiesTests.cpp

@@ -624,3 +624,15 @@ TEST(UtilitiesTests, ephemTimeToCalendarTime) {
 
   EXPECT_STREQ(timeStr.c_str(), "2000-01-01T00:00:00Z");
 }
+
+TEST(UtilitiesTests, fileReaderTest) {
+  std::string fromFile;
+  readFileInString("data/hello.json", fromFile);
+  EXPECT_STREQ(fromFile.c_str(), "\"Hello\"\n");
+}
+
+TEST(UtilitiesTests, sanitizeTest) {
+  std::string input = "\nHello World\007";
+  sanitize(input);
+  EXPECT_STREQ(input.c_str(), "\nHello World\n");
+}

tests/data/hello.json

+"Hello"