VlkbCli.java

import java.util.logging.Logger;
import java.util.logging.Level;
import java.util.List;
import java.util.ArrayList;
import java.util.Arrays;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileNotFoundException;
import java.nio.file.StandardOpenOption;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.commons.io.FilenameUtils;

import java.time.*;// Timestamp in cut-filename
import java.io.ByteArrayOutputStream; // for SODA direct streaming doSubimgStream

// tar.gz compress
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import java.io.BufferedOutputStream;
import java.util.Date;
import java.text.SimpleDateFormat;
import org.json.simple.JSONArray;

import vo.parameter.*;
import vo.error.*;

class VlkbCli implements Vlkb
{
   static final Logger LOGGER = Logger.getLogger(VlkbCli.class.getName());

   private Settings    settings   = null;
   private Subsurvey[] subsurveys = null;
   private Resolver    resolver   = null;
   private Soda        soda       = null;


   public VlkbCli()
   {
      LOGGER.fine("trace VlkbCli()");
      this.settings = Settings.getInstance();
      this.soda = new SodaImpl(settings.fitsPaths);
      this.resolver = (settings.dbConn.isDbUriEmpty() ? new ResolverFromId(subsurveys)
            : new ResolverByObsCore(settings.dbConn, subsurveys));
   }


   public VlkbCli(Settings settings)
   {
      LOGGER.fine("trace VlkbCli(settings)");
      this.settings = settings;
      this.soda = new SodaImpl(settings.fitsPaths);
      this.resolver = (settings.dbConn.isDbUriEmpty() ? new ResolverFromId(subsurveys)
            : new ResolverByObsCore(settings.dbConn, subsurveys));
   }


   public VlkbCli(Settings settings, Subsurvey[] subsurveys)
   {
      LOGGER.fine("trace VlkbCli(settings, subsurveys)");
      this.settings = settings;
      this.subsurveys = subsurveys;
      this.soda = new SodaImpl(settings.fitsPaths);
      this.resolver = new ResolverFromId(subsurveys);
      //this.resolver = (settings.dbConn.isDbUriEmpty() ? new ResolverFromId(subsurveys)
      //      : new ResolverByObsCore(settings.dbConn, subsurveys));
   }


   public CutResult doMerge(String[] idArr, Coord coord, boolean countNullValues)
      throws FileNotFoundException, IOException
   {
      LOGGER.fine("trace doMerge by CLI is NOT IMPLEMENTED (only by AMQP)");

      return new CutResult();
   }

   public CutResult doFileAmqp(String relPathname, int hdunum,
         Pos pos, Band band, Time time, Pol pol,
         boolean countNullValues, FitsCard[] extraCards,
         String cutAbsPathname)
         throws IOException, InterruptedException
   {
      LOGGER.fine("trace doFileAmqp by CLI is NOT IMPLEMENTED (only by AMQP)");

      return new CutResult();
   }



   public void doFile(String relPathname, int hdunum,
         Pos pos, Band band, Time time, Pol pol,
			String cutAbsPathname)
         throws IOException, InterruptedException
			{
				LOGGER.fine("trace: " + cutAbsPathname );

				Pixeli[] pixeli = null;

				try(OutputStream fileOutputStream = new FileOutputStream(new File(cutAbsPathname)))
				{
					soda.doStream(relPathname, hdunum, pos, band, time, pol, pixeli, fileOutputStream);
				}
			}

   public void doFile(String relPathname, int hdunum,
         String pixels,	String cutAbsPathname)
         throws IOException, InterruptedException
			{
				LOGGER.fine("trace: " + cutAbsPathname );

				try(OutputStream fileOutputStream = new FileOutputStream(new File(cutAbsPathname)))
				{
					soda.doStream(relPathname, hdunum, pixels, fileOutputStream);
				}
			}



	public MCutResult doMCutout(String jdlJson, String workDir)
		throws IOException, InterruptedException
		{
			LOGGER.fine("trace");

			MCutResult mCutResult;

			CutArgs[] cutArgsArr = CutArgs.parseCutArgsArr(jdlJson);
			MCutResult.Cut[] cutResultArr = doCutouts( cutArgsArr, workDir );
			String respJsonString = genResponseJson( cutResultArr );
			mCutResult = doCompressCutFiles( cutResultArr, respJsonString );

			return mCutResult;
		}



	private MCutResult.Cut[] doCutouts(CutArgs[] cutArgsArr, String workDir)
	{
		LOGGER.fine("trace, count of cuts : " + String.valueOf( cutArgsArr.length ) );

		List<MCutResult.Cut> cutResList = new ArrayList<MCutResult.Cut>();

		int ix = 0;
		for(CutArgs cutArgs: cutArgsArr)
		{
			MCutResult.Cut cut = doFileByIdWithErr(cutArgs.id,
					cutArgs.pos, cutArgs.band, cutArgs.time,  cutArgs.pol,  cutArgs.pixels,
					cutArgs.countNullValues, workDir, ix);//, null);//cutArgs.extraCards);

			cut.index = ix++;

			LOGGER.finest("cut" + String.valueOf(cut.index) + " : " + cut.content);

			cutResList.add(cut);
		}

		return cutResList.toArray(new MCutResult.Cut[0]);
	}


	private MCutResult.Cut doFileByIdWithErr(String id, Pos pos, Band band, Time time, Pol pol,
			String pixels, boolean countNullValues/*, Subsurvey[] subsurveys*/,
			String workDir, int ix)
	{
		LOGGER.fine("trace");

		MCutResult mCutResult = new MCutResult();
		MCutResult.Cut cut = mCutResult.new Cut(/* FIXME eventually add here new Inputs(id, pos,..) */);

		try
		{
			this.resolver.resolve(id);
			String relPathname = this.resolver.relPathname();
			int hdunum         = this.resolver.hdunum();
			String subsurveyId = this.resolver.obsCollection();

			FitsCard[] extraCards = null;

			if(subsurveyId != null)
			{
				extraCards = Subsurvey.subsurveysFindCards(this.subsurveys, subsurveyId);
			}
			else
			{
				LOGGER.finer("Resolver returns subsurveyId null: no extraCards loaded.");
			}

			final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from config file

			final int MAX_FILENAME_LEN = 100; // bytes, tar-entry max length is 100bytes
			String cutAbsPathname = workDir + "/"
				+ generateSubimgPathname(ix, relPathname, hdunum, MAX_FILENAME_LEN);

         if(pixels != null)
            doFile(relPathname, hdunum, pixels, cutAbsPathname);
         else
            doFile(relPathname, hdunum, pos, band, time, pol, cutAbsPathname);

			cut.content     = cutAbsPathname;
			cut.contentType = MCutResult.Cut.ContentType.FILENAME;
		}
		catch(MultiValuedParamNotSupported ex)
		{
			cut.content = "MultiValuedParamNotSupported: " + ex.getMessage();
			cut.contentType = MCutResult.Cut.ContentType.BAD_REQUEST;
			LOGGER.warning(cut.content);
		}
		catch(IllegalArgumentException ex)
		{
			cut.content = "IllegalArgumentException: " + ex.getMessage();
			cut.contentType = MCutResult.Cut.ContentType.BAD_REQUEST;
			LOGGER.warning(cut.content);
		}
		catch(Exception ex)
		{
			cut.content     = "Exception: " + ex.getMessage();
			cut.contentType = MCutResult.Cut.ContentType.SERVICE_ERROR;
			LOGGER.severe(cut.content);
			ex.printStackTrace();
		}

		return cut;
	}



	// FIXME implement similar for Merge: MCutResult = call-Montage-demosaic-sequence(cutResultArr)
	private MCutResult doCompressCutFiles(MCutResult.Cut[] cutArr, String respJsonString)
		throws IOException, InterruptedException
		{
			Instant instant = Instant.now();
			String timestamp = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + "_" + instant.getNano();

			final String tgzFileName = settings.fitsPaths.cutouts()
				+ "/mcutout_" + timestamp + ".tar.gz";
			// generate response-*.json with timestamp
			String respJsonPathname = settings.fitsPaths.cutouts()
				+ "/response_" + timestamp + ".json";

			try (PrintWriter out = new PrintWriter(respJsonPathname))
			{
				out.print(respJsonString);
			}

			List<Path> paths = new ArrayList<Path>();

			paths.add(Paths.get(respJsonPathname));

			for(MCutResult.Cut cut : cutArr)
			{
				LOGGER.finest("cut-index"+ String.valueOf(cut.index) + " -> " + cut.content);
				if(cut.contentType == MCutResult.Cut.ContentType.FILENAME)
				{
					Path p = Paths.get(cut.content);
					paths.add(p);
				}
			}

			Path output = Paths.get(tgzFileName);
			createTarGzipFiles(paths, output);

			MCutResult mCutResult = new MCutResult();
			mCutResult.cutResArr = cutArr;
			mCutResult.fileName = tgzFileName;
			mCutResult.resJsonPathname = respJsonPathname;
			mCutResult.fileSize = Files.size(output);

			return mCutResult;
		}


	private String genResponseJson(MCutResult.Cut[] cutArr)
	{
		List<Path> paths = new ArrayList<Path>();

		JSONArray jArr = new JSONArray();

		for(MCutResult.Cut cut : cutArr)
		{
			jArr.add(cut.toJsonObject());
		}

		return jArr.toString();
	}


	/*
		<dependency>
		<groupId>org.apache.commons</groupId>
		<artifactId>commons-compress</artifactId>
		<version>1.20</version>
		</dependency>

		import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
		import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
		import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
	 */
	private static void createTarGzipFiles(List<Path> paths, Path output)
		throws IOException
		{
			try (OutputStream fOut = Files.newOutputStream(output);
					BufferedOutputStream buffOut = new BufferedOutputStream(fOut);
					GzipCompressorOutputStream gzOut = new GzipCompressorOutputStream(buffOut);
					TarArchiveOutputStream tOut = new TarArchiveOutputStream(gzOut))
			{
				for (Path path : paths)
				{
					if (!Files.isRegularFile(path))
					{
						throw new IOException("Must be regular file, but was : " + path.toString());
					}

					TarArchiveEntry tarEntry = new TarArchiveEntry(
							path.toFile(),
							path.getFileName().toString());

					tOut.putArchiveEntry(tarEntry);
					Files.copy(path, tOut); // NOTE tar limits max filename length <100bytes
					tOut.closeArchiveEntry();
				}

				tOut.finish();
			}
		}


	public NullValueCount doCountNullValues(String absPathname, int hdunum)
		throws IOException, InterruptedException
		{
			ByteArrayOutputStream bos = new ByteArrayOutputStream();
			if(bos == null)
				throw new AssertionError("byte output stream for bounds was not created, is null");

			String[] cmdBounds = new String[3];
			cmdBounds[0] = "/usr/local/bin/vlkb";
			cmdBounds[1] = "nullvals";
			cmdBounds[2] = absPathname;

			StringBuilder errorStrBuilder = new StringBuilder();
			ExecCmd exec = new ExecCmd();
			exec.doRun(bos, cmdBounds, errorStrBuilder);
			LOGGER.finest("exec NullVals exitValue: " + exec.exitValue);

			bos.close();

			boolean hasResult = (exec.exitValue == 0);
			if(hasResult)
			{
				String nullValsString = new String(bos.toByteArray());
				LOGGER.finest("vlkb nullvals: " + nullValsString);

				if((nullValsString != null) && nullValsString.trim().isEmpty())
				{
					throw new AssertionError("'vlkb nullvals' returned empty string");
				}

				// parse result: '<fill-ratio> <nullvals-count> <tot-count>'

				String[] splitStr = nullValsString.trim().split("\\s+");
				if(splitStr.length != 3)
					throw new AssertionError("'vlkb nullvals' did not return 3 numbers but: " + nullValsString);

				NullValueCount nvc = new NullValueCount();
				nvc.percent = Double.parseDouble(splitStr[0]);
				nvc.nullCount = Long.parseLong(splitStr[1]);
				nvc.totalCount = Long.parseLong(splitStr[2]);
				return nvc;
			}
			else
			{
				throw new AssertionError("'vlkb nullvals' exited with error: "
						+ errorStrBuilder.toString() + " for: " + absPathname);
			}
		}



	private  String generateSubimgPathname(int ix, String relPathname, int hdunum, int maxLen)
	{
		String cutfitsname = String.valueOf(ix);

		String tempPathname1 = relPathname.replaceAll("/","-");
		String tempPathname2 = tempPathname1.replaceAll(" ","_");
		String filename;

		if(hdunum == 1)
		{
			filename = cutfitsname + "_" + tempPathname2;
		}
		else
		{
			String extnum = "E" + String.valueOf(hdunum-1);
			filename = cutfitsname + "_" + extnum + "_" + tempPathname2 ;
		}

		// shorten filename to maxLen

		if(filename.length() > maxLen) // FIXME consider UTF-8 ? string-len != string-buff-len
		{
			String ext  = FilenameUtils.getExtension(filename);// without dot
			String name = FilenameUtils.removeExtension(filename);

			filename = name.substring(0, maxLen - 1 - ext.length() ) + "." + ext;
		}

		return filename;
	}


}