Skip to content
Snippets Groups Projects
Commit 25f373f6 authored by Grégory Mantelet's avatar Grégory Mantelet
Browse files

[TAP] Fix disk space consumption with UPLOAD for synchronous jobs.

Files uploaded by the user when creating/executing a synchronous job were never
deleted after the job execution.

The same problem applied for the tables already uploaded in the database (in
`TAP_UPLOAD`) when an error occurred before the end of the UPLOAD process.

Now, in case of error when uploading one or more files, or in case of success of
the job, all uploaded files and their corresponding database tables are deleted
after the end of the job.
parent 77257d61
No related branches found
No related tags found
No related merge requests found
...@@ -22,6 +22,7 @@ package tap; ...@@ -22,6 +22,7 @@ package tap;
import java.io.IOException; import java.io.IOException;
import java.util.Date; import java.util.Date;
import java.util.Iterator;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
...@@ -29,6 +30,7 @@ import tap.parameters.TAPParameters; ...@@ -29,6 +30,7 @@ import tap.parameters.TAPParameters;
import uws.UWSException; import uws.UWSException;
import uws.job.JobThread; import uws.job.JobThread;
import uws.service.log.UWSLog.LogLevel; import uws.service.log.UWSLog.LogLevel;
import uws.service.request.UploadFile;
/** /**
* <p>This class represent a TAP synchronous job. * <p>This class represent a TAP synchronous job.
...@@ -240,6 +242,9 @@ public class TAPSyncJob { ...@@ -240,6 +242,9 @@ public class TAPSyncJob {
}finally{ }finally{
// Whatever the way the execution stops (normal, cancel or error), an execution report must be fulfilled: // Whatever the way the execution stops (normal, cancel or error), an execution report must be fulfilled:
execReport = thread.getExecutionReport(); execReport = thread.getExecutionReport();
// Delete uploaded files:
deleteUploads(tapParams);
} }
// Report any error that may have occurred while the thread execution: // Report any error that may have occurred while the thread execution:
...@@ -296,6 +301,25 @@ public class TAPSyncJob { ...@@ -296,6 +301,25 @@ public class TAPSyncJob {
return thread.isSuccess(); return thread.isSuccess();
} }
/**
* Delete all uploaded files.
*
* @param tapParams Input parameters (listing all uploaded files, if any).
*
* @since 2.3
*/
protected void deleteUploads(final TAPParameters tapParams){
Iterator<UploadFile> itFiles = tapParams.getFiles();
while(itFiles.hasNext()){
UploadFile uf = itFiles.next();
try{
uf.deleteFile();
}catch(IOException ioe){
service.getLogger().logTAP(LogLevel.WARNING, this, "END", "Unable to delete the uploaded file \"" + uf.getLocation() + "\"!", ioe);
}
}
}
/** /**
* <p>Thread which will process the job execution.</p> * <p>Thread which will process the job execution.</p>
* *
......
...@@ -16,13 +16,15 @@ package tap.upload; ...@@ -16,13 +16,15 @@ package tap.upload;
* You should have received a copy of the GNU Lesser General Public License * You should have received a copy of the GNU Lesser General Public License
* along with TAPLibrary. If not, see <http://www.gnu.org/licenses/>. * along with TAPLibrary. If not, see <http://www.gnu.org/licenses/>.
* *
* Copyright 2012-2015 - UDS/Centre de Données astronomiques de Strasbourg (CDS), * Copyright 2012-2018 - UDS/Centre de Données astronomiques de Strasbourg (CDS),
* Astronomisches Rechen Institut (ARI) * Astronomisches Rechen Institut (ARI)
*/ */
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import com.oreilly.servlet.multipart.ExceededSizeException;
import tap.ServiceConnection; import tap.ServiceConnection;
import tap.ServiceConnection.LimitUnit; import tap.ServiceConnection.LimitUnit;
import tap.TAPException; import tap.TAPException;
...@@ -31,6 +33,7 @@ import tap.data.LimitedTableIterator; ...@@ -31,6 +33,7 @@ import tap.data.LimitedTableIterator;
import tap.data.TableIterator; import tap.data.TableIterator;
import tap.data.VOTableIterator; import tap.data.VOTableIterator;
import tap.db.DBConnection; import tap.db.DBConnection;
import tap.db.DBException;
import tap.metadata.TAPColumn; import tap.metadata.TAPColumn;
import tap.metadata.TAPMetadata; import tap.metadata.TAPMetadata;
import tap.metadata.TAPMetadata.STDSchema; import tap.metadata.TAPMetadata.STDSchema;
...@@ -40,18 +43,16 @@ import tap.parameters.DALIUpload; ...@@ -40,18 +43,16 @@ import tap.parameters.DALIUpload;
import uws.UWSException; import uws.UWSException;
import uws.service.file.UnsupportedURIProtocolException; import uws.service.file.UnsupportedURIProtocolException;
import com.oreilly.servlet.multipart.ExceededSizeException;
/** /**
* <p>Let create properly given VOTable inputs in the "database".</p> * Let create properly given VOTable inputs in the "database".
* *
* <p> * <p>
* This class manages particularly the upload limit in rows and in bytes by creating a {@link LimitedTableIterator} * This class manages particularly the upload limit in rows and in bytes by
* with a {@link VOTableIterator}. * creating a {@link LimitedTableIterator} with a {@link VOTableIterator}.
* </p> * </p>
* *
* @author Gr&eacute;gory Mantelet (CDS;ARI) * @author Gr&eacute;gory Mantelet (CDS;ARI)
* @version 2.0 (04/2015) * @version 2.3 (08/2018)
* *
* @see LimitedTableIterator * @see LimitedTableIterator
* @see VOTableIterator * @see VOTableIterator
...@@ -59,14 +60,17 @@ import com.oreilly.servlet.multipart.ExceededSizeException; ...@@ -59,14 +60,17 @@ import com.oreilly.servlet.multipart.ExceededSizeException;
public class Uploader { public class Uploader {
/** Specification of the TAP service. */ /** Specification of the TAP service. */
protected final ServiceConnection service; protected final ServiceConnection service;
/** Connection to the "database" (which lets upload the content of any given VOTable). */ /** Connection to the "database" (which lets upload the content of any given
* VOTable). */
protected final DBConnection dbConn; protected final DBConnection dbConn;
/** Description of the TAP_UPLOAD schema to use. /** Description of the TAP_UPLOAD schema to use.
* @since 2.0 */ * @since 2.0 */
protected final TAPSchema uploadSchema; protected final TAPSchema uploadSchema;
/** Type of limit to set: ROWS or BYTES. <i>MAY be NULL ; if NULL, no limit will be set.</i> */ /** Type of limit to set: ROWS or BYTES. <i>MAY be NULL ; if NULL, no limit
* will be set.</i> */
protected final LimitUnit limitUnit; protected final LimitUnit limitUnit;
/** Limit on the number of rows or bytes (depending of {@link #limitUnit}) allowed to be uploaded in once (whatever is the number of tables). */ /** Limit on the number of rows or bytes (depending of {@link #limitUnit})
* allowed to be uploaded in once (whatever is the number of tables). */
protected final int limit; protected final int limit;
/** Number of rows already loaded. */ /** Number of rows already loaded. */
...@@ -78,7 +82,8 @@ public class Uploader { ...@@ -78,7 +82,8 @@ public class Uploader {
* @param service Specification of the TAP service using this uploader. * @param service Specification of the TAP service using this uploader.
* @param dbConn A valid (open) connection to the "database". * @param dbConn A valid (open) connection to the "database".
* *
* @throws TAPException If any error occurs while building this {@link Uploader}. * @throws TAPException If any error occurs while building this
* {@link Uploader}.
*/ */
public Uploader(final ServiceConnection service, final DBConnection dbConn) throws TAPException{ public Uploader(final ServiceConnection service, final DBConnection dbConn) throws TAPException{
this(service, dbConn, null); this(service, dbConn, null);
...@@ -90,7 +95,8 @@ public class Uploader { ...@@ -90,7 +95,8 @@ public class Uploader {
* @param service Specification of the TAP service using this uploader. * @param service Specification of the TAP service using this uploader.
* @param dbConn A valid (open) connection to the "database". * @param dbConn A valid (open) connection to the "database".
* *
* @throws TAPException If any error occurs while building this {@link Uploader}. * @throws TAPException If any error occurs while building this
* {@link Uploader}.
* *
* @since 2.0 * @since 2.0
*/ */
...@@ -131,19 +137,30 @@ public class Uploader { ...@@ -131,19 +137,30 @@ public class Uploader {
} }
/** /**
* <p>Upload all the given VOTable inputs.</p> * Upload all the given VOTable inputs.
* *
* <p><i>Note: * <p><b>Note 1:</b>
* The {@link TAPTable} objects representing the uploaded tables will be associated with the TAP_UPLOAD schema specified at the creation of this {@link Uploader}. * The {@link TAPTable} objects representing the uploaded tables will be
* If no such schema was specified, a default one (whose DB name will be equals to the ADQL name, that's to say {@link STDSchema#UPLOADSCHEMA}) * associated with the TAP_UPLOAD schema specified at the creation of this
* is created, will be associated with the uploaded tables and will be returned by this function. * {@link Uploader}. If no such schema was specified, a default one (whose
* </i></p> * DB name will be equals to the ADQL name, that's to say
* {@link STDSchema#UPLOADSCHEMA}) is created, will be associated with the
* uploaded tables and will be returned by this function.
* </p>
*
* <p><b>Note 2:</b>
* In case of error while ingesting one or all of the uploaded tables,
* all tables created in the database before the error occurs are dropped
* <i>(see {@link #dropUploadedTables()})</i>.
* </p>
* *
* @param uploads Array of tables to upload. * @param uploads Array of tables to upload.
* *
* @return A {@link TAPSchema} containing the list and the description of all uploaded tables. * @return A {@link TAPSchema} containing the list and the description of
* all uploaded tables.
* *
* @throws TAPException If any error occurs while reading the VOTable inputs or while uploading the table into the "database". * @throws TAPException If any error occurs while reading the VOTable inputs
* or while uploading the table into the "database".
* *
* @see DBConnection#addUploadedTable(TAPTable, tap.data.TableIterator) * @see DBConnection#addUploadedTable(TAPTable, tap.data.TableIterator)
*/ */
...@@ -181,14 +198,28 @@ public class Uploader { ...@@ -181,14 +198,28 @@ public class Uploader {
votable = null; votable = null;
} }
}catch(DataReadException dre){ }catch(DataReadException dre){
// Drop uploaded tables:
dropUploadedTables();
// Report the error:
if (dre.getCause() instanceof ExceededSizeException) if (dre.getCause() instanceof ExceededSizeException)
throw dre; throw dre;
else else
throw new TAPException("Error while reading the VOTable \"" + tableName + "\": " + dre.getMessage(), dre, UWSException.BAD_REQUEST); throw new TAPException("Error while reading the VOTable \"" + tableName + "\": " + dre.getMessage(), dre, UWSException.BAD_REQUEST);
}catch(IOException ioe){ }catch(IOException ioe){
// Drop uploaded tables:
dropUploadedTables();
// Report the error:
throw new TAPException("IO error while reading the VOTable of \"" + tableName + "\"!", ioe); throw new TAPException("IO error while reading the VOTable of \"" + tableName + "\"!", ioe);
}catch(UnsupportedURIProtocolException e){ }catch(UnsupportedURIProtocolException e){
// Drop uploaded tables:
dropUploadedTables();
// Report the error:
throw new TAPException("URI error while trying to open the VOTable of \"" + tableName + "\"!", e); throw new TAPException("URI error while trying to open the VOTable of \"" + tableName + "\"!", e);
}catch(TAPException te){
// Drop uploaded tables:
dropUploadedTables();
// Report the error:
throw te;
}finally{ }finally{
try{ try{
if (dataIt != null) if (dataIt != null)
...@@ -200,8 +231,27 @@ public class Uploader { ...@@ -200,8 +231,27 @@ public class Uploader {
} }
} }
// Return the TAP_UPLOAD schema (containing just the description of the uploaded tables): /* Return the TAP_UPLOAD schema (containing just the description of the
* uploaded tables): */
return uploadSchema; return uploadSchema;
} }
/**
* Drop all tables already uploaded in the database.
*
* @since 2.3
*/
protected void dropUploadedTables(){
if (uploadSchema == null || uploadSchema.getNbTables() == 0)
return;
for(TAPTable table : uploadSchema){
try{
dbConn.dropUploadedTable(table);
}catch(DBException e){
service.getLogger().error("Unable to drop the uploaded table " + table.getFullName() + "!", e);
}
}
}
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment