Skip to content
Snippets Groups Projects
Commit 0db34a50 authored by gmantele's avatar gmantele
Browse files

TAP: Add the upload limits in TAPConfiguration and its default implementation

parent 813ded32
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
src/tap/ServiceConnection.java
+ 77
− 2
View file @ 0db34a50
......@@ -16,7 +16,7 @@ package tap;
* You should have received a copy of the GNU Lesser General Public License
* along with TAPLibrary. If not, see <http://www.gnu.org/licenses/>.
*
* Copyright 2012-2013 - UDS/Centre de Données astronomiques de Strasbourg (CDS),
* Copyright 2012-2014 - UDS/Centre de Données astronomiques de Strasbourg (CDS),
* Astronomisches Rechen Institute (ARI)
*/
......@@ -33,14 +33,89 @@ import uws.service.UserIdentifier;
*
*
* @author Gr&eacute;gory Mantelet (CDS;ARI) - gmantele@ari.uni-heidelberg.de
* @version 1.1 (12/2013)
* @version 1.1 (01/2014)
*
* @param <R>
*/
public interface ServiceConnection< R > {
/**
* Units used to express any limit of the TAP service.
*
* @author Gr&eacute;gory Mantelet (ARI) - gmantele@ari.uni-heidelberg.de
* @version 1.1 (01/2014)
*/
public static enum LimitUnit{
rows, bytes, kilobytes, megabytes, gigabytes;
/**
* Tells whether the given unit has the same type (bytes or rows).
*
* @param anotherUnit A unit.
*
* @return true if the given unit has the same type, false otherwise.
*
* @since 1.1
*/
public boolean isCompatibleWith(final LimitUnit anotherUnit){
if (this == rows)
return anotherUnit == rows;
else
return anotherUnit != rows;
}
/**
* Gets the factor to convert into bytes the value expressed in this unit.
* <i>Note: if this unit is not a factor of bytes, 1 is returned (so that the factor does not affect the value).</i>
*
* @return The factor need to convert a value expressed in this unit into bytes, or 1 if not a bytes derived unit.
*
* @since 1.1
*/
public long bytesFactor(){
switch(this){
case bytes:
return 1;
case kilobytes:
return 1000;
case megabytes:
return 1000000;
case gigabytes:
return 1000000000l;
default:
return 1;
}
}
/**
* Compares the 2 given values (each one expressed in the given unit).
* Conversions are done internally in order to make a correct comparison between the 2 limits.
*
* @param leftLimit Value/Limit of the comparison left part.
* @param leftUnit Unit of the comparison left part value.
* @param rightLimit Value/Limit of the comparison right part.
* @param rightUnit Unit of the comparison right part value.
*
* @return the value 0 if x == y; a value less than 0 if x < y; and a value greater than 0 if x > y
*
* @throws TAPException If the two given units are not compatible.
*
* @see #isCompatibleWith(LimitUnit)
* @see #bytesFactor()
* @see Integer#compare(int, int)
* @see Long#compare(long, long)
*
* @since 1.1
*/
public static int compare(final int leftLimit, final LimitUnit leftUnit, final int rightLimit, final LimitUnit rightUnit) throws TAPException{
if (!leftUnit.isCompatibleWith(rightUnit))
throw new TAPException("Limit units (" + leftUnit + " and " + rightUnit + ") are not compatible!");
if (leftUnit == rows || leftUnit == rightUnit)
return Integer.compare(leftLimit, rightLimit);
else
return Long.compare(leftLimit * leftUnit.bytesFactor(), rightLimit * rightUnit.bytesFactor());
}
}
public String getProviderName();
......
src/tap/config/DefaultServiceConnection.java
+ 105
− 43
View file @ 0db34a50
package tap.config;
/*
* This file is part of TAPLibrary.
*
* TAPLibrary is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* TAPLibrary is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with TAPLibrary. If not, see <http://www.gnu.org/licenses/>.
*
* Copyright 2013 - Astronomisches Rechen Institute (ARI)
*/
import static tap.config.TAPConfiguration.DEFAULT_DIRECTORY_PER_USER;
import static tap.config.TAPConfiguration.DEFAULT_EXECUTION_DURATION;
import static tap.config.TAPConfiguration.DEFAULT_GROUP_USER_DIRECTORIES;
import static tap.config.TAPConfiguration.DEFAULT_IS_AVAILABLE;
import static tap.config.TAPConfiguration.DEFAULT_RETENTION_PERIOD;
import static tap.config.TAPConfiguration.DEFAULT_UPLOAD_MAX_FILE_SIZE;
import static tap.config.TAPConfiguration.KEY_DEFAULT_EXECUTION_DURATION;
import static tap.config.TAPConfiguration.KEY_DEFAULT_OUTPUT_LIMIT;
import static tap.config.TAPConfiguration.KEY_DEFAULT_RETENTION_PERIOD;
import static tap.config.TAPConfiguration.KEY_DEFAULT_UPLOAD_LIMIT;
import static tap.config.TAPConfiguration.KEY_DIRECTORY_PER_USER;
import static tap.config.TAPConfiguration.KEY_DISABILITY_REASON;
import static tap.config.TAPConfiguration.KEY_FILE_MANAGER;
......@@ -36,9 +19,12 @@ import static tap.config.TAPConfiguration.KEY_IS_AVAILABLE;
import static tap.config.TAPConfiguration.KEY_MAX_EXECUTION_DURATION;
import static tap.config.TAPConfiguration.KEY_MAX_OUTPUT_LIMIT;
import static tap.config.TAPConfiguration.KEY_MAX_RETENTION_PERIOD;
import static tap.config.TAPConfiguration.KEY_MAX_UPLOAD_LIMIT;
import static tap.config.TAPConfiguration.KEY_OUTPUT_FORMATS;
import static tap.config.TAPConfiguration.KEY_PROVIDER_NAME;
import static tap.config.TAPConfiguration.KEY_SERVICE_DESCRIPTION;
import static tap.config.TAPConfiguration.KEY_UPLOAD_ENABLED;
import static tap.config.TAPConfiguration.KEY_UPLOAD_MAX_FILE_SIZE;
import static tap.config.TAPConfiguration.VALUE_CSV;
import static tap.config.TAPConfiguration.VALUE_JSON;
import static tap.config.TAPConfiguration.VALUE_LOCAL;
......@@ -72,11 +58,6 @@ import tap.metadata.TAPMetadata;
import uws.UWSException;
import uws.service.UserIdentifier;
/**
*
* @author Gr&eacute;gory Mantelet (ARI) - gmantele@ari.uni-heidelberg.de
* @version 1.1 (12/2013)
*/
public final class DefaultServiceConnection implements ServiceConnection<ResultSet> {
private TAPFileManager fileManager;
......@@ -88,7 +69,7 @@ public final class DefaultServiceConnection implements ServiceConnection<ResultS
private final String providerName;
private final String serviceDescription;
private boolean isAvailable = false;
private boolean isAvailable = false; // the TAP service must be disabled until the end of its connection initialization
private String availability = null;
private int[] executionDuration = new int[2];
......@@ -96,9 +77,14 @@ public final class DefaultServiceConnection implements ServiceConnection<ResultS
private final ArrayList<OutputFormat<ResultSet>> outputFormats;
private int[] outputLimits = new int[2];
private int[] outputLimits = new int[]{-1,-1};
private LimitUnit[] outputLimitTypes = new LimitUnit[2];
private boolean isUploadEnabled = false;
private int[] uploadLimits = new int[]{-1,-1};
private LimitUnit[] uploadLimitTypes = new LimitUnit[2];
private int maxUploadSize = DEFAULT_UPLOAD_MAX_FILE_SIZE;
public DefaultServiceConnection(final Properties tapConfig) throws NullPointerException, TAPException, UWSException{
// 1. INITIALIZE THE FILE MANAGER:
initFileManager(tapConfig);
......@@ -125,7 +111,15 @@ public final class DefaultServiceConnection implements ServiceConnection<ResultS
// set output limits:
initOutputLimits(tapConfig);
// 5. MAKE THE SERVICE AVAILABLE (or not, depending on the property value):
// 6. CONFIGURE THE UPLOAD:
// is upload enabled ?
isUploadEnabled = Boolean.parseBoolean(getProperty(tapConfig, KEY_UPLOAD_ENABLED));
// set upload limits:
initUploadLimits(tapConfig);
// set the maximum upload file size:
initMaxUploadSize(tapConfig);
// 7. MAKE THE SERVICE AVAILABLE (or not, depending on the property value):
String propValue = getProperty(tapConfig, KEY_IS_AVAILABLE);
isAvailable = (propValue == null) ? DEFAULT_IS_AVAILABLE : Boolean.parseBoolean(propValue);
}
......@@ -302,16 +296,46 @@ public final class DefaultServiceConnection implements ServiceConnection<ResultS
private void initOutputLimits(final Properties tapConfig) throws TAPException{
Object[] limit = parseLimit(getProperty(tapConfig, KEY_DEFAULT_OUTPUT_LIMIT), KEY_DEFAULT_OUTPUT_LIMIT, false);
outputLimitTypes[0] = (LimitUnit)limit[1];
outputLimitTypes[0] = (LimitUnit)limit[1]; // it should be "rows" since the parameter areBytesAllowed of parseLimit =false
setDefaultOutputLimit((int)limit[0]);
limit = parseLimit(getProperty(tapConfig, KEY_MAX_OUTPUT_LIMIT), KEY_DEFAULT_OUTPUT_LIMIT, false);
outputLimitTypes[1] = (LimitUnit)limit[1];
outputLimitTypes[1] = (LimitUnit)limit[1]; // it should be "rows" since the parameter areBytesAllowed of parseLimit =false
if (!setMaxOutputLimit((int)limit[0]))
throw new TAPException("The default output limit (here: " + outputLimits[0] + ") MUST be less or equal to the maximum output limit (here: " + limit[0] + ")!");
}
private void initUploadLimits(final Properties tapConfig) throws TAPException{
Object[] limit = parseLimit(getProperty(tapConfig, KEY_DEFAULT_UPLOAD_LIMIT), KEY_DEFAULT_UPLOAD_LIMIT, true);
uploadLimitTypes[0] = (LimitUnit)limit[1];
setDefaultUploadLimit((int)limit[0]);
limit = parseLimit(getProperty(tapConfig, KEY_MAX_UPLOAD_LIMIT), KEY_MAX_UPLOAD_LIMIT, true);
if (!((LimitUnit)limit[1]).isCompatibleWith(uploadLimitTypes[0]))
throw new TAPException("The default upload limit (in " + uploadLimitTypes[0] + ") and the maximum upload limit (in " + limit[1] + ") MUST be expressed in the same unit!");
else
uploadLimitTypes[1] = (LimitUnit)limit[1];
if (!setMaxUploadLimit((int)limit[0]))
throw new TAPException("The default upload limit (here: " + getProperty(tapConfig, KEY_DEFAULT_UPLOAD_LIMIT) + ") MUST be less or equal to the maximum upload limit (here: " + getProperty(tapConfig, KEY_MAX_UPLOAD_LIMIT) + ")!");
}
private void initMaxUploadSize(final Properties tapConfig) throws TAPException{
String propValue = getProperty(tapConfig, KEY_UPLOAD_MAX_FILE_SIZE);
// If a value is specified...
if (propValue != null){
// ...parse the value:
Object[] limit = parseLimit(propValue, KEY_UPLOAD_MAX_FILE_SIZE, true);
// ...check that the unit is correct (bytes):
if (!LimitUnit.bytes.isCompatibleWith((LimitUnit)limit[1]))
throw new TAPException("The maximum upload file size " + KEY_UPLOAD_MAX_FILE_SIZE + " (here: " + propValue + ") can not be expressed in a unit different from bytes (B, kB, MB, GB)!");
// ...set the max file size:
int value = (int)((int)limit[0] * ((LimitUnit)limit[1]).bytesFactor());
setMaxUploadSize(value);
}
}
@Override
public String getProviderName(){
return providerName;
......@@ -459,38 +483,76 @@ public final class DefaultServiceConnection implements ServiceConnection<ResultS
}
@Override
public UserIdentifier getUserIdentifier(){
// TODO Auto-generated method stub
return null;
public boolean uploadEnabled(){
return isUploadEnabled;
}
@Override
public boolean uploadEnabled(){
// TODO Auto-generated method stub
return false;
public void setUploadEnabled(final boolean enabled){
isUploadEnabled = enabled;
}
@Override
public int[] getUploadLimit(){
// TODO Auto-generated method stub
return null;
return uploadLimits;
}
@Override
public LimitUnit[] getUploadLimitType(){
// TODO Auto-generated method stub
return null;
return uploadLimitTypes;
}
public void setUploadLimitType(final LimitUnit type){
if (type != null)
uploadLimitTypes = new LimitUnit[]{type,type};
}
public boolean setDefaultUploadLimit(final int limit){
try{
if ((uploadLimits[1] <= 0) || (limit > 0 && LimitUnit.compare(limit, uploadLimitTypes[0], uploadLimits[1], uploadLimitTypes[1]) <= 0)){
uploadLimits[0] = limit;
return true;
}
}catch(TAPException e){}
return false;
}
public boolean setMaxUploadLimit(final int limit){
try{
if (limit > 0 && uploadLimits[0] > 0 && LimitUnit.compare(limit, uploadLimitTypes[1], uploadLimits[0], uploadLimitTypes[0]) < 0)
return false;
else{
uploadLimits[1] = limit;
return true;
}
}catch(TAPException e){
return false;
}
}
@Override
public int getMaxUploadSize(){
// TODO Auto-generated method stub
return 0;
return maxUploadSize;
}
public boolean setMaxUploadSize(final int maxSize){
// No "unlimited" value possible there:
if (maxSize <= 0)
return false;
// Otherwise, set the maximum upload file size:
maxUploadSize = maxSize;
return true;
}
@Override
public UserIdentifier getUserIdentifier(){
// TODO DefaultServiceConnection.getUserIdentifier
return null;
}
@Override
public TAPMetadata getTAPMetadata(){
// TODO Auto-generated method stub
// TODO DefaultServiceConnection.getTAPMetadata
return null;
}
......
src/tap/config/TAPConfiguration.java
+ 7
− 24
View file @ 0db34a50
package tap.config;
/*
* This file is part of TAPLibrary.
*
* TAPLibrary is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* TAPLibrary is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with TAPLibrary. If not, see <http://www.gnu.org/licenses/>.
*
* Copyright 2013 - Astronomisches Rechen Institute (ARI)
*/
import java.io.File;
import java.io.FileInputStream;
import java.util.Enumeration;
......@@ -29,11 +10,6 @@ import tap.ServiceConnection.LimitUnit;
import tap.TAPException;
import tap.backup.DefaultTAPBackupManager;
/**
*
* @author Gr&eacute;gory Mantelet (ARI) - gmantele@ari.uni-heidelberg.de
* @version 1.1 (12/2013)
*/
public final class TAPConfiguration {
/* FILE MANAGER KEYS */
......@@ -86,6 +62,13 @@ public final class TAPConfiguration {
public final static boolean DEFAULT_IS_AVAILABLE = true;
public final static String KEY_DISABILITY_REASON = "disability_reason";
/* UPLOAD KEYS */
public final static String KEY_UPLOAD_ENABLED = "upload_enabled";
public final static String KEY_DEFAULT_UPLOAD_LIMIT = "upload_default_db_limit";
public final static String KEY_MAX_UPLOAD_LIMIT = "upload_max_db_limit";
public final static String KEY_UPLOAD_MAX_FILE_SIZE = "upload_max_file_size";
public final static int DEFAULT_UPLOAD_MAX_FILE_SIZE = Integer.MAX_VALUE;
/* OUTPUT KEYS */
public final static String KEY_OUTPUT_FORMATS = "output_add_formats";
public final static String VALUE_JSON = "json";
......
src/tap/config/tap_configuration_file.html
+ 23
− 17
View file @ 0db34a50
......@@ -67,6 +67,10 @@
<p>Here is an empty minimum TAP configuration file: <a href="tap_min.properties">tap_min.properties</a> and a complete one: <a href="tap_full.properties">tap_full.properties</a>.</p>
<p><b>Important note:</b> Any limit value is an integer and so can be at most: 2<sup>31</sup>-1 bytes/rows = 2147483647B/R (or also for the byte unit: = 2147483kB = 2147MB = 2GB).
Otherwise, you should use the null value 0 to raise the limit constraint.</p>
<p><i><u>Legend:</u> <b>M</b> means that the property is mandatory. If nothing is written for the second column, the property is optional.</i>
<table>
......@@ -413,33 +417,34 @@
<tr><td colspan="5">Upload</td></tr>
<tr>
<td class="todo">upload_enabled</td>
<td class="done">upload_enabled</td>
<td></td>
<td>boolean</td>
<td>
<p>Tells whether the Upload must be enabled. If enabled, files can be uploaded in the file_root_path,
the corresponding tables can be added inside the UPLOAD_SCHEMA of the database, queried and then deleted.</p>
<p>By default, the Upload is disabled: upload_enabled=false.</p>
<p><i>By default, the Upload is disabled: upload_enabled=false.</i></p>
</td>
<td><ul><li>false <em>(default)</em></li><li>true</li></ul></td>
</tr>
<tr>
<td class="todo">upload_default_db_limit</td>
<td class="done">upload_default_db_limit</td>
<td></td>
<td>text</td>
<td>
<p>Default limit for the number of uploaded records that can be inserted inside the database. The prefix "default" means here that this value will be set if the client does not provide one.</p>
<p>This limit can be expressed with 2 types: rows or bytes. For rows, you just have to suffix the value by a "r" (upper- or lower-case),
with nothing (by default, nothing will mean "rows"). For bytes, you have to suffix the numeric value by "B", "kB", "MB" or "GB".
<p>This limit can be expressed with 2 types: rows or bytes. For rows, you just have to suffix the value by a "r" (upper- or lower-case)
or by nothing (by default, nothing will mean "rows"). For bytes, you have to suffix the numeric value by "B", "kB", "MB" or "GB".
Here, unit is case sensitive. No other storage unit is allowed.</p>
<p>A negative or null value means there is no restriction over this limit. Float values are not allowed.</p>
<p>Obviously this limit MUST be less or equal than upload_max_db_limit.</p>
<p><em>By default, there is no restriction: upload_default_db_limit=0</em></p>
<p><b>Warning!</b> Obviously this limit MUST be less or equal than upload_max_db_limit, and MUST be of the same type as it.
If the chosen type is rows, this limit MUST also be strictly less than upload_max_file_size.</p>
<p><i>By default, there is no restriction: upload_default_db_limit=0</i></p>
</td>
<td><ul><li>0 <em>(default)</em></li><li>20r</li><li>20R</li><li>200kB</li></ul></td>
<td><ul><li>0 <em>(default)</em></li><li>20</li><li>20r</li><li>20R</li><li>200kB</li></ul></td>
</tr>
<tr>
<td class="todo">upload_max_db_limit</td>
<td class="done">upload_max_db_limit</td>
<td></td>
<td>text</td>
<td>
......@@ -448,24 +453,25 @@
with nothing (by default, nothing will mean "rows"). For bytes, you have to suffix the numeric value by "B", "kB", "MB" or "GB".
Here, unit is case sensitive. No other storage unit is allowed.</p>
<p>A negative or null value means there is no restriction over this limit. Float values are not allowed.</p>
<p>Obviously this limit MUST be greater or equal than upload_default_db_limit.</p>
<p><em>By default, there is no restriction: upload_max_db_limit=0</em></p>
<p><b>Warning!</b> Obviously this limit MUST be greater or equal than upload_default_db_limit, and MUST be of the same type as it.
If the chosen type is rows, this limit MUST also be strictly less than upload_max_file_size.</p>
<p><i>By default, there is no restriction: upload_max_db_limit=0</i></p>
</td>
<td><ul><li>0 <em>(default)</em></li><li>10000r</li><li>10000R</li><li>1MB</li></ul></td>
<td><ul><li>0 <em>(default)</em></li><li>10000</li><li>10000r</li><li>10000R</li><li>1MB</li></ul></td>
</tr>
<tr>
<td class="todo">upload_max_file_size</td>
<td class="done">upload_max_file_size</td>
<td></td>
<td>text</td>
<td>
<p>Maximum allowed size for the uploaded file.</p>
<p>This limit MUST be expressed in bytes. Thus, you have to suffix the numeric value by "B", "kB", "MB" or "GB".
Here, unit is case sensitive. No other storage unit is allowed.</p>
<p>A negative or null value means there is no restriction over this limit. Float values are not allowed.</p>
<p>In function of the chosen upload_max_db_limit type, upload_max_file_size should be greater in order to figure out the metadata part.</p>
<p><em>By default, there is no restriction: upload_max_file_size=0</em></p>
<p><b>Warning!</b> When the upload is enabled, there must be a maximum file size. Here, no "unlimited" value is possible ; 0 and any negative value are not allowed.</p>
<p><b>Warning!</b> In function of the chosen upload_max_db_limit type, upload_max_file_size MUST be greater in order to figure out the file metadata part.</p>
<p><i>By default, the maximum size is set to its maximum possible value: upload_max_file_size=2147483647B (~2GB)</i></p>
</td>
<td><ul><li>0 <em>(default)</em></li><li>2MB</li></ul></td>
<td><ul><li>2147483647B <em>(default)</em></li><li>2MB</li></ul></td>
</tr>
</table>
</body>
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment