From 0db34a50f09794d35f99079a7c553a9462e3db5e Mon Sep 17 00:00:00 2001
From: gmantele <gmantele@ari.uni-heidelberg.de>
Date: Thu, 10 Apr 2014 12:10:58 +0200
Subject: [PATCH] TAP: Add the upload limits in TAPConfiguration and its
default implementation
---
src/tap/ServiceConnection.java | 79 +++++++++-
src/tap/config/DefaultServiceConnection.java | 148 +++++++++++++------
src/tap/config/TAPConfiguration.java | 31 +---
src/tap/config/tap_configuration_file.html | 40 ++---
4 files changed, 212 insertions(+), 86 deletions(-)
diff --git a/src/tap/ServiceConnection.java b/src/tap/ServiceConnection.java
index 1017e8d..2f83b1a 100644
--- a/src/tap/ServiceConnection.java
+++ b/src/tap/ServiceConnection.java
@@ -16,7 +16,7 @@ package tap;
* You should have received a copy of the GNU Lesser General Public License
* along with TAPLibrary. If not, see <http://www.gnu.org/licenses/>.
*
- * Copyright 2012-2013 - UDS/Centre de Données astronomiques de Strasbourg (CDS),
+ * Copyright 2012-2014 - UDS/Centre de Données astronomiques de Strasbourg (CDS),
* Astronomisches Rechen Institute (ARI)
*/
@@ -33,14 +33,89 @@ import uws.service.UserIdentifier;
*
*
* @author Grégory Mantelet (CDS;ARI) - gmantele@ari.uni-heidelberg.de
- * @version 1.1 (12/2013)
+ * @version 1.1 (01/2014)
*
* @param <R>
*/
public interface ServiceConnection< R > {
+ /**
+ * Units used to express any limit of the TAP service.
+ *
+ * @author Grégory Mantelet (ARI) - gmantele@ari.uni-heidelberg.de
+ * @version 1.1 (01/2014)
+ */
public static enum LimitUnit{
rows, bytes, kilobytes, megabytes, gigabytes;
+
+ /**
+ * Tells whether the given unit has the same type (bytes or rows).
+ *
+ * @param anotherUnit A unit.
+ *
+ * @return true if the given unit has the same type, false otherwise.
+ *
+ * @since 1.1
+ */
+ public boolean isCompatibleWith(final LimitUnit anotherUnit){
+ if (this == rows)
+ return anotherUnit == rows;
+ else
+ return anotherUnit != rows;
+ }
+
+ /**
+ * Gets the factor to convert into bytes the value expressed in this unit.
+ * <i>Note: if this unit is not a factor of bytes, 1 is returned (so that the factor does not affect the value).</i>
+ *
+ * @return The factor need to convert a value expressed in this unit into bytes, or 1 if not a bytes derived unit.
+ *
+ * @since 1.1
+ */
+ public long bytesFactor(){
+ switch(this){
+ case bytes:
+ return 1;
+ case kilobytes:
+ return 1000;
+ case megabytes:
+ return 1000000;
+ case gigabytes:
+ return 1000000000l;
+ default:
+ return 1;
+ }
+ }
+
+ /**
+ * Compares the 2 given values (each one expressed in the given unit).
+ * Conversions are done internally in order to make a correct comparison between the 2 limits.
+ *
+ * @param leftLimit Value/Limit of the comparison left part.
+ * @param leftUnit Unit of the comparison left part value.
+ * @param rightLimit Value/Limit of the comparison right part.
+ * @param rightUnit Unit of the comparison right part value.
+ *
+ * @return the value 0 if x == y; a value less than 0 if x < y; and a value greater than 0 if x > y
+ *
+ * @throws TAPException If the two given units are not compatible.
+ *
+ * @see #isCompatibleWith(LimitUnit)
+ * @see #bytesFactor()
+ * @see Integer#compare(int, int)
+ * @see Long#compare(long, long)
+ *
+ * @since 1.1
+ */
+ public static int compare(final int leftLimit, final LimitUnit leftUnit, final int rightLimit, final LimitUnit rightUnit) throws TAPException{
+ if (!leftUnit.isCompatibleWith(rightUnit))
+ throw new TAPException("Limit units (" + leftUnit + " and " + rightUnit + ") are not compatible!");
+
+ if (leftUnit == rows || leftUnit == rightUnit)
+ return Integer.compare(leftLimit, rightLimit);
+ else
+ return Long.compare(leftLimit * leftUnit.bytesFactor(), rightLimit * rightUnit.bytesFactor());
+ }
}
public String getProviderName();
diff --git a/src/tap/config/DefaultServiceConnection.java b/src/tap/config/DefaultServiceConnection.java
index c3bc559..42094b7 100644
--- a/src/tap/config/DefaultServiceConnection.java
+++ b/src/tap/config/DefaultServiceConnection.java
@@ -1,32 +1,15 @@
package tap.config;
-/*
- * This file is part of TAPLibrary.
- *
- * TAPLibrary is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Lesser General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * TAPLibrary is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with TAPLibrary. If not, see <http://www.gnu.org/licenses/>.
- *
- * Copyright 2013 - Astronomisches Rechen Institute (ARI)
- */
-
import static tap.config.TAPConfiguration.DEFAULT_DIRECTORY_PER_USER;
import static tap.config.TAPConfiguration.DEFAULT_EXECUTION_DURATION;
import static tap.config.TAPConfiguration.DEFAULT_GROUP_USER_DIRECTORIES;
import static tap.config.TAPConfiguration.DEFAULT_IS_AVAILABLE;
import static tap.config.TAPConfiguration.DEFAULT_RETENTION_PERIOD;
+import static tap.config.TAPConfiguration.DEFAULT_UPLOAD_MAX_FILE_SIZE;
import static tap.config.TAPConfiguration.KEY_DEFAULT_EXECUTION_DURATION;
import static tap.config.TAPConfiguration.KEY_DEFAULT_OUTPUT_LIMIT;
import static tap.config.TAPConfiguration.KEY_DEFAULT_RETENTION_PERIOD;
+import static tap.config.TAPConfiguration.KEY_DEFAULT_UPLOAD_LIMIT;
import static tap.config.TAPConfiguration.KEY_DIRECTORY_PER_USER;
import static tap.config.TAPConfiguration.KEY_DISABILITY_REASON;
import static tap.config.TAPConfiguration.KEY_FILE_MANAGER;
@@ -36,9 +19,12 @@ import static tap.config.TAPConfiguration.KEY_IS_AVAILABLE;
import static tap.config.TAPConfiguration.KEY_MAX_EXECUTION_DURATION;
import static tap.config.TAPConfiguration.KEY_MAX_OUTPUT_LIMIT;
import static tap.config.TAPConfiguration.KEY_MAX_RETENTION_PERIOD;
+import static tap.config.TAPConfiguration.KEY_MAX_UPLOAD_LIMIT;
import static tap.config.TAPConfiguration.KEY_OUTPUT_FORMATS;
import static tap.config.TAPConfiguration.KEY_PROVIDER_NAME;
import static tap.config.TAPConfiguration.KEY_SERVICE_DESCRIPTION;
+import static tap.config.TAPConfiguration.KEY_UPLOAD_ENABLED;
+import static tap.config.TAPConfiguration.KEY_UPLOAD_MAX_FILE_SIZE;
import static tap.config.TAPConfiguration.VALUE_CSV;
import static tap.config.TAPConfiguration.VALUE_JSON;
import static tap.config.TAPConfiguration.VALUE_LOCAL;
@@ -72,11 +58,6 @@ import tap.metadata.TAPMetadata;
import uws.UWSException;
import uws.service.UserIdentifier;
-/**
- *
- * @author Grégory Mantelet (ARI) - gmantele@ari.uni-heidelberg.de
- * @version 1.1 (12/2013)
- */
public final class DefaultServiceConnection implements ServiceConnection<ResultSet> {
private TAPFileManager fileManager;
@@ -88,7 +69,7 @@ public final class DefaultServiceConnection implements ServiceConnection<ResultS
private final String providerName;
private final String serviceDescription;
- private boolean isAvailable = false;
+ private boolean isAvailable = false; // the TAP service must be disabled until the end of its connection initialization
private String availability = null;
private int[] executionDuration = new int[2];
@@ -96,9 +77,14 @@ public final class DefaultServiceConnection implements ServiceConnection<ResultS
private final ArrayList<OutputFormat<ResultSet>> outputFormats;
- private int[] outputLimits = new int[2];
+ private int[] outputLimits = new int[]{-1,-1};
private LimitUnit[] outputLimitTypes = new LimitUnit[2];
+ private boolean isUploadEnabled = false;
+ private int[] uploadLimits = new int[]{-1,-1};
+ private LimitUnit[] uploadLimitTypes = new LimitUnit[2];
+ private int maxUploadSize = DEFAULT_UPLOAD_MAX_FILE_SIZE;
+
public DefaultServiceConnection(final Properties tapConfig) throws NullPointerException, TAPException, UWSException{
// 1. INITIALIZE THE FILE MANAGER:
initFileManager(tapConfig);
@@ -125,7 +111,15 @@ public final class DefaultServiceConnection implements ServiceConnection<ResultS
// set output limits:
initOutputLimits(tapConfig);
- // 5. MAKE THE SERVICE AVAILABLE (or not, depending on the property value):
+ // 6. CONFIGURE THE UPLOAD:
+ // is upload enabled ?
+ isUploadEnabled = Boolean.parseBoolean(getProperty(tapConfig, KEY_UPLOAD_ENABLED));
+ // set upload limits:
+ initUploadLimits(tapConfig);
+ // set the maximum upload file size:
+ initMaxUploadSize(tapConfig);
+
+ // 7. MAKE THE SERVICE AVAILABLE (or not, depending on the property value):
String propValue = getProperty(tapConfig, KEY_IS_AVAILABLE);
isAvailable = (propValue == null) ? DEFAULT_IS_AVAILABLE : Boolean.parseBoolean(propValue);
}
@@ -302,16 +296,46 @@ public final class DefaultServiceConnection implements ServiceConnection<ResultS
private void initOutputLimits(final Properties tapConfig) throws TAPException{
Object[] limit = parseLimit(getProperty(tapConfig, KEY_DEFAULT_OUTPUT_LIMIT), KEY_DEFAULT_OUTPUT_LIMIT, false);
- outputLimitTypes[0] = (LimitUnit)limit[1];
+ outputLimitTypes[0] = (LimitUnit)limit[1]; // it should be "rows" since the parameter areBytesAllowed of parseLimit =false
setDefaultOutputLimit((int)limit[0]);
limit = parseLimit(getProperty(tapConfig, KEY_MAX_OUTPUT_LIMIT), KEY_DEFAULT_OUTPUT_LIMIT, false);
- outputLimitTypes[1] = (LimitUnit)limit[1];
+ outputLimitTypes[1] = (LimitUnit)limit[1]; // it should be "rows" since the parameter areBytesAllowed of parseLimit =false
if (!setMaxOutputLimit((int)limit[0]))
throw new TAPException("The default output limit (here: " + outputLimits[0] + ") MUST be less or equal to the maximum output limit (here: " + limit[0] + ")!");
}
+ private void initUploadLimits(final Properties tapConfig) throws TAPException{
+ Object[] limit = parseLimit(getProperty(tapConfig, KEY_DEFAULT_UPLOAD_LIMIT), KEY_DEFAULT_UPLOAD_LIMIT, true);
+ uploadLimitTypes[0] = (LimitUnit)limit[1];
+ setDefaultUploadLimit((int)limit[0]);
+
+ limit = parseLimit(getProperty(tapConfig, KEY_MAX_UPLOAD_LIMIT), KEY_MAX_UPLOAD_LIMIT, true);
+ if (!((LimitUnit)limit[1]).isCompatibleWith(uploadLimitTypes[0]))
+ throw new TAPException("The default upload limit (in " + uploadLimitTypes[0] + ") and the maximum upload limit (in " + limit[1] + ") MUST be expressed in the same unit!");
+ else
+ uploadLimitTypes[1] = (LimitUnit)limit[1];
+
+ if (!setMaxUploadLimit((int)limit[0]))
+ throw new TAPException("The default upload limit (here: " + getProperty(tapConfig, KEY_DEFAULT_UPLOAD_LIMIT) + ") MUST be less or equal to the maximum upload limit (here: " + getProperty(tapConfig, KEY_MAX_UPLOAD_LIMIT) + ")!");
+ }
+
+ private void initMaxUploadSize(final Properties tapConfig) throws TAPException{
+ String propValue = getProperty(tapConfig, KEY_UPLOAD_MAX_FILE_SIZE);
+ // If a value is specified...
+ if (propValue != null){
+ // ...parse the value:
+ Object[] limit = parseLimit(propValue, KEY_UPLOAD_MAX_FILE_SIZE, true);
+ // ...check that the unit is correct (bytes):
+ if (!LimitUnit.bytes.isCompatibleWith((LimitUnit)limit[1]))
+ throw new TAPException("The maximum upload file size " + KEY_UPLOAD_MAX_FILE_SIZE + " (here: " + propValue + ") can not be expressed in a unit different from bytes (B, kB, MB, GB)!");
+ // ...set the max file size:
+ int value = (int)((int)limit[0] * ((LimitUnit)limit[1]).bytesFactor());
+ setMaxUploadSize(value);
+ }
+ }
+
@Override
public String getProviderName(){
return providerName;
@@ -459,38 +483,76 @@ public final class DefaultServiceConnection implements ServiceConnection<ResultS
}
@Override
- public UserIdentifier getUserIdentifier(){
- // TODO Auto-generated method stub
- return null;
+ public boolean uploadEnabled(){
+ return isUploadEnabled;
}
- @Override
- public boolean uploadEnabled(){
- // TODO Auto-generated method stub
- return false;
+ public void setUploadEnabled(final boolean enabled){
+ isUploadEnabled = enabled;
}
@Override
public int[] getUploadLimit(){
- // TODO Auto-generated method stub
- return null;
+ return uploadLimits;
}
@Override
public LimitUnit[] getUploadLimitType(){
- // TODO Auto-generated method stub
- return null;
+ return uploadLimitTypes;
+ }
+
+ public void setUploadLimitType(final LimitUnit type){
+ if (type != null)
+ uploadLimitTypes = new LimitUnit[]{type,type};
+ }
+
+ public boolean setDefaultUploadLimit(final int limit){
+ try{
+ if ((uploadLimits[1] <= 0) || (limit > 0 && LimitUnit.compare(limit, uploadLimitTypes[0], uploadLimits[1], uploadLimitTypes[1]) <= 0)){
+ uploadLimits[0] = limit;
+ return true;
+ }
+ }catch(TAPException e){}
+ return false;
+ }
+
+ public boolean setMaxUploadLimit(final int limit){
+ try{
+ if (limit > 0 && uploadLimits[0] > 0 && LimitUnit.compare(limit, uploadLimitTypes[1], uploadLimits[0], uploadLimitTypes[0]) < 0)
+ return false;
+ else{
+ uploadLimits[1] = limit;
+ return true;
+ }
+ }catch(TAPException e){
+ return false;
+ }
}
@Override
public int getMaxUploadSize(){
- // TODO Auto-generated method stub
- return 0;
+ return maxUploadSize;
+ }
+
+ public boolean setMaxUploadSize(final int maxSize){
+ // No "unlimited" value possible there:
+ if (maxSize <= 0)
+ return false;
+
+ // Otherwise, set the maximum upload file size:
+ maxUploadSize = maxSize;
+ return true;
+ }
+
+ @Override
+ public UserIdentifier getUserIdentifier(){
+ // TODO DefaultServiceConnection.getUserIdentifier
+ return null;
}
@Override
public TAPMetadata getTAPMetadata(){
- // TODO Auto-generated method stub
+ // TODO DefaultServiceConnection.getTAPMetadata
return null;
}
diff --git a/src/tap/config/TAPConfiguration.java b/src/tap/config/TAPConfiguration.java
index 4e15c37..4a7e6e9 100644
--- a/src/tap/config/TAPConfiguration.java
+++ b/src/tap/config/TAPConfiguration.java
@@ -1,24 +1,5 @@
package tap.config;
-/*
- * This file is part of TAPLibrary.
- *
- * TAPLibrary is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Lesser General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * TAPLibrary is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with TAPLibrary. If not, see <http://www.gnu.org/licenses/>.
- *
- * Copyright 2013 - Astronomisches Rechen Institute (ARI)
- */
-
import java.io.File;
import java.io.FileInputStream;
import java.util.Enumeration;
@@ -29,11 +10,6 @@ import tap.ServiceConnection.LimitUnit;
import tap.TAPException;
import tap.backup.DefaultTAPBackupManager;
-/**
- *
- * @author Grégory Mantelet (ARI) - gmantele@ari.uni-heidelberg.de
- * @version 1.1 (12/2013)
- */
public final class TAPConfiguration {
/* FILE MANAGER KEYS */
@@ -86,6 +62,13 @@ public final class TAPConfiguration {
public final static boolean DEFAULT_IS_AVAILABLE = true;
public final static String KEY_DISABILITY_REASON = "disability_reason";
+ /* UPLOAD KEYS */
+ public final static String KEY_UPLOAD_ENABLED = "upload_enabled";
+ public final static String KEY_DEFAULT_UPLOAD_LIMIT = "upload_default_db_limit";
+ public final static String KEY_MAX_UPLOAD_LIMIT = "upload_max_db_limit";
+ public final static String KEY_UPLOAD_MAX_FILE_SIZE = "upload_max_file_size";
+ public final static int DEFAULT_UPLOAD_MAX_FILE_SIZE = Integer.MAX_VALUE;
+
/* OUTPUT KEYS */
public final static String KEY_OUTPUT_FORMATS = "output_add_formats";
public final static String VALUE_JSON = "json";
diff --git a/src/tap/config/tap_configuration_file.html b/src/tap/config/tap_configuration_file.html
index 6662167..fafb6bc 100644
--- a/src/tap/config/tap_configuration_file.html
+++ b/src/tap/config/tap_configuration_file.html
@@ -67,6 +67,10 @@
<p>Here is an empty minimum TAP configuration file: <a href="tap_min.properties">tap_min.properties</a> and a complete one: <a href="tap_full.properties">tap_full.properties</a>.</p>
+
+ <p><b>Important note:</b> Any limit value is an integer and so can be at most: 2<sup>31</sup>-1 bytes/rows = 2147483647B/R (or also for the byte unit: = 2147483kB = 2147MB = 2GB).
+ Otherwise, you should use the null value 0 to raise the limit constraint.</p>
+
<p><i><u>Legend:</u> <b>M</b> means that the property is mandatory. If nothing is written for the second column, the property is optional.</i>
<table>
@@ -413,33 +417,34 @@
<tr><td colspan="5">Upload</td></tr>
<tr>
- <td class="todo">upload_enabled</td>
+ <td class="done">upload_enabled</td>
<td></td>
<td>boolean</td>
<td>
<p>Tells whether the Upload must be enabled. If enabled, files can be uploaded in the file_root_path,
the corresponding tables can be added inside the UPLOAD_SCHEMA of the database, queried and then deleted.</p>
- <p>By default, the Upload is disabled: upload_enabled=false.</p>
+ <p><i>By default, the Upload is disabled: upload_enabled=false.</i></p>
</td>
<td><ul><li>false <em>(default)</em></li><li>true</li></ul></td>
</tr>
<tr>
- <td class="todo">upload_default_db_limit</td>
+ <td class="done">upload_default_db_limit</td>
<td></td>
<td>text</td>
<td>
<p>Default limit for the number of uploaded records that can be inserted inside the database. The prefix "default" means here that this value will be set if the client does not provide one.</p>
- <p>This limit can be expressed with 2 types: rows or bytes. For rows, you just have to suffix the value by a "r" (upper- or lower-case),
- with nothing (by default, nothing will mean "rows"). For bytes, you have to suffix the numeric value by "B", "kB", "MB" or "GB".
+ <p>This limit can be expressed with 2 types: rows or bytes. For rows, you just have to suffix the value by a "r" (upper- or lower-case)
+ or by nothing (by default, nothing will mean "rows"). For bytes, you have to suffix the numeric value by "B", "kB", "MB" or "GB".
Here, unit is case sensitive. No other storage unit is allowed.</p>
<p>A negative or null value means there is no restriction over this limit. Float values are not allowed.</p>
- <p>Obviously this limit MUST be less or equal than upload_max_db_limit.</p>
- <p><em>By default, there is no restriction: upload_default_db_limit=0</em></p>
+ <p><b>Warning!</b> Obviously this limit MUST be less or equal than upload_max_db_limit, and MUST be of the same type as it.
+ If the chosen type is rows, this limit MUST also be strictly less than upload_max_file_size.</p>
+ <p><i>By default, there is no restriction: upload_default_db_limit=0</i></p>
</td>
- <td><ul><li>0 <em>(default)</em></li><li>20r</li><li>20R</li><li>200kB</li></ul></td>
+ <td><ul><li>0 <em>(default)</em></li><li>20</li><li>20r</li><li>20R</li><li>200kB</li></ul></td>
</tr>
<tr>
- <td class="todo">upload_max_db_limit</td>
+ <td class="done">upload_max_db_limit</td>
<td></td>
<td>text</td>
<td>
@@ -448,24 +453,25 @@
with nothing (by default, nothing will mean "rows"). For bytes, you have to suffix the numeric value by "B", "kB", "MB" or "GB".
Here, unit is case sensitive. No other storage unit is allowed.</p>
<p>A negative or null value means there is no restriction over this limit. Float values are not allowed.</p>
- <p>Obviously this limit MUST be greater or equal than upload_default_db_limit.</p>
- <p><em>By default, there is no restriction: upload_max_db_limit=0</em></p>
+ <p><b>Warning!</b> Obviously this limit MUST be greater or equal than upload_default_db_limit, and MUST be of the same type as it.
+ If the chosen type is rows, this limit MUST also be strictly less than upload_max_file_size.</p>
+ <p><i>By default, there is no restriction: upload_max_db_limit=0</i></p>
</td>
- <td><ul><li>0 <em>(default)</em></li><li>10000r</li><li>10000R</li><li>1MB</li></ul></td>
+ <td><ul><li>0 <em>(default)</em></li><li>10000</li><li>10000r</li><li>10000R</li><li>1MB</li></ul></td>
</tr>
<tr>
- <td class="todo">upload_max_file_size</td>
+ <td class="done">upload_max_file_size</td>
<td></td>
<td>text</td>
<td>
<p>Maximum allowed size for the uploaded file.</p>
<p>This limit MUST be expressed in bytes. Thus, you have to suffix the numeric value by "B", "kB", "MB" or "GB".
Here, unit is case sensitive. No other storage unit is allowed.</p>
- <p>A negative or null value means there is no restriction over this limit. Float values are not allowed.</p>
- <p>In function of the chosen upload_max_db_limit type, upload_max_file_size should be greater in order to figure out the metadata part.</p>
- <p><em>By default, there is no restriction: upload_max_file_size=0</em></p>
+ <p><b>Warning!</b> When the upload is enabled, there must be a maximum file size. Here, no "unlimited" value is possible ; 0 and any negative value are not allowed.</p>
+ <p><b>Warning!</b> In function of the chosen upload_max_db_limit type, upload_max_file_size MUST be greater in order to figure out the file metadata part.</p>
+ <p><i>By default, the maximum size is set to its maximum possible value: upload_max_file_size=2147483647B (~2GB)</i></p>
</td>
- <td><ul><li>0 <em>(default)</em></li><li>2MB</li></ul></td>
+ <td><ul><li>2147483647B <em>(default)</em></li><li>2MB</li></ul></td>
</tr>
</table>
</body>
--
GitLab