diff --git a/build.gradle b/build.gradle index da03fdba9c4490385d08e5c7410bddf1dbec301d..5bbf284f2114a19f113bf57977ac54b7d3ad141f 100644 --- a/build.gradle +++ b/build.gradle @@ -8,17 +8,18 @@ repositories { } dependencies { - compile fileTree(dir: 'lib', includes: ['stil_3.1.jar','cos-1.5beta.jar']) + compile fileTree(dir: 'lib', includes: ['stil_3.1.jar']) compile 'javax.servlet:javax.servlet-api:3.0.1' compile 'postgresql:postgresql:9.1-901.jdbc4' compile 'org.slf4j:slf4j-api:1.7.25' + compile 'commons-io:commons-io:2.6' + compile 'commons-fileupload:commons-fileupload:1.3.3' testCompile 'junit:junit:4.12' testCompile 'com.h2database:h2:1.4.193' testCompile fileTree(dir: 'lib', include: 'astroh2-0.3.jar') testCompile 'org.slf4j:slf4j-simple:1.7.25' - testRuntime 'simple-jndi:simple-jndi:0.11.4.1' testRuntime 'com.vividsolutions:jts-core:1.14.0' testRuntime 'org.locationtech.spatial4j:spatial4j:0.6' diff --git a/buildTAP.xml b/buildTAP.xml index 8a10aaa15fd76e135c749e95533eb8fcc0ad8a0a..9ac8cae5615dc4d1de4e4e0899e98e6131914369 100644 --- a/buildTAP.xml +++ b/buildTAP.xml @@ -12,8 +12,10 @@ <property name="classesDir" value="${compileDir}"/> <property name="javadocDir" value="javadoc/tap" /> - <property name="cosJarName" value="cos-1.5beta.jar" /> - <property name="cosJar" value="${libDir}/${cosJarName}" /> + <property name="commonsIOJarName" value="commons-io-2.6.jar" /> + <property name="commonsIOJar" value="${libDir}/${commonsIOJarName}" /> + <property name="commonsFileUploadJarName" value="commons-fileupload-1.3.3.jar" /> + <property name="commonsFileUploadJar" value="${libDir}/${commonsFileUploadJarName}" /> <property name="stilJarName" value="stil_3.1.jar" /> <property name="stilJar" value="${libDir}/${stilJarName}" /> @@ -70,7 +72,8 @@ <!-- CLASSPATHS --> <path id="tap.classpath"> - <pathelement location="${cosJar}" /> + <pathelement location="${commonsIOJar}" /> + <pathelement location="${commonsFileUploadJar}" /> <pathelement location="${slf4jApiJar}" /> <pathelement location="${jsonJar}" /> <pathelement location="${stilJar}" /> @@ -168,12 +171,14 @@ <target name="buildLib" depends="compileLib,junitValidation" description="After 'clean', build the library JAR (only classes)."> <echo>Generate the library:</echo> <jar basedir="${classesDir}" destfile="${libJarFileWithSTIL}" includes="${includesList}"> - <zipfileset src="${cosJar}" excludes="META-INF/*" /> + <zipfileset src="${commonsIOJar}" excludes="META-INF/**" /> + <zipfileset src="${commonsFileUploadJar}" excludes="META-INF/**" /> <zipfileset src="${stilJar}" excludes="META-INF/*" /> <zipfileset dir="${srcDir}" includes="${licensePath}" /> </jar> <jar basedir="${classesDir}" destfile="${libJarFile}" includes="${includesList}"> - <zipfileset src="${cosJar}" excludes="META-INF/*" /> + <zipfileset src="${commonsIOJar}" excludes="META-INF/**" /> + <zipfileset src="${commonsFileUploadJar}" excludes="META-INF/**" /> <zipfileset dir="${srcDir}" includes="${licensePath}" /> </jar> <delete dir="${compileDir}" failonerror="true" /> @@ -182,12 +187,14 @@ <target name="buildLibAndSrc" depends="buildLib" description="After 'clean' and 'buildLib', build the sources JAR (only .java)."> <jar compress="false" destfile="${srcJarFileWithSTIL}"> <zipfileset dir="${srcDir}" includes="${includesList},${licensePath}" prefix="${srcDir}" /> - <zipfileset dir="${libDir}" includes="${cosJarName}" fullPath="${cosJar}" /> + <zipfileset dir="${libDir}" includes="${commonsIOJarName}" fullPath="${commonsIOJar}" /> + <zipfileset dir="${libDir}" includes="${commonsFileUploadJarName}" fullPath="${commonsFileUploadJar}" /> <zipfileset dir="${libDir}" includes="${stilJarName}" fullPath="${stilJar}" /> </jar> <jar compress="false" destfile="${srcJarFile}"> <zipfileset dir="${srcDir}" includes="${includesList},${licensePath}" prefix="${srcDir}" /> - <zipfileset dir="${libDir}" includes="${cosJarName}" fullPath="${cosJar}" /> + <zipfileset dir="${libDir}" includes="${commonsIOJarName}" fullPath="${commonsIOJar}" /> + <zipfileset dir="${libDir}" includes="${commonsFileUploadJarName}" fullPath="${commonsFileUploadJar}" /> </jar> </target> diff --git a/buildUWS.xml b/buildUWS.xml index 3371f5b11d36cc4a247404e3c742e9d96a75537d..f70a14718d10c03827d0d5e4243754e6f4f113b3 100644 --- a/buildUWS.xml +++ b/buildUWS.xml @@ -12,8 +12,10 @@ <property name="classesDir" value="${compileDir}"/> <property name="javadocDir" value="javadoc/uws" /> - <property name="cosJarName" value="cos-1.5beta.jar" /> - <property name="cosJar" value="${libDir}/${cosJarName}" /> + <property name="commonsIOJarName" value="commons-io-2.6.jar" /> + <property name="commonsIOJar" value="${libDir}/${commonsIOJarName}" /> + <property name="commonsFileUploadJarName" value="commons-fileupload-1.3.3.jar" /> + <property name="commonsFileUploadJar" value="${libDir}/${commonsFileUploadJarName}" /> <property name="slf4jApiJarName" value="slf4j-api-1.7.25.jar" /> <property name="slf4jApiJar" value="${libDir}/${slf4jApiJarName}" /> @@ -38,7 +40,8 @@ <!-- CLASSPATHS --> <path id="uws.classpath"> - <pathelement location="${cosJar}" /> + <pathelement location="${commonsIOJar}" /> + <pathelement location="${commonsFileUploadJar}" /> <pathelement location="${slf4jApiJar}" /> <pathelement location="${SERVLET-API}" /> </path> @@ -110,7 +113,8 @@ <target name="buildLib" depends="compileLib,junitValidation" description="After 'clean', build the library JAR (only classes)."> <echo>Generate the library:</echo> <jar basedir="${classesDir}" destfile="${libJarFile}" includes="${includesList}"> - <zipfileset src="${cosJar}" excludes="META-INF/*" /> + <zipfileset src="${commonsIOJar}" excludes="META-INF/**" /> + <zipfileset src="${commonsFileUploadJar}" excludes="META-INF/**" /> <zipfileset dir="${srcDir}" includes="${licensePath}" /> </jar> <delete dir="${compileDir}" failonerror="true" /> @@ -119,7 +123,8 @@ <target name="buildLibAndSrc" depends="buildLib" description="After 'clean' and 'buildLib', build the sources JAR (only .java)."> <jar compress="false" destfile="${srcJarFile}"> <zipfileset dir="${srcDir}" includes="${includesList},${licensePath}" prefix="${srcDir}" /> - <zipfileset dir="${libDir}" includes="${cosJarName}" fullPath="${cosJar}" /> + <zipfileset dir="${libDir}" includes="${commonsIOJarName}" fullPath="${commonsIOJar}" /> + <zipfileset dir="${libDir}" includes="${commonsFileUploadJarName}" fullPath="${commonsFileUploadJar}" /> </jar> </target> diff --git a/lib/commons-fileupload-1.3.3.jar b/lib/commons-fileupload-1.3.3.jar new file mode 100644 index 0000000000000000000000000000000000000000..915d87e744a865e82ea1183cb935f1d2a74d3257 Binary files /dev/null and b/lib/commons-fileupload-1.3.3.jar differ diff --git a/lib/commons-io-2.6.jar b/lib/commons-io-2.6.jar new file mode 100644 index 0000000000000000000000000000000000000000..00556b119d45dd85a3c3073b1826916c3c60b9c4 Binary files /dev/null and b/lib/commons-io-2.6.jar differ diff --git a/lib/cos-1.5beta.jar b/lib/cos-1.5beta.jar deleted file mode 100644 index c30b6238dd5372f873e46acc19273a1d3394edb9..0000000000000000000000000000000000000000 Binary files a/lib/cos-1.5beta.jar and /dev/null differ diff --git a/lib/cos.jar b/lib/cos.jar deleted file mode 100644 index ea39c98967475b5b5e53f65a673ae9cd545e636e..0000000000000000000000000000000000000000 Binary files a/lib/cos.jar and /dev/null differ diff --git a/src/tap/data/LimitedTableIterator.java b/src/tap/data/LimitedTableIterator.java index 9a98d9431e57fb234b44b80a5215d706275a0a53..22353eee2c6c1ce0b79d828283fdcaaffe630231 100644 --- a/src/tap/data/LimitedTableIterator.java +++ b/src/tap/data/LimitedTableIterator.java @@ -2,20 +2,20 @@ package tap.data; /* * This file is part of TAPLibrary. - * + * * ADQLLibrary is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + * * ADQLLibrary is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with ADQLLibrary. If not, see <http://www.gnu.org/licenses/>. - * + * * Copyright 2014 - Astronomisches Rechen Institut (ARI) */ @@ -25,35 +25,34 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.NoSuchElementException; +import adql.db.DBType; import tap.ServiceConnection.LimitUnit; import tap.metadata.TAPColumn; +import tap.upload.ExceededSizeException; import tap.upload.LimitedSizeInputStream; -import adql.db.DBType; - -import com.oreilly.servlet.multipart.ExceededSizeException; /** * <p>Wrap a {@link TableIterator} in order to limit its reading to a fixed number of rows.</p> - * + * * <p> * This wrapper can be "mixed" with a {@link LimitedSizeInputStream}, by wrapping the original input stream by a {@link LimitedSizeInputStream} * and then by wrapping the {@link TableIterator} based on this wrapped input stream by {@link LimitedTableIterator}. * Thus, this wrapper will be able to detect embedded {@link ExceededSizeException} thrown by a {@link LimitedSizeInputStream} through another {@link TableIterator}. * If a such exception is detected, it will declare this wrapper as overflowed as it would be if a rows limit is reached. * </p> - * + * * <p><b>Warning:</b> * To work together with a {@link LimitedSizeInputStream}, this wrapper relies on the hypothesis that any {@link IOException} (including {@link ExceededSizeException}) * will be embedded in a {@link DataReadException} as cause of this exception (using {@link DataReadException#DataReadException(Throwable)} * or {@link DataReadException#DataReadException(String, Throwable)}). If it is not the case, no overflow detection could be done and the exception will just be forwarded. * </p> - * + * * <p> * If a limit - either of rows or of bytes - is reached, a flag "overflow" is set to true. This flag can be got with {@link #isOverflow()}. * Thus, when a {@link DataReadException} is caught, it will be easy to detect whether the error occurred because of an overflow - * or of another problem. + * or of another problem. * </p> - * + * * @author Grégory Mantelet (ARI) * @version 2.0 (01/2015) * @since 2.0 @@ -74,7 +73,7 @@ public class LimitedTableIterator implements TableIterator { /** * Wrap the given {@link TableIterator} so that limiting the number of rows to read. - * + * * @param it The iterator to wrap. <i>MUST NOT be NULL</i> * @param nbMaxRows Maximum number of rows that can be read. There is overflow if more than this number of rows is asked. <i>A negative value means "no limit".</i> */ @@ -87,36 +86,36 @@ public class LimitedTableIterator implements TableIterator { /** * <p>Build the specified {@link TableIterator} instance and wrap it so that limiting the number of rows OR bytes to read.</p> - * + * * <p> * If the limit is on the <b>number of bytes</b>, the given input stream will be first wrapped inside a {@link LimitedSizeInputStream}. * Then, it will be given as only parameter of the constructor of the specified {@link TableIterator} instance. * </p> - * + * * <p>If the limit is on the <b>number of rows</b>, this {@link LimitedTableIterator} will count and limit itself the number of rows.</p> - * + * * <p><i><b>IMPORTANT:</b> The specified class must:</i></p> * <i><ul> * <li>extend {@link TableIterator},</li> * <li>be a concrete class,</li> * <li>have at least one constructor with only one parameter of type {@link InputStream}.</li> * </ul></i> - * + * * <p><i>Note: * If the given limit type is NULL (or different from ROWS and BYTES), or the limit value is <=0, no limit will be set. * All rows and bytes will be read until the end of input is reached. * </i></p> - * + * * @param classIt Class of the {@link TableIterator} implementation to create and whose the output must be limited. * @param input Input stream toward the table to read. * @param type Type of the limit: ROWS or BYTES. <i>MAY be NULL</i> * @param limit Limit in rows or bytes, depending of the "type" parameter. <i>MAY BE <=0</i> - * + * * @throws DataReadException If no instance of the given class can be created, * or if the {@link TableIterator} instance can not be initialized, * or if the limit (in rows or bytes) has been reached. */ - public < T extends TableIterator > LimitedTableIterator(final Class<T> classIt, final InputStream input, final LimitUnit type, final int limit) throws DataReadException{ + public <T extends TableIterator> LimitedTableIterator(final Class<T> classIt, final InputStream input, final LimitUnit type, final int limit) throws DataReadException{ try{ Constructor<T> construct = classIt.getConstructor(InputStream.class); if (LimitUnit.bytes.isCompatibleWith(type) && limit > 0){ @@ -130,7 +129,7 @@ public class LimitedTableIterator implements TableIterator { Throwable t = ite.getCause(); if (t != null && t instanceof DataReadException){ ExceededSizeException exceedEx = getExceededSizeException(t); - // if an error caused by an ExceedSizeException occurs, set this iterator as overflowed and throw the exception: + // if an error caused by an ExceedSizeException occurs, set this iterator as overflowed and throw the exception: if (exceedEx != null) throw new DataReadException(exceedEx.getMessage(), exceedEx); else @@ -144,7 +143,7 @@ public class LimitedTableIterator implements TableIterator { /** * Get the iterator wrapped by this {@link TableIterator} instance. - * + * * @return The wrapped iterator. */ public final TableIterator getWrappedIterator(){ @@ -153,12 +152,12 @@ public class LimitedTableIterator implements TableIterator { /** * <p>Tell whether a limit (in rows or bytes) has been reached.</p> - * + * * <p><i>Note: * If <i>true</i> is returned (that's to say, if a limit has been reached) no more rows or column values * can be read ; an {@link IllegalStateException} would then be thrown. * </i></p> - * + * * @return <i>true</i> if a limit has been reached, <i>false</i> otherwise. */ public final boolean isOverflow(){ @@ -188,7 +187,7 @@ public class LimitedTableIterator implements TableIterator { countRow++; }catch(DataReadException ex){ ExceededSizeException exceedEx = getExceededSizeException(ex); - // if an error caused by an ExceedSizeException occurs, set this iterator as overflowed and throw the exception: + // if an error caused by an ExceedSizeException occurs, set this iterator as overflowed and throw the exception: if (exceedEx != null){ overflow = true; throw new DataReadException(exceedEx.getMessage()); @@ -226,7 +225,7 @@ public class LimitedTableIterator implements TableIterator { /** * Test the overflow flag and throw an {@link IllegalStateException} if <i>true</i>. - * + * * @throws IllegalStateException If this iterator is overflowed (because of either a bytes limit or a rows limit). */ private void testOverflow() throws IllegalStateException{ @@ -236,9 +235,9 @@ public class LimitedTableIterator implements TableIterator { /** * Get the first {@link ExceededSizeException} found in the given {@link Throwable} trace. - * + * * @param ex A {@link Throwable} - * + * * @return The first {@link ExceededSizeException} encountered, or NULL if none has been found. */ private ExceededSizeException getExceededSizeException(Throwable ex){ diff --git a/src/tap/resource/TAP.java b/src/tap/resource/TAP.java index cb8379eeed029ff3546dd86340a0f63c0bcf2f06..a3fe3d8cc59bbf4c95df27f39d3af0d128aee47b 100644 --- a/src/tap/resource/TAP.java +++ b/src/tap/resource/TAP.java @@ -16,7 +16,7 @@ package tap.resource; * You should have received a copy of the GNU Lesser General Public License * along with TAPLibrary. If not, see <http://www.gnu.org/licenses/>. * - * Copyright 2012-2017 - UDS/Centre de Données astronomiques de Strasbourg (CDS), + * Copyright 2012-2018 - UDS/Centre de Données astronomiques de Strasbourg (CDS), * Astronomisches Rechen Institut (ARI) */ @@ -51,12 +51,16 @@ import uws.service.error.ServiceErrorWriter; import uws.service.log.UWSLog.LogLevel; /** - * <p>Root/Home of the TAP service. It is also the resource (HOME) which gathers all the others of the same TAP service.</p> + * Root/Home of the TAP service. It is also the resource (HOME) which gathers + * all the others of the same TAP service. * - * <p>At its creation it is creating and configuring the other resources in function of the given description of the TAP service.</p> + * <p> + * At its creation it is creating and configuring the other resources in + * function of the given description of the TAP service. + * </p> * * @author Grégory Mantelet (CDS;ARI) - * @version 2.1 (09/2017) + * @version 2.3 (08/2018) */ public class TAP implements VOSIResource { @@ -64,29 +68,77 @@ public class TAP implements VOSIResource { * @since 2.1 */ public final static String VERSION = "1.0"; - /** <p>Name of the TAP AVAILABILITY resource. - * This resource tells whether the TAP service is available (i.e. whether it accepts queries or not).</p> - * <p><i>Note: this name is suffixing the root TAP URL in order to access one of its resources.</i></p> + /** + * Name of the TAP AVAILABILITY resource. + * + * <p> + * This resource tells whether the TAP service is available (i.e. whether + * it accepts queries or not). + * </p> + * + * <p><i>Note: + * This name is suffixing the root TAP URL in order to access one of its + * resources. + * </i></p> + * * @since 2.0 */ public final static String RESOURCE_AVAILABILITY = "availability"; - /** <p>Name of the TAP CAPABILITIES resource. - * This resource list all capabilities (e.g. output limits and formats, uploads, ...) of this TAP resource.</p> - * <p><i>Note: this name is suffixing the root TAP URL in order to access one of its resources.</i></p> + + /** + * Name of the TAP CAPABILITIES resource. + * + * <p> + * This resource list all capabilities (e.g. output limits and formats, + * uploads, ...) of this TAP resource. + * </p> + * + * <p><i>Note: + * This name is suffixing the root TAP URL in order to access one of its + * resources. + * </i></p> + * * @since 2.0 */ public final static String RESOURCE_CAPABILITIES = "capabilities"; - /** <p>Name of the TAP HOME PAGE resource. - * This resource lists and describes all published and query-able schemas, tables and columns.</p> - * <p><i>Note: this name is suffixing the root TAP URL in order to access one of its resources.</i></p> + + /** + * Name of the TAP TABLES resource. + * + * <p> + * This resource lists and describes all published and query-able schemas, + * tables and columns. + * </p> + * + * <p><i>Note: + * This name is suffixing the root TAP URL in order to access one of its + * resources. + * </i></p> + * * @since 2.0 */ public final static String RESOURCE_METADATA = "tables"; - /** <p>Name of the TAP HOME PAGE resource. - * This resource is used to submit ADQL queries to run asynchronously.</p> - * <p><i>Note: this name is suffixing the root TAP URL in order to access one of its resources.</i></p> + + /** + * Name of the TAP ASYNC resource. + * + * <p>This resource is used to submit ADQL queries to run asynchronously.</p> + * + * <p><i>Note: + * This name is suffixing the root TAP URL in order to access one of its + * resources. + * </i></p> + * * @since 2.0 */ public final static String RESOURCE_ASYNC = "async"; - /** <p>Name of the TAP HOME PAGE resource. - * This resource is used to submit ADQL queries to run synchronously.</p> - * <p><i>Note: this name is suffixing the root TAP URL in order to access one of its resources.</i></p> + + /** + * Name of the TAP SYNC resource. + * + * <p>This resource is used to submit ADQL queries to run synchronously.</p> + * + * <p><i>Note: + * This name is suffixing the root TAP URL in order to access one of its + * resources. + * </i></p> + * * @since 2.0 */ public final static String RESOURCE_SYNC = "sync"; @@ -94,7 +146,7 @@ public class TAP implements VOSIResource { protected final ServiceConnection service; /** List of all the other TAP resources of the service. */ - protected final Map<String,TAPResource> resources; + protected final Map<String, TAPResource> resources; /** Base URL of the TAP service. It is also the URL of this resource (HOME). */ protected String tapBaseURL = null; @@ -126,17 +178,19 @@ public class TAP implements VOSIResource { protected static String lastRequestID = null; /** - * Build a HOME resource of a TAP service whose the description is given in parameter. - * All the other TAP resources will be created and configured here thanks to the given {@link ServiceConnection}. + * Build a HOME resource of a TAP service whose the description is given in + * parameter. All the other TAP resources will be created and configured + * here thanks to the given {@link ServiceConnection}. * * @param serviceConnection Description of the TAP service. * - * @throws UWSException If an error occurs while creating the /async resource. + * @throws UWSException If an error occurs while creating the /async + * resource. * @throws TAPException If any other error occurs. */ public TAP(final ServiceConnection serviceConnection) throws UWSException, TAPException{ service = serviceConnection; - resources = new HashMap<String,TAPResource>(); + resources = new HashMap<String, TAPResource>(); // Get the error writer to use, or create a default instance if none are provided by the factory: errorWriter = serviceConnection.getFactory().getErrorWriter(); @@ -922,15 +976,6 @@ public class TAP implements VOSIResource { if (request.getAttribute(UWS.REQ_ATTRIBUTE_ID) == null) request.setAttribute(UWS.REQ_ATTRIBUTE_ID, reqID); - // Extract all parameters: - if (request.getAttribute(UWS.REQ_ATTRIBUTE_PARAMETERS) == null){ - try{ - request.setAttribute(UWS.REQ_ATTRIBUTE_PARAMETERS, getUWS().getRequestParser().parse(request)); - }catch(UWSException ue){ - getLogger().log(LogLevel.WARNING, "REQUEST_PARSER", "Can not extract the HTTP request parameters!", ue); - } - } - // Retrieve the resource path parts: String[] resourcePath = (request.getPathInfo() == null) ? null : request.getPathInfo().split("/"); String resourceName = (resourcePath == null || resourcePath.length < 1) ? "" : resourcePath[1].trim(); @@ -960,6 +1005,15 @@ public class TAP implements VOSIResource { // Set the character encoding: response.setCharacterEncoding(UWSToolBox.DEFAULT_CHAR_ENCODING); + // Extract all parameters: + if (request.getAttribute(UWS.REQ_ATTRIBUTE_PARAMETERS) == null){ + try{ + request.setAttribute(UWS.REQ_ATTRIBUTE_PARAMETERS, getUWS().getRequestParser().parse(request)); + }catch(UWSException ue){ + throw new TAPException(ue, ue.getHttpErrorCode()); + } + } + // Display the TAP Home Page: if (resourceName.length() == 0){ resourceName = homePage.getName(); diff --git a/src/tap/upload/ExceededSizeException.java b/src/tap/upload/ExceededSizeException.java new file mode 100644 index 0000000000000000000000000000000000000000..0a92dd10bc9429f19be6ccc3fa9aa1dca147f3de --- /dev/null +++ b/src/tap/upload/ExceededSizeException.java @@ -0,0 +1,41 @@ +package tap.upload; + +/* + * This file is part of TAPLibrary. + * + * TAPLibrary is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * TAPLibrary is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with TAPLibrary. If not, see <http://www.gnu.org/licenses/>. + * + * Copyright 2018 - UDS/Centre de Données astronomiques de Strasbourg (CDS) + */ + +import java.io.IOException; + +/** + * Thrown to indicate an upload exceeded the maximum size. + * + * @author Grégory Mantelet (CDS) + * @version 4.4 (08/2018) + * @since 4.4 + */ +public class ExceededSizeException extends IOException { + private static final long serialVersionUID = 1L; + + public ExceededSizeException(){ + super(); + } + + public ExceededSizeException(final String message){ + super(message); + } +} diff --git a/src/tap/upload/LimitedSizeInputStream.java b/src/tap/upload/LimitedSizeInputStream.java index 60acbe8a2c9ece299946d737f210501d9e13ecc4..22a1fb5e46ce3779da635bf8822ba47a188e5813 100644 --- a/src/tap/upload/LimitedSizeInputStream.java +++ b/src/tap/upload/LimitedSizeInputStream.java @@ -2,20 +2,20 @@ package tap.upload; /* * This file is part of TAPLibrary. - * + * * TAPLibrary is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. - * + * * TAPLibrary is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public License * along with TAPLibrary. If not, see <http://www.gnu.org/licenses/>. - * + * * Copyright 2012,2014 - UDS/Centre de Données astronomiques de Strasbourg (CDS), * Astronomisches Rechen Institut (ARI) */ @@ -24,11 +24,9 @@ import java.io.IOException; import java.io.InputStream; import java.security.InvalidParameterException; -import com.oreilly.servlet.multipart.ExceededSizeException; - /** * Let limit the number of bytes that can be read from a given input stream. - * + * * @author Grégory Mantelet (CDS;ARI) * @version 2.0 (08/2014) */ @@ -48,10 +46,10 @@ public final class LimitedSizeInputStream extends InputStream { /** * Wrap the given input stream so that limiting the number of bytes that can be read. - * + * * @param stream Stream to limit. * @param sizeLimit Maximum number of bytes that can be read. <i>If <=0 an {@link InvalidParameterException} will be thrown.</i> - * + * * @throws NullPointerException If the input stream is missing. */ public LimitedSizeInputStream(final InputStream stream, final long sizeLimit) throws NullPointerException{ @@ -66,7 +64,7 @@ public final class LimitedSizeInputStream extends InputStream { /** * Get the input stream wrapped by this instance of {@link LimitedSizeInputStream}. - * + * * @return The wrapped input stream. * @since 2.0 */ @@ -77,11 +75,11 @@ public final class LimitedSizeInputStream extends InputStream { /** * <p>Update the number of bytes currently read and them check whether the limit has been exceeded. * If the limit has been exceeded, an {@link ExceededSizeException} is thrown.</p> - * + * * <p>Besides, the flag {@link #exceed} is set to true in order to forbid the further reading of bytes.</p> - * + * * @param nbReads Number of bytes read. - * + * * @throws ExceededSizeException If, after update, the limit of bytes has been exceeded. */ private void updateCounter(final long nbReads) throws ExceededSizeException{ @@ -96,11 +94,11 @@ public final class LimitedSizeInputStream extends InputStream { /** * <p>Tell whether the limit has already been exceeded or not.</p> - * + * * <p><i>Note: * If <i>true</i> is returned, no more read will be allowed, and any attempt to read a byte will throw an {@link ExceededSizeException}. * </i></p> - * + * * @return <i>true</i> if the byte limit has been exceeded, <i>false</i> otherwise. */ public final boolean sizeExceeded(){ diff --git a/src/tap/upload/Uploader.java b/src/tap/upload/Uploader.java index aaa84557c97e576548867dbab4eb5d2288cbb42d..d2b5f6138b1e9fa5052c7f1aebbc19ebd2cfbc88 100644 --- a/src/tap/upload/Uploader.java +++ b/src/tap/upload/Uploader.java @@ -24,7 +24,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.HashSet; -import com.oreilly.servlet.multipart.ExceededSizeException; +import tap.upload.ExceededSizeException; import tap.ServiceConnection; import tap.ServiceConnection.LimitUnit; diff --git a/src/uws/service/UWSService.java b/src/uws/service/UWSService.java index 8bf3944b2f6f3cb79086bc259683a0f8a60c4855..519211aac55d9468a0b0de73d41ff999f806984f 100644 --- a/src/uws/service/UWSService.java +++ b/src/uws/service/UWSService.java @@ -1142,15 +1142,6 @@ public class UWSService implements UWS { if (request.getAttribute(UWS.REQ_ATTRIBUTE_ID) == null) request.setAttribute(UWS.REQ_ATTRIBUTE_ID, reqID); - // Extract all parameters: - if (request.getAttribute(UWS.REQ_ATTRIBUTE_PARAMETERS) == null){ - try{ - request.setAttribute(UWS.REQ_ATTRIBUTE_PARAMETERS, requestParser.parse(request)); - }catch(UWSException ue){ - logger.log(LogLevel.ERROR, "REQUEST_PARSER", "Can not extract the HTTP request parameters!", ue); - } - } - // Log the reception of the request: logger.logHttp(LogLevel.INFO, request, reqID, null, null); @@ -1177,6 +1168,10 @@ public class UWSService implements UWS { // Set the character encoding: response.setCharacterEncoding(UWSToolBox.DEFAULT_CHAR_ENCODING); + // Extract all parameters: + if (request.getAttribute(UWS.REQ_ATTRIBUTE_PARAMETERS) == null) + request.setAttribute(UWS.REQ_ATTRIBUTE_PARAMETERS, requestParser.parse(request)); + // Apply the appropriate UWS action: for(int i = 0; action == null && i < uwsActions.size(); i++){ if (uwsActions.get(i).match(urlInterpreter, user, request)){ diff --git a/src/uws/service/UWSServlet.java b/src/uws/service/UWSServlet.java index ebbde879ce2dd59affa044b7e61d62805f640da6..6a461637231845af8fb4274f7e59e4c5042b46bf 100644 --- a/src/uws/service/UWSServlet.java +++ b/src/uws/service/UWSServlet.java @@ -16,7 +16,7 @@ package uws.service; * You should have received a copy of the GNU Lesser General Public License * along with UWSLibrary. If not, see <http://www.gnu.org/licenses/>. * - * Copyright 2012-2017 - UDS/Centre de Données astronomiques de Strasbourg (CDS), + * Copyright 2012-2018 - UDS/Centre de Données astronomiques de Strasbourg (CDS), * Astronomisches Rechen Institut (ARI) */ @@ -153,7 +153,7 @@ import uws.service.wait.BlockingPolicy; * </p> * * @author Grégory Mantelet (CDS;ARI) - * @version 4.3 (11/2017) + * @version 4.4 (08/2018) */ public abstract class UWSServlet extends HttpServlet implements UWS, UWSFactory { private static final long serialVersionUID = 1L; @@ -165,10 +165,10 @@ public abstract class UWSServlet extends HttpServlet implements UWS, UWSFactory protected String description = null; /** List of all managed jobs lists. <i>(it is a LinkedHashMap so that jobs lists are ordered by insertion)</i> */ - private Map<String,JobList> mapJobLists; + private Map<String, JobList> mapJobLists; /** List of available serializers. */ - private Map<String,UWSSerializer> serializers; + private Map<String, UWSSerializer> serializers; /** The MIME type of the default serialization format. */ protected String defaultSerializer = null; @@ -183,7 +183,7 @@ public abstract class UWSServlet extends HttpServlet implements UWS, UWSFactory protected final ArrayList<String> expectedAdditionalParams = new ArrayList<String>(10); /** List the controllers of all the input parameters. See {@link UWSParameters} and {@link InputParamController} for more details. */ - protected final HashMap<String,InputParamController> inputParamControllers = new HashMap<String,InputParamController>(10); + protected final HashMap<String, InputParamController> inputParamControllers = new HashMap<String, InputParamController>(10); /** Lets managing all UWS files (i.e. log, result, backup, ...). */ private UWSFileManager fileManager = null; @@ -247,10 +247,10 @@ public abstract class UWSServlet extends HttpServlet implements UWS, UWSFactory } // Initialize the list of jobs: - mapJobLists = new LinkedHashMap<String,JobList>(); + mapJobLists = new LinkedHashMap<String, JobList>(); // Initialize the list of available serializers: - serializers = new HashMap<String,UWSSerializer>(); + serializers = new HashMap<String, UWSSerializer>(); addSerializer(new XMLSerializer()); addSerializer(new JSONSerializer()); @@ -348,13 +348,6 @@ public abstract class UWSServlet extends HttpServlet implements UWS, UWSFactory final String reqID = generateRequestID(req); req.setAttribute(UWS.REQ_ATTRIBUTE_ID, reqID); - // Extract all parameters: - try{ - req.setAttribute(UWS.REQ_ATTRIBUTE_PARAMETERS, requestParser.parse(req)); - }catch(UWSException ue){ - logger.log(LogLevel.WARNING, "REQUEST_PARSER", "Can not extract the HTTP request parameters!", ue); - } - // Log the reception of the request: logger.logHttp(LogLevel.INFO, req, reqID, null, null); @@ -375,6 +368,9 @@ public abstract class UWSServlet extends HttpServlet implements UWS, UWSFactory // Set the character encoding: resp.setCharacterEncoding(UWSToolBox.DEFAULT_CHAR_ENCODING); + // Extract all parameters: + req.setAttribute(UWS.REQ_ATTRIBUTE_PARAMETERS, requestParser.parse(req)); + // METHOD GET: if (method.equals("GET")){ // HOME PAGE: @@ -843,7 +839,7 @@ public abstract class UWSServlet extends HttpServlet implements UWS, UWSFactory } @Override - public UWSParameters createUWSParameters(final Map<String,Object> params) throws UWSException{ + public UWSParameters createUWSParameters(final Map<String, Object> params) throws UWSException{ return new UWSParameters(params, expectedAdditionalParams, inputParamControllers); } @@ -1029,7 +1025,7 @@ public abstract class UWSServlet extends HttpServlet implements UWS, UWSFactory * Gets the list of all UWS input parameter controllers. * @return All parameter controllers. */ - public final Map<String,InputParamController> getInputParamControllers(){ + public final Map<String, InputParamController> getInputParamControllers(){ return inputParamControllers; } @@ -1037,7 +1033,7 @@ public abstract class UWSServlet extends HttpServlet implements UWS, UWSFactory * Gets an iterator on the list of all UWS input parameter controllers. * @return An iterator on all parameter controllers. */ - public final Iterator<Map.Entry<String,InputParamController>> getInputParamControllersIterator(){ + public final Iterator<Map.Entry<String, InputParamController>> getInputParamControllersIterator(){ return inputParamControllers.entrySet().iterator(); } diff --git a/src/uws/service/request/MultipartParser.java b/src/uws/service/request/MultipartParser.java index 3e71762a354e647ed13d2a78bd9d5b3870c569ac..da2023a306754ba9c5a12d62fb3b1a1e1603d740 100644 --- a/src/uws/service/request/MultipartParser.java +++ b/src/uws/service/request/MultipartParser.java @@ -22,15 +22,21 @@ package uws.service.request; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.Date; -import java.util.Enumeration; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import javax.servlet.http.HttpServletRequest; -import com.oreilly.servlet.MultipartRequest; -import com.oreilly.servlet.multipart.FileRenamePolicy; +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.FileUploadException; +import org.apache.commons.fileupload.disk.DiskFileItemFactory; +import org.apache.commons.fileupload.servlet.ServletFileUpload; +import org.apache.commons.fileupload.util.Streams; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; import uws.UWSException; import uws.service.UWS; @@ -72,11 +78,15 @@ public class MultipartParser implements RequestParser { /** Default maximum allowed size for an HTTP request content: 10 MiB. */ public static final int DEFAULT_SIZE_LIMIT = 10 * 1024 * 1024; - /** <p>Maximum allowed size for an HTTP request content. Over this limit, an exception is thrown and the request is aborted.</p> - * <p><i>Note: - * The default value is {@link #DEFAULT_SIZE_LIMIT} (= {@value #DEFAULT_SIZE_LIMIT} MiB). + /** Maximum allowed size for an HTTP request content. Over this limit, an + * exception is thrown and the request is aborted. + * + * <p><i><b>Note 1:</b> + * The default value is {@link #DEFAULT_SIZE_LIMIT} + * (= {@value #DEFAULT_SIZE_LIMIT} MiB). * </i></p> - * <p><i>Note: + * + * <p><i><b>Note 2:</b> * This limit is expressed in bytes and can not be negative. * Its smallest possible value is 0. If the set value is though negative, * it will be ignored and {@link #DEFAULT_SIZE_LIMIT} will be used instead. @@ -87,14 +97,20 @@ public class MultipartParser implements RequestParser { public final boolean allowUpload; /** File manager to use to create {@link UploadFile} instances. - * It is required by this new object to execute open, move and delete operations whenever it could be asked. */ + * It is required by this new object to execute open, move and delete + * operations whenever it could be asked. */ protected final UWSFileManager fileManager; + /** Tool to parse Multipart HTTP request and fetch files when necessary. + * @since 4.4 */ + protected final ServletFileUpload fileUpload; + /** - * <p>Build a {@link MultipartParser} forbidding uploads (i.e. inline files).</p> + * Build a {@link MultipartParser} forbidding uploads (i.e. inline files). * * <p> - * With this parser, when an upload (i.e. submitted inline files) is detected, an exception is thrown by {@link #parse(HttpServletRequest)} + * With this parser, when an upload (i.e. submitted inline files) is + * detected, an exception is thrown by {@link #parse(HttpServletRequest)} * which cancels immediately the request. * </p> */ @@ -105,23 +121,28 @@ public class MultipartParser implements RequestParser { /** * Build a {@link MultipartParser} allowing uploads (i.e. inline files). * - * @param fileManager The file manager to use in order to store any eventual upload. <b>MUST NOT be NULL</b> + * @param fileManager The file manager to use in order to store any + * eventual upload. <b>MUST NOT be NULL</b> */ public MultipartParser(final UWSFileManager fileManager){ this(true, fileManager); } /** - * <p>Build a {@link MultipartParser}.</p> + * Build a {@link MultipartParser}. * * <p> - * If the first parameter is <i>false</i>, then when an upload (i.e. submitted inline files) is detected, an exception is thrown - * by {@link #parse(HttpServletRequest)} which cancels immediately the request. + * If the first parameter is <i>false</i>, then when an upload + * (i.e. submitted inline files) is detected, an exception is thrown + * by {@link #parse(HttpServletRequest)} which cancels immediately the + * request. * </p> * - * @param uploadEnabled <i>true</i> to allow uploads (i.e. inline files), <i>false</i> otherwise. - * If <i>false</i>, the two other parameters are useless. - * @param fileManager The file manager to use in order to store any eventual upload. <b>MUST NOT be NULL</b> + * @param uploadEnabled <i>true</i> to allow uploads (i.e. inline files), + * <i>false</i> otherwise. If <i>false</i>, the two + * other parameters are useless. + * @param fileManager The file manager to use in order to store any + * eventual upload. <b>MUST NOT be NULL</b> */ protected MultipartParser(final boolean uploadEnabled, final UWSFileManager fileManager){ if (uploadEnabled && fileManager == null) @@ -129,88 +150,99 @@ public class MultipartParser implements RequestParser { this.allowUpload = uploadEnabled; this.fileManager = fileManager; + + // Create a factory for disk-based file items: + DiskFileItemFactory factory = new DiskFileItemFactory(); + + // Configure a repository: + factory.setRepository(fileManager.getTmpDirectory()); + + // Create a new file upload handler + fileUpload = new ServletFileUpload(factory); + fileUpload.setFileSizeMax(SIZE_LIMIT); } @Override - @SuppressWarnings("unchecked") public final Map<String, Object> parse(final HttpServletRequest request) throws UWSException{ LinkedHashMap<String, Object> parameters = new LinkedHashMap<String, Object>(); - MultipartRequest multipart = null; try{ - - // Parse the request body: - multipart = new MultipartRequest(request, fileManager.getTmpDirectory().getPath(), (SIZE_LIMIT < 0 ? DEFAULT_SIZE_LIMIT : SIZE_LIMIT), new FileRenamePolicy() { - @Override - public File rename(File file){ - Object reqID = request.getAttribute(UWS.REQ_ATTRIBUTE_ID); - if (reqID == null || !(reqID instanceof String)) - reqID = (new Date()).getTime(); - char uniq = 'A'; - File f = new File(file.getParentFile(), "UPLOAD_" + reqID + uniq + "_" + file.getName()); - while(f.exists()){ - uniq++; - f = new File(file.getParentFile(), "UPLOAD_" + reqID + "_" + file.getName()); + List<FileItem> fileItems = fileUpload.parseRequest(request); + for(FileItem item : fileItems){ + String name = item.getFieldName(); + InputStream stream = item.getInputStream(); + if (item.isFormField()) + consumeParameter(name, Streams.asString(stream), parameters); + else{ + if (!allowUpload) + throw new UWSException(UWSException.BAD_REQUEST, "Uploads are not allowed by this service!"); + else{ + // keep the file: + File file = getFileFromParam(request, fileManager.getTmpDirectory().getPath(), FilenameUtils.getName(item.getName())); + FileUtils.copyInputStreamToFile(stream, file); + // build its description/pointer: + UploadFile lob = new UploadFile(name, FilenameUtils.getName(item.getName()), file.toURI().toString(), fileManager); + lob.mimeType = item.getContentType(); + lob.length = file.length(); + // add it inside the parameters map: + consumeParameter(name, lob, parameters); } - return f; } - }); - - // Extract all "normal" parameters: - String param; - Enumeration<String> e = multipart.getParameterNames(); - while(e.hasMoreElements()){ - param = e.nextElement(); - for(String occurence : multipart.getParameterValues(param)) - consumeParameter(param, occurence, parameters); + // finally delete the file item stored by FileUpload: + item.delete(); } - - // Extract all inline files as additional parameters: - e = multipart.getFileNames(); - if (!allowUpload && e.hasMoreElements()) - throw new UWSException(UWSException.BAD_REQUEST, "Uploads are not allowed by this service!"); - while(e.hasMoreElements()){ - param = e.nextElement(); - if (multipart.getFile(param) == null) - continue; - - /* - * TODO !!!POSSIBLE ISSUE!!! - * MultipartRequest is not able to deal with multiple files having the same parameter name. However, all files are created/uploaded - * but only the last one is accessible through this object....so only the last can be deleted, which could be a problem later - * (hence the usage of the system temporary directory). - */ - - // build its description/pointer: - UploadFile lob = new UploadFile(param, multipart.getOriginalFileName(param), multipart.getFile(param).toURI().toString(), fileManager); - lob.mimeType = multipart.getContentType(param); - lob.length = multipart.getFile(param).length(); - // add it inside the parameters map: - consumeParameter(param, lob, parameters); - } - + }catch(FileUploadException fue){ + throw new UWSException(UWSException.BAD_REQUEST, fue, "Incorrect HTTP request: " + fue.getMessage() + "."); }catch(IOException ioe){ - throw new UWSException(UWSException.INTERNAL_SERVER_ERROR, ioe, "Internal Error => Impossible to extract parameters from the Multipart HTTP request!"); + throw new UWSException(UWSException.BAD_REQUEST, ioe, "Incorrect HTTP request: " + ioe.getMessage() + "."); }catch(IllegalArgumentException iae){ String confError = iae.getMessage(); if (fileManager.getTmpDirectory() == null) confError = "Missing upload directory!"; - throw new UWSException(UWSException.INTERNAL_SERVER_ERROR, iae, "Internal Error: Incorrect UPLOAD configuration: " + confError); + throw new UWSException(UWSException.INTERNAL_SERVER_ERROR, iae, "Internal Error! Incorrect UPLOAD configuration: " + confError); } return parameters; } /** - * <p>Consume the specified parameter: add it inside the given map.</p> + * Return the path of a non-existing file inside the given directory and + * whose the name is built using the given file name and the HTTP request + * ID. + * + * @param request The received HTTP request. + * @param parentFile The directory in which the file should be created. + * @param inputFileName The file name provided by the user. + * + * @return Path of a non-existing file for the specified input file. + * + * @since 4.4 + */ + protected File getFileFromParam(final HttpServletRequest request, final String parentFile, final String inputFileName){ + Object reqID = request.getAttribute(UWS.REQ_ATTRIBUTE_ID); + if (reqID == null || !(reqID instanceof String)) + reqID = (new Date()).getTime(); + char uniq = 'A'; + File f = new File(parentFile, "UPLOAD_" + reqID + uniq + "_" + inputFileName); + while(f.exists()){ + uniq++; + f = new File(parentFile, "UPLOAD_" + reqID + "_" + inputFileName); + } + return f; + } + + /** + * Consume the specified parameter: add it inside the given map. * * <p> - * By default, this function is just putting the given value inside the map. So, if the parameter already exists in the map, - * its old value will be overwritten by the given one. + * By default, this function is just putting the given value inside the + * map. So, if the parameter already exists in the map, its old value will + * be overwritten by the given one. * </p> * - * <p><i>Note: - * If the old value was a file, it will be deleted from the file system before its replacement in the map. + * <p><i><b>Note:</b> + * If the old value was a file, it will be deleted from the file system + * before its replacement in the map. * </i></p> * * @param name Name of the parameter to consume. @@ -245,14 +277,7 @@ public class MultipartParser implements RequestParser { * <code>false</code> otherwise. */ public static final boolean isMultipartContent(final HttpServletRequest request){ - // Extract the content type and determine if it is a multipart request (its content type should start by multipart/form-data"): - String contentType = request.getContentType(); - if (contentType == null) - return false; - else if (contentType.toLowerCase().startsWith(EXPECTED_CONTENT_TYPE)) - return true; - else - return false; + return ServletFileUpload.isMultipartContent(request); } }