diff --git a/Dockerfile b/Dockerfile index 824a8cb08abae1d1783b195371a422dc48b6e8c9..a69bb4fd1f41ab1efae03a2b316eeff2e44aa00f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,9 @@ FROM maven:3.6.3-openjdk-14 -ADD src /src ADD pom.xml . +RUN mvn dependency:go-offline + +ADD src /src RUN mvn clean package -DskipTests -DfinalName=vospace-file-service diff --git a/docker-env b/docker-env index 0562a6668e63697a4b158a221c0c0e8e3ca1adce..148fff4d36ac7767b14a53b808f59c47556a55f4 100644 --- a/docker-env +++ b/docker-env @@ -1,4 +1,4 @@ -path_prefix=/tmp/fsdemo +upload_location_id=3 file-catalog.datasource.jdbc-url=jdbc:postgresql://172.19.0.1:5432/vospace_testdb file-catalog.datasource.username=postgres diff --git a/src/main/java/it/inaf/ia2/transfer/controller/FileInfo.java b/src/main/java/it/inaf/ia2/transfer/controller/FileInfo.java index bd1ffd14c09d87287729b78d902f70e79640bc1f..bee3c6485c4cb59d41c0555492499095d287e984 100644 --- a/src/main/java/it/inaf/ia2/transfer/controller/FileInfo.java +++ b/src/main/java/it/inaf/ia2/transfer/controller/FileInfo.java @@ -4,7 +4,8 @@ import java.util.List; public class FileInfo { - private String osRelPath; + private int nodeId; + private String osPath; private String virtualPath; private boolean isPublic; private List<String> groupRead; @@ -14,12 +15,20 @@ public class FileInfo { private List<String> acceptViews; private List<String> provideViews; - public String getOsRelPath() { - return osRelPath; + public int getNodeId() { + return nodeId; } - public void setOsRelPath(String osRelPath) { - this.osRelPath = osRelPath; + public void setNodeId(int nodeId) { + this.nodeId = nodeId; + } + + public String getOsPath() { + return osPath; + } + + public void setOsPath(String osPath) { + this.osPath = osPath; } public String getVirtualPath() { diff --git a/src/main/java/it/inaf/ia2/transfer/controller/GetFileController.java b/src/main/java/it/inaf/ia2/transfer/controller/GetFileController.java index 50a5c1dee528204e7136e27d4707eb3ed0861879..4a24771868eb2407dbf48384cd370fb6c91b192e 100644 --- a/src/main/java/it/inaf/ia2/transfer/controller/GetFileController.java +++ b/src/main/java/it/inaf/ia2/transfer/controller/GetFileController.java @@ -15,7 +15,6 @@ import javax.servlet.http.HttpServletResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR; import static org.springframework.http.HttpStatus.NOT_FOUND; import static org.springframework.http.HttpStatus.UNAUTHORIZED; @@ -28,9 +27,6 @@ public class GetFileController { private static final Logger LOG = LoggerFactory.getLogger(GetFileController.class); - @Value("${path_prefix}") - String pathPrefix; - @Autowired private FileDAO fileDAO; @@ -90,13 +86,7 @@ public class GetFileController { private ResponseEntity<?> getFileResponse(FileInfo fileInfo) { - String path = pathPrefix + fileInfo.getOsRelPath(); - - if (fileInfo.isAsyncTrans()) { - // TODO: add /retrieve part - } - - File file = new File(path); + File file = new File(fileInfo.getOsPath()); if (!file.exists()) { LOG.error("File not found: " + file.getAbsolutePath()); diff --git a/src/main/java/it/inaf/ia2/transfer/controller/PutFileController.java b/src/main/java/it/inaf/ia2/transfer/controller/PutFileController.java index 0314f7b9a08d7ad2caef13b26ff11db6d1d420de..08a984f9ad8f55b8d6e207dea14eb97e06d784cf 100644 --- a/src/main/java/it/inaf/ia2/transfer/controller/PutFileController.java +++ b/src/main/java/it/inaf/ia2/transfer/controller/PutFileController.java @@ -8,13 +8,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Optional; import javax.servlet.http.HttpServletRequest; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import static org.springframework.http.HttpStatus.NOT_FOUND; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PutMapping; @@ -25,9 +23,6 @@ import org.springframework.web.multipart.MultipartFile; @RestController public class PutFileController { - @Value("${path_prefix}") - private String pathPrefix; - @Autowired private FileDAO fileDAO; @@ -79,9 +74,8 @@ public class PutFileController { } private void storeGenericFile(FileInfo fileInfo, InputStream is) throws IOException { - Path path = Path.of(pathPrefix, fileInfo.getOsRelPath()); - File file = path.toFile(); + File file = new File(fileInfo.getOsPath()); /** * This block must be synchronized, to avoid concurrency issues when diff --git a/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java b/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java index 70c91631efaa1be1e8cd3312a8ac52ab631555cc..762f0dde4e6ae61e91325aea3a53895c2991725a 100644 --- a/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java +++ b/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java @@ -1,8 +1,10 @@ package it.inaf.ia2.transfer.persistence; import it.inaf.ia2.transfer.controller.FileInfo; +import java.nio.file.Path; import java.sql.Array; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; @@ -10,12 +12,16 @@ import java.util.List; import java.util.Optional; import javax.sql.DataSource; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; @Repository public class FileDAO { + @Value("${upload_location_id}") + private int uploadLocationId; + private final JdbcTemplate jdbcTemplate; @Autowired @@ -25,19 +31,25 @@ public class FileDAO { public Optional<FileInfo> getFileInfo(String virtualPath) { - String sql = "select os_path, is_public, group_read, group_write, owner_id, async_trans,\n" - + "accept_views, provide_views\n" - + "from node n join node_path p on n.node_id = p.node_id\n" - + "and vos_path = ?"; + String sql = "SELECT n.node_id, is_public, group_read, group_write, owner_id, async_trans,\n" + + "accept_views, provide_views, l.location_type,\n" + + "(SELECT user_name FROM users WHERE rap_id = owner_id) AS username,\n" + + "base_path, os_path\n" + + "FROM node_path p\n" + + "JOIN node n ON p.node_id = n.node_id\n" + + "JOIN location l ON (n.location_id IS NOT NULL AND n.location_id = l.location_id) OR (n.location_id IS NULL AND l.location_id = ?)\n" + + "LEFT JOIN storage s ON s.storage_id = l.storage_dest_id\n" + + "WHERE p.vos_path = ?"; FileInfo fileInfo = jdbcTemplate.query(conn -> { PreparedStatement ps = conn.prepareStatement(sql); - ps.setString(1, virtualPath); + ps.setInt(1, uploadLocationId); + ps.setString(2, virtualPath); return ps; }, rs -> { if (rs.next()) { FileInfo fi = new FileInfo(); - fi.setOsRelPath(rs.getString("os_path")); + fi.setNodeId(rs.getInt("node_id")); fi.setIsPublic(rs.getBoolean("is_public")); fi.setGroupRead(toList(rs.getArray("group_read"))); fi.setGroupWrite(toList(rs.getArray("group_write"))); @@ -46,6 +58,9 @@ public class FileDAO { fi.setAcceptViews(toList(rs.getArray("accept_views"))); fi.setProvideViews(toList(rs.getArray("provide_views"))); fi.setVirtualPath(virtualPath); + + fillOsPath(fi, rs); + return fi; } return null; @@ -54,6 +69,27 @@ public class FileDAO { return Optional.ofNullable(fileInfo); } + private void fillOsPath(FileInfo fi, ResultSet rs) throws SQLException { + + String basePath = rs.getString("base_path"); + String osPath = rs.getString("os_path"); + if (osPath.startsWith("/")) { + osPath = osPath.substring(1); + } + + Path completeOsPath = Path.of(basePath); + + boolean asyncLocation = "async".equals(rs.getString("location_type")); + + if (asyncLocation) { + String username = rs.getString("username"); + completeOsPath = completeOsPath.resolve(username).resolve("retrieve"); + } + + completeOsPath = completeOsPath.resolve(osPath); + fi.setOsPath(completeOsPath.toString()); + } + private List<String> toList(Array array) throws SQLException { if (array == null) { return new ArrayList<>(); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 10ecdb19b6b61333db8ae819efd5472522ab6961..9e5a988d2a0fed70a6d5906b684d2506f8daa9ad 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -8,6 +8,9 @@ file-catalog.datasource.password= gms_base_url=http://localhost:8082/gms jwks_uri=http://localhost/rap-ia2/auth/oidc/jwks -path_prefix=/tmp +upload_location_id=3 -cors.allowed.origin=http://localhost:8080,http://localhost:8085 \ No newline at end of file +cors.allowed.origin=http://localhost:8080,http://localhost:8085 + +spring.servlet.multipart.max-file-size=10GB +spring.servlet.multipart.max-request-size=10GB diff --git a/src/test/java/it/inaf/ia2/transfer/controller/GetFileControllerTest.java b/src/test/java/it/inaf/ia2/transfer/controller/GetFileControllerTest.java index 9eddbfa6fcbdb8ec07ad60a70bdf0144273ed671..37fbc8ff27cab539703982e5c929d86b99e7d8c4 100644 --- a/src/test/java/it/inaf/ia2/transfer/controller/GetFileControllerTest.java +++ b/src/test/java/it/inaf/ia2/transfer/controller/GetFileControllerTest.java @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.MockMvc; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; @@ -28,7 +27,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @SpringBootTest @AutoConfigureMockMvc -@TestPropertySource(properties = {"path_prefix=/"}) public class GetFileControllerTest { @MockBean @@ -59,7 +57,7 @@ public class GetFileControllerTest { public void getPublicFile() throws Exception { FileInfo fileInfo = new FileInfo(); - fileInfo.setOsRelPath(tempFile.getAbsolutePath()); + fileInfo.setOsPath(tempFile.getAbsolutePath()); fileInfo.setIsPublic(true); when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo)); @@ -80,7 +78,7 @@ public class GetFileControllerTest { public void testFileNotFoundOnDisk() throws Exception { FileInfo fileInfo = new FileInfo(); - fileInfo.setOsRelPath("/this/doesnt/exists"); + fileInfo.setOsPath("/this/doesnt/exists"); fileInfo.setIsPublic(true); when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo)); @@ -125,7 +123,7 @@ public class GetFileControllerTest { when(gmsClient.isMemberOf(any(), any())).thenReturn(true); FileInfo fileInfo = new FileInfo(); - fileInfo.setOsRelPath(tempFile.getAbsolutePath()); + fileInfo.setOsPath(tempFile.getAbsolutePath()); fileInfo.setGroupRead(Collections.singletonList("group1")); when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo)); @@ -140,7 +138,7 @@ public class GetFileControllerTest { when(tokenParser.getClaims(any())).thenReturn(claims); FileInfo fileInfo = new FileInfo(); - fileInfo.setOsRelPath(tempFile.getAbsolutePath()); + fileInfo.setOsPath(tempFile.getAbsolutePath()); fileInfo.setOwnerId("123"); when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo)); diff --git a/src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java b/src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java index 6c8733cb090c0ca4d9c3270ae724aaca73fc43e4..cf9b8f4e08cb139af37991812e9669fd5ee4b829 100644 --- a/src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java +++ b/src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java @@ -23,7 +23,6 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockMultipartFile; -import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; @@ -33,7 +32,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @SpringBootTest @AutoConfigureMockMvc -@TestPropertySource(properties = {"path_prefix=/tmp"}) public class PutFileControllerTest { @MockBean @@ -86,7 +84,7 @@ public class PutFileControllerTest { private FileInfo createBaseFileInfo(String fileName) { FileInfo fileInfo = new FileInfo(); - fileInfo.setOsRelPath(fileName); + fileInfo.setOsPath("/tmp/" + fileName); fileInfo.setIsPublic(false); when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo)); diff --git a/src/test/java/it/inaf/ia2/transfer/persistence/FileDAOTest.java b/src/test/java/it/inaf/ia2/transfer/persistence/FileDAOTest.java new file mode 100644 index 0000000000000000000000000000000000000000..6fe17662475c192ae7f893d74883070fbd1fae79 --- /dev/null +++ b/src/test/java/it/inaf/ia2/transfer/persistence/FileDAOTest.java @@ -0,0 +1,42 @@ +package it.inaf.ia2.transfer.persistence; + +import it.inaf.ia2.transfer.controller.FileInfo; +import java.util.Optional; +import javax.sql.DataSource; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {DataSourceConfig.class}) +@TestPropertySource(locations = "classpath:test.properties") +public class FileDAOTest { + + @Autowired + private DataSource dataSource; + + private FileDAO dao; + + @BeforeEach + public void init() { + dao = new FileDAO(dataSource); + } + + @Test + public void testGetFileInfo() { + + Optional<FileInfo> optFileInfo = dao.getFileInfo("/test1/file1.txt"); + + assertTrue(optFileInfo.isPresent()); + + FileInfo fileInfo = optFileInfo.get(); + + assertEquals("/home/username1/retrieve/file1.txt", fileInfo.getOsPath()); + } +} diff --git a/src/test/resources/test-data.sql b/src/test/resources/test-data.sql index 3dff11c62abdb00724ab989b040e9db16e6ed12d..cf0fb583998325437c2c19139d5b22cc88602f50 100644 --- a/src/test/resources/test-data.sql +++ b/src/test/resources/test-data.sql @@ -1,9 +1,12 @@ DELETE FROM node; ALTER SEQUENCE node_node_id_seq RESTART WITH 1; +DELETE FROM users; +INSERT INTO users (rap_id, user_name, e_mail) VALUES ('user1', 'username1', 'ia2@inaf.it'); + INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id) VALUES (NULL, NULL, '', 'container', '0', '0'); INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write) VALUES ('', NULL, 'test1', 'container', 'user1', 'user1', '{"group1","group2"}','{"group2"}'); -- /test1 INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write) VALUES ('2', NULL, '.tmp-123.txt', 'structured', 'user1', 'user1', '{"group1","group2"}','{"group2"}'); -- /test1/.tmp-123.txt -INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write) VALUES ('2', NULL, 'file1.txt', 'data', 'user1', 'user1', '{"group1","group2"}','{"group2"}'); -- /test1/file1.txt -INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write) VALUES ('2', NULL, 'file2.txt', 'data', 'user1', 'user1', '{"group1","group2"}','{"group2"}'); -- /test1/file2.txt +INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write, location_id) VALUES ('2', '', 'file1.txt', 'data', 'user1', 'user1', '{"group1","group2"}','{"group2"}', 1); -- /test1/file1.txt +INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write, location_id) VALUES ('2', '', 'file2.txt', 'data', 'user1', 'user1', '{"group1","group2"}','{"group2"}', 1); -- /test1/file2.txt diff --git a/src/test/resources/test.properties b/src/test/resources/test.properties index 9dd67fabc9af378d8b30a62f05d2c74cff22988d..985146c3197d2ec6bb7a1bff5fcde4bbc871413e 100644 --- a/src/test/resources/test.properties +++ b/src/test/resources/test.properties @@ -1,2 +1,4 @@ # File catalog repository directory (filled by pom.xml, overridable passing environment variable) init_database_scripts_path=@init_database_scripts_path@ + +upload_location_id=3 \ No newline at end of file