From b52fc744d0a050d81e412c0605bff60b5080f9ab Mon Sep 17 00:00:00 2001 From: Sonia Zorba Date: Wed, 20 Jan 2021 17:41:56 +0100 Subject: [PATCH] Implemented list of files upload --- .../ia2/transfer/controller/FileInfo.java | 27 +++++ .../controller/PutFileController.java | 34 +++++- .../ia2/transfer/persistence/FileDAO.java | 8 +- .../transfer/persistence/ListOfFilesDAO.java | 91 +++++++++++++++ .../controller/PutFileControllerTest.java | 108 ++++++++++++++++++ .../persistence/ListOfFilesDAOTest.java | 33 ++++++ 6 files changed, 298 insertions(+), 3 deletions(-) create mode 100644 src/main/java/it/inaf/ia2/transfer/persistence/ListOfFilesDAO.java create mode 100644 src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java create mode 100644 src/test/java/it/inaf/ia2/transfer/persistence/ListOfFilesDAOTest.java diff --git a/src/main/java/it/inaf/ia2/transfer/controller/FileInfo.java b/src/main/java/it/inaf/ia2/transfer/controller/FileInfo.java index 2f339af..bd1ffd1 100644 --- a/src/main/java/it/inaf/ia2/transfer/controller/FileInfo.java +++ b/src/main/java/it/inaf/ia2/transfer/controller/FileInfo.java @@ -5,11 +5,14 @@ import java.util.List; public class FileInfo { private String osRelPath; + private String virtualPath; private boolean isPublic; private List groupRead; private List groupWrite; private String ownerId; private boolean asyncTrans; + private List acceptViews; + private List provideViews; public String getOsRelPath() { return osRelPath; @@ -19,6 +22,14 @@ public class FileInfo { this.osRelPath = osRelPath; } + public String getVirtualPath() { + return virtualPath; + } + + public void setVirtualPath(String virtualPath) { + this.virtualPath = virtualPath; + } + public boolean isIsPublic() { return isPublic; } @@ -58,4 +69,20 @@ public class FileInfo { public void setAsyncTrans(boolean asyncTrans) { this.asyncTrans = asyncTrans; } + + public List getAcceptViews() { + return acceptViews; + } + + public void setAcceptViews(List acceptViews) { + this.acceptViews = acceptViews; + } + + public List getProvideViews() { + return provideViews; + } + + public void setProvideViews(List provideViews) { + this.provideViews = provideViews; + } } diff --git a/src/main/java/it/inaf/ia2/transfer/controller/PutFileController.java b/src/main/java/it/inaf/ia2/transfer/controller/PutFileController.java index 3666f85..440f8a3 100644 --- a/src/main/java/it/inaf/ia2/transfer/controller/PutFileController.java +++ b/src/main/java/it/inaf/ia2/transfer/controller/PutFileController.java @@ -1,11 +1,16 @@ package it.inaf.ia2.transfer.controller; import it.inaf.ia2.transfer.persistence.FileDAO; +import it.inaf.ia2.transfer.persistence.ListOfFilesDAO; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import javax.servlet.http.HttpServletRequest; import org.springframework.beans.factory.annotation.Autowired; @@ -26,12 +31,15 @@ public class PutFileController { @Autowired private FileDAO fileDAO; + @Autowired + private ListOfFilesDAO listOfFilesDAO; + @Autowired private HttpServletRequest request; @PutMapping("/**") public ResponseEntity putFile(@RequestParam("file") MultipartFile file) throws IOException { - + String path = request.getServletPath(); Optional optFileInfo = fileDAO.getFileInfo(path); @@ -47,6 +55,30 @@ public class PutFileController { private void storeFile(FileInfo fileInfo, InputStream is) throws IOException { + if (fileInfo.getAcceptViews() != null && fileInfo.getAcceptViews().contains("urn:list-of-files")) { + storeListOfFiles(fileInfo, is); + } else { + storeGenericFile(fileInfo, is); + } + } + + private void storeListOfFiles(FileInfo fileInfo, InputStream is) throws IOException { + List filePaths = parseListOfFiles(is); + listOfFilesDAO.createList(fileInfo.getVirtualPath(), filePaths); + } + + private List parseListOfFiles(InputStream is) throws IOException { + List filePaths = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new InputStreamReader(is))) { + String line; + while ((line = br.readLine()) != null && !line.isBlank()) { + filePaths.add(line.trim()); + } + } + return filePaths; + } + + private void storeGenericFile(FileInfo fileInfo, InputStream is) throws IOException { Path path = Path.of(pathPrefix, fileInfo.getOsRelPath()); File file = path.toFile(); diff --git a/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java b/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java index 9e6928b..70c9163 100644 --- a/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java +++ b/src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java @@ -25,8 +25,9 @@ public class FileDAO { public Optional getFileInfo(String virtualPath) { - String sql = "select os_path, is_public, group_read, group_write, owner_id, async_trans from\n" - + "node n join node_path p on n.node_id = p.node_id\n" + String sql = "select os_path, is_public, group_read, group_write, owner_id, async_trans,\n" + + "accept_views, provide_views\n" + + "from node n join node_path p on n.node_id = p.node_id\n" + "and vos_path = ?"; FileInfo fileInfo = jdbcTemplate.query(conn -> { @@ -42,6 +43,9 @@ public class FileDAO { fi.setGroupWrite(toList(rs.getArray("group_write"))); fi.setOwnerId(rs.getString("owner_id")); fi.setAsyncTrans(rs.getBoolean("async_trans")); + fi.setAcceptViews(toList(rs.getArray("accept_views"))); + fi.setProvideViews(toList(rs.getArray("provide_views"))); + fi.setVirtualPath(virtualPath); return fi; } return null; diff --git a/src/main/java/it/inaf/ia2/transfer/persistence/ListOfFilesDAO.java b/src/main/java/it/inaf/ia2/transfer/persistence/ListOfFilesDAO.java new file mode 100644 index 0000000..763a61e --- /dev/null +++ b/src/main/java/it/inaf/ia2/transfer/persistence/ListOfFilesDAO.java @@ -0,0 +1,91 @@ +package it.inaf.ia2.transfer.persistence; + +import java.sql.PreparedStatement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import javax.sql.DataSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +@Repository +public class ListOfFilesDAO { + + private final JdbcTemplate jdbcTemplate; + + @Autowired + public ListOfFilesDAO(DataSource fileCatalogDatasource) { + this.jdbcTemplate = new JdbcTemplate(fileCatalogDatasource); + } + + public void createList(String vosListNodePath, List vosPaths) { + + List idPaths = getAllNodeIdPaths(vosListNodePath, vosPaths); + + if (idPaths.size() != (vosPaths.size() + 1)) { + throw new IllegalStateException("Unable to retrieve some file identifiers from paths"); + } + + Integer listNodeId = null; + + List nodeIds = new ArrayList<>(); + for (NodeIdPath idPath : idPaths) { + if (vosListNodePath.equals(idPath.nodePath)) { + listNodeId = idPath.nodeId; + } else { + nodeIds.add(idPath.nodeId); + } + } + + if (listNodeId == null) { + throw new IllegalStateException("List node id not found for path " + vosListNodePath); + } + + createList(listNodeId, nodeIds); + } + + private List getAllNodeIdPaths(String vosListNodePath, List vosPaths) { + List allPaths = new ArrayList<>(vosPaths); + allPaths.add(vosListNodePath); + + String sql = "SELECT node_id, vos_path FROM node_vos_path WHERE vos_path IN (" + + String.join(",", Collections.nCopies(allPaths.size(), "?")) + ")"; + + return jdbcTemplate.query(conn -> { + PreparedStatement ps = conn.prepareStatement(sql); + int i = 0; + for (String path : allPaths) { + ps.setString(++i, path); + } + return ps; + }, (row, index) -> { + NodeIdPath nodeIdPath = new NodeIdPath(); + nodeIdPath.nodeId = row.getInt("node_id"); + nodeIdPath.nodePath = row.getString("vos_path"); + return nodeIdPath; + }); + } + + private void createList(int listNodeId, List nodes) { + + String sql = "INSERT INTO list_of_files (list_node_id, node_id) VALUES " + + String.join(",", Collections.nCopies(nodes.size(), "(?, ?)")); + + jdbcTemplate.update(conn -> { + PreparedStatement ps = conn.prepareStatement(sql); + int i = 0; + for (int nodeId : nodes) { + ps.setInt(++i, listNodeId); + ps.setInt(++i, nodeId); + } + return ps; + }); + } + + static class NodeIdPath { + + int nodeId; + String nodePath; + } +} diff --git a/src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java b/src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java new file mode 100644 index 0000000..6c8733c --- /dev/null +++ b/src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java @@ -0,0 +1,108 @@ +package it.inaf.ia2.transfer.controller; + +import it.inaf.ia2.transfer.persistence.FileDAO; +import it.inaf.ia2.transfer.persistence.ListOfFilesDAO; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Optional; +import java.util.UUID; +import org.assertj.core.util.Files; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.Test; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.request.RequestPostProcessor; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest +@AutoConfigureMockMvc +@TestPropertySource(properties = {"path_prefix=/tmp"}) +public class PutFileControllerTest { + + @MockBean + private FileDAO fileDao; + + @MockBean + private ListOfFilesDAO listOfFilesDAO; + + @Autowired + private MockMvc mockMvc; + + @Test + public void putGenericFile() throws Exception { + + String randomFileName = UUID.randomUUID().toString(); + createBaseFileInfo(randomFileName); + + MockMultipartFile fakeFile = new MockMultipartFile("file", "test.txt", "text/plain", "content".getBytes()); + + mockMvc.perform(putMultipart("/path/to/test.txt") + .file(fakeFile)) + .andDo(print()) + .andExpect(status().isOk()); + + File file = Path.of("/tmp", randomFileName).toFile(); + + assertTrue(file.exists()); + assertEquals("content", Files.contentOf(file, StandardCharsets.UTF_8)); + assertTrue(file.delete()); + } + + @Test + public void putListOfFiles() throws Exception { + + String randomFileName = UUID.randomUUID().toString(); + FileInfo fileInfo = createBaseFileInfo(randomFileName); + fileInfo.setAcceptViews(Collections.singletonList("urn:list-of-files")); + + String content = "/path/to/file1\n/path/to/file2"; + + MockMultipartFile fakeFile = new MockMultipartFile("file", "test.txt", "text/plain", content.getBytes()); + + mockMvc.perform(putMultipart("/path/to/test.txt") + .file(fakeFile)) + .andDo(print()) + .andExpect(status().isOk()); + + verify(listOfFilesDAO, times(1)).createList(any(), argThat(list -> list.size() == 2)); + } + + private FileInfo createBaseFileInfo(String fileName) { + FileInfo fileInfo = new FileInfo(); + fileInfo.setOsRelPath(fileName); + fileInfo.setIsPublic(false); + + when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo)); + + return fileInfo; + } + + private MockMultipartHttpServletRequestBuilder putMultipart(String uri) { + MockMultipartHttpServletRequestBuilder builder = MockMvcRequestBuilders.multipart(uri); + builder.with(new RequestPostProcessor() { + @Override + public MockHttpServletRequest postProcessRequest(MockHttpServletRequest request) { + request.setMethod("PUT"); + return request; + } + }); + return builder; + } +} diff --git a/src/test/java/it/inaf/ia2/transfer/persistence/ListOfFilesDAOTest.java b/src/test/java/it/inaf/ia2/transfer/persistence/ListOfFilesDAOTest.java new file mode 100644 index 0000000..da8473f --- /dev/null +++ b/src/test/java/it/inaf/ia2/transfer/persistence/ListOfFilesDAOTest.java @@ -0,0 +1,33 @@ +package it.inaf.ia2.transfer.persistence; + +import java.util.Arrays; +import java.util.List; +import javax.sql.DataSource; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {DataSourceConfig.class}) +@TestPropertySource(locations = "classpath:test.properties") +public class ListOfFilesDAOTest { + + @Autowired + private DataSource dataSource; + private ListOfFilesDAO listOfFilesDao; + + @BeforeEach + public void init() { + listOfFilesDao = new ListOfFilesDAO(dataSource); + } + + @Test + public void testCreateListOfFiles() { + List vosPaths = Arrays.asList("/test1/file1.txt", "/test1/file2.txt"); + listOfFilesDao.createList("/test1/.tmp-123.txt", vosPaths); + } +} -- GitLab