Changes for file retrieval using dynamic location

Dockerfile

 FROM maven:3.6.3-openjdk-14
 
-ADD src /src
 ADD pom.xml .
+RUN mvn dependency:go-offline
+
+ADD src /src
 
 RUN mvn clean package -DskipTests -DfinalName=vospace-file-service
 

docker-env

-path_prefix=/tmp/fsdemo
+upload_location_id=3
 
 file-catalog.datasource.jdbc-url=jdbc:postgresql://172.19.0.1:5432/vospace_testdb
 file-catalog.datasource.username=postgres

src/main/java/it/inaf/ia2/transfer/controller/FileInfo.java

@@ -4,7 +4,8 @@ import java.util.List;
 
 public class FileInfo {
 
-    private String osRelPath;
+    private int nodeId;
+    private String osPath;
     private String virtualPath;
     private boolean isPublic;
     private List<String> groupRead;
@@ -14,12 +15,20 @@ public class FileInfo {
     private List<String> acceptViews;
     private List<String> provideViews;
 
-    public String getOsRelPath() {
-        return osRelPath;
+    public int getNodeId() {
+        return nodeId;
     }
 
-    public void setOsRelPath(String osRelPath) {
-        this.osRelPath = osRelPath;
+    public void setNodeId(int nodeId) {
+        this.nodeId = nodeId;
+    }
+
+    public String getOsPath() {
+        return osPath;
+    }
+
+    public void setOsPath(String osPath) {
+        this.osPath = osPath;
     }
 
     public String getVirtualPath() {

src/main/java/it/inaf/ia2/transfer/controller/GetFileController.java

@@ -15,7 +15,6 @@ import javax.servlet.http.HttpServletResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import static org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR;
 import static org.springframework.http.HttpStatus.NOT_FOUND;
 import static org.springframework.http.HttpStatus.UNAUTHORIZED;
@@ -28,9 +27,6 @@ public class GetFileController {
 
     private static final Logger LOG = LoggerFactory.getLogger(GetFileController.class);
 
-    @Value("${path_prefix}")
-    String pathPrefix;
-
     @Autowired
     private FileDAO fileDAO;
 
@@ -90,13 +86,7 @@ public class GetFileController {
 
     private ResponseEntity<?> getFileResponse(FileInfo fileInfo) {
 
-        String path = pathPrefix + fileInfo.getOsRelPath();
-
-        if (fileInfo.isAsyncTrans()) {
-            // TODO: add /retrieve part
-        }
-
-        File file = new File(path);
+        File file = new File(fileInfo.getOsPath());
 
         if (!file.exists()) {
             LOG.error("File not found: " + file.getAbsolutePath());

src/main/java/it/inaf/ia2/transfer/controller/PutFileController.java

@@ -8,13 +8,11 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import javax.servlet.http.HttpServletRequest;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import static org.springframework.http.HttpStatus.NOT_FOUND;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.PutMapping;
@@ -25,9 +23,6 @@ import org.springframework.web.multipart.MultipartFile;
 @RestController
 public class PutFileController {
 
-    @Value("${path_prefix}")
-    private String pathPrefix;
-
     @Autowired
     private FileDAO fileDAO;
 
@@ -79,9 +74,8 @@ public class PutFileController {
     }
 
     private void storeGenericFile(FileInfo fileInfo, InputStream is) throws IOException {
-        Path path = Path.of(pathPrefix, fileInfo.getOsRelPath());
 
-        File file = path.toFile();
+        File file = new File(fileInfo.getOsPath());
 
         /**
          * This block must be synchronized, to avoid concurrency issues when

src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java

 package it.inaf.ia2.transfer.persistence;
 
 import it.inaf.ia2.transfer.controller.FileInfo;
+import java.nio.file.Path;
 import java.sql.Array;
 import java.sql.PreparedStatement;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -10,12 +12,16 @@ import java.util.List;
 import java.util.Optional;
 import javax.sql.DataSource;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Repository;
 
 @Repository
 public class FileDAO {
 
+    @Value("${upload_location_id}")
+    private int uploadLocationId;
+
     private final JdbcTemplate jdbcTemplate;
 
     @Autowired
@@ -25,19 +31,25 @@ public class FileDAO {
 
     public Optional<FileInfo> getFileInfo(String virtualPath) {
 
-        String sql = "select os_path, is_public, group_read, group_write, owner_id, async_trans,\n"
-                + "accept_views, provide_views\n"
-                + "from node n join node_path p on n.node_id = p.node_id\n"
-                + "and vos_path = ?";
+        String sql = "SELECT n.node_id, is_public, group_read, group_write, owner_id, async_trans,\n"
+                + "accept_views, provide_views, l.location_type,\n"
+                + "(SELECT user_name FROM users WHERE rap_id = owner_id) AS username,\n"
+                + "base_path, os_path\n"
+                + "FROM node_path p\n"
+                + "JOIN node n ON p.node_id = n.node_id\n"
+                + "JOIN location l ON (n.location_id IS NOT NULL AND n.location_id = l.location_id) OR (n.location_id IS NULL AND l.location_id = ?)\n"
+                + "LEFT JOIN storage s ON s.storage_id = l.storage_dest_id\n"
+                + "WHERE p.vos_path = ?";
 
         FileInfo fileInfo = jdbcTemplate.query(conn -> {
             PreparedStatement ps = conn.prepareStatement(sql);
-            ps.setString(1, virtualPath);
+            ps.setInt(1, uploadLocationId);
+            ps.setString(2, virtualPath);
             return ps;
         }, rs -> {
             if (rs.next()) {
                 FileInfo fi = new FileInfo();
-                fi.setOsRelPath(rs.getString("os_path"));
+                fi.setNodeId(rs.getInt("node_id"));
                 fi.setIsPublic(rs.getBoolean("is_public"));
                 fi.setGroupRead(toList(rs.getArray("group_read")));
                 fi.setGroupWrite(toList(rs.getArray("group_write")));
@@ -46,6 +58,9 @@ public class FileDAO {
                 fi.setAcceptViews(toList(rs.getArray("accept_views")));
                 fi.setProvideViews(toList(rs.getArray("provide_views")));
                 fi.setVirtualPath(virtualPath);
+
+                fillOsPath(fi, rs);
+
                 return fi;
             }
             return null;
@@ -54,6 +69,27 @@ public class FileDAO {
         return Optional.ofNullable(fileInfo);
     }
 
+    private void fillOsPath(FileInfo fi, ResultSet rs) throws SQLException {
+
+        String basePath = rs.getString("base_path");
+        String osPath = rs.getString("os_path");
+        if (osPath.startsWith("/")) {
+            osPath = osPath.substring(1);
+        }
+
+        Path completeOsPath = Path.of(basePath);
+
+        boolean asyncLocation = "async".equals(rs.getString("location_type"));
+
+        if (asyncLocation) {
+            String username = rs.getString("username");
+            completeOsPath = completeOsPath.resolve(username).resolve("retrieve");
+        }
+
+        completeOsPath = completeOsPath.resolve(osPath);
+        fi.setOsPath(completeOsPath.toString());
+    }
+
     private List<String> toList(Array array) throws SQLException {
         if (array == null) {
             return new ArrayList<>();

src/main/resources/application.properties

@@ -8,6 +8,9 @@ file-catalog.datasource.password=
 gms_base_url=http://localhost:8082/gms
 jwks_uri=http://localhost/rap-ia2/auth/oidc/jwks
 
-path_prefix=/tmp
+upload_location_id=3
 
 cors.allowed.origin=http://localhost:8080,http://localhost:8085
+
+spring.servlet.multipart.max-file-size=10GB
+spring.servlet.multipart.max-request-size=10GB

src/test/java/it/inaf/ia2/transfer/controller/GetFileControllerTest.java

@@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.web.servlet.MockMvc;
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
 import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print;
@@ -28,7 +27,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
 
 @SpringBootTest
 @AutoConfigureMockMvc
-@TestPropertySource(properties = {"path_prefix=/"})
 public class GetFileControllerTest {
 
     @MockBean
@@ -59,7 +57,7 @@ public class GetFileControllerTest {
     public void getPublicFile() throws Exception {
 
         FileInfo fileInfo = new FileInfo();
-        fileInfo.setOsRelPath(tempFile.getAbsolutePath());
+        fileInfo.setOsPath(tempFile.getAbsolutePath());
         fileInfo.setIsPublic(true);
 
         when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo));
@@ -80,7 +78,7 @@ public class GetFileControllerTest {
     public void testFileNotFoundOnDisk() throws Exception {
 
         FileInfo fileInfo = new FileInfo();
-        fileInfo.setOsRelPath("/this/doesnt/exists");
+        fileInfo.setOsPath("/this/doesnt/exists");
         fileInfo.setIsPublic(true);
 
         when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo));
@@ -125,7 +123,7 @@ public class GetFileControllerTest {
         when(gmsClient.isMemberOf(any(), any())).thenReturn(true);
 
         FileInfo fileInfo = new FileInfo();
-        fileInfo.setOsRelPath(tempFile.getAbsolutePath());
+        fileInfo.setOsPath(tempFile.getAbsolutePath());
         fileInfo.setGroupRead(Collections.singletonList("group1"));
 
         when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo));
@@ -140,7 +138,7 @@ public class GetFileControllerTest {
         when(tokenParser.getClaims(any())).thenReturn(claims);
 
         FileInfo fileInfo = new FileInfo();
-        fileInfo.setOsRelPath(tempFile.getAbsolutePath());
+        fileInfo.setOsPath(tempFile.getAbsolutePath());
         fileInfo.setOwnerId("123");
 
         when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo));

src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java

@@ -23,7 +23,6 @@ import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.mock.web.MockHttpServletRequest;
 import org.springframework.mock.web.MockMultipartFile;
-import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.web.servlet.MockMvc;
 import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder;
 import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
@@ -33,7 +32,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
 
 @SpringBootTest
 @AutoConfigureMockMvc
-@TestPropertySource(properties = {"path_prefix=/tmp"})
 public class PutFileControllerTest {
 
     @MockBean
@@ -86,7 +84,7 @@ public class PutFileControllerTest {
 
     private FileInfo createBaseFileInfo(String fileName) {
         FileInfo fileInfo = new FileInfo();
-        fileInfo.setOsRelPath(fileName);
+        fileInfo.setOsPath("/tmp/" + fileName);
         fileInfo.setIsPublic(false);
 
         when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo));

src/test/java/it/inaf/ia2/transfer/persistence/FileDAOTest.java

+package it.inaf.ia2.transfer.persistence;
+
+import it.inaf.ia2.transfer.controller.FileInfo;
+import java.util.Optional;
+import javax.sql.DataSource;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+@ExtendWith(SpringExtension.class)
+@ContextConfiguration(classes = {DataSourceConfig.class})
+@TestPropertySource(locations = "classpath:test.properties")
+public class FileDAOTest {
+
+    @Autowired
+    private DataSource dataSource;
+
+    private FileDAO dao;
+
+    @BeforeEach
+    public void init() {
+        dao = new FileDAO(dataSource);
+    }
+
+    @Test
+    public void testGetFileInfo() {
+
+        Optional<FileInfo> optFileInfo = dao.getFileInfo("/test1/file1.txt");
+
+        assertTrue(optFileInfo.isPresent());
+
+        FileInfo fileInfo = optFileInfo.get();
+
+        assertEquals("/home/username1/retrieve/file1.txt", fileInfo.getOsPath());
+    }
+}

src/test/resources/test-data.sql

 DELETE FROM node;
 ALTER SEQUENCE node_node_id_seq RESTART WITH 1;
 
+DELETE FROM users;
+INSERT INTO users (rap_id, user_name, e_mail) VALUES ('user1', 'username1', 'ia2@inaf.it');
+
 INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id) VALUES (NULL, NULL, '', 'container', '0', '0');
 
 INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write) VALUES ('', NULL, 'test1', 'container', 'user1', 'user1', '{"group1","group2"}','{"group2"}');      -- /test1
 INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write) VALUES ('2', NULL, '.tmp-123.txt', 'structured', 'user1', 'user1', '{"group1","group2"}','{"group2"}');      -- /test1/.tmp-123.txt
-INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write) VALUES ('2', NULL, 'file1.txt', 'data', 'user1', 'user1', '{"group1","group2"}','{"group2"}');      -- /test1/file1.txt
-INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write) VALUES ('2', NULL, 'file2.txt', 'data', 'user1', 'user1', '{"group1","group2"}','{"group2"}');      -- /test1/file2.txt
+INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write, location_id) VALUES ('2', '', 'file1.txt', 'data', 'user1', 'user1', '{"group1","group2"}','{"group2"}', 1);      -- /test1/file1.txt
+INSERT INTO node (parent_path, parent_relative_path, name, type, owner_id, creator_id, group_read, group_write, location_id) VALUES ('2', '', 'file2.txt', 'data', 'user1', 'user1', '{"group1","group2"}','{"group2"}', 1);      -- /test1/file2.txt

src/test/resources/test.properties

 # File catalog repository directory (filled by pom.xml, overridable passing environment variable)
 init_database_scripts_path=@init_database_scripts_path@
+
+upload_location_id=3
\ No newline at end of file