From 9ed6821061c275bee2126132589b799bf5d338e4 Mon Sep 17 00:00:00 2001 From: Sonia Zorba <sonia.zorba@inaf.it> Date: Wed, 9 Dec 2020 17:03:13 +0100 Subject: [PATCH] Implemented basic node listing feature --- .gitignore | 3 +- nb-configuration.xml | 18 --- pom.xml | 64 +++++----- .../inaf/oats/vospace/ListNodeController.java | 47 +++---- .../oats/vospace/persistence/NodeDAO.java | 91 ++++++++++---- src/main/resources/application.properties | 8 +- .../oats/vospace/ListNodeControllerTest.java | 57 +++++++++ .../vospace/persistence/DataSourceConfig.java | 117 ++++++++++++++++++ .../oats/vospace/persistence/NodeDAOTest.java | 33 +++++ src/test/resources/test.properties | 1 + 10 files changed, 334 insertions(+), 105 deletions(-) delete mode 100644 nb-configuration.xml create mode 100644 src/test/java/it/inaf/oats/vospace/ListNodeControllerTest.java create mode 100644 src/test/java/it/inaf/oats/vospace/persistence/DataSourceConfig.java create mode 100644 src/test/java/it/inaf/oats/vospace/persistence/NodeDAOTest.java create mode 100644 src/test/resources/test.properties diff --git a/.gitignore b/.gitignore index 97d3695..6cd12df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ target/** nbactions.xml - +nb-configuration.xml +/target/ diff --git a/nb-configuration.xml b/nb-configuration.xml deleted file mode 100644 index 71095dc..0000000 --- a/nb-configuration.xml +++ /dev/null @@ -1,18 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project-shared-configuration> - <!-- -This file contains additional configuration written by modules in the NetBeans IDE. -The configuration is intended to be shared among all the users of project and -therefore it is assumed to be part of version control checkout. -Without this configuration present, some functionality in the IDE may be limited or fail altogether. ---> - <properties xmlns="http://www.netbeans.org/ns/maven-properties-data/1"> - <!-- -Properties that influence various parts of the IDE, especially code formatting and the like. -You can copy and paste the single properties, into the pom.xml file and the IDE will pick them up. -That way multiple projects can share the same settings (useful for formatting rules for example). -Any value defined here will override the pom.xml file value but is only applicable to the current project. ---> - <netbeans.hint.jdkPlatform>JDK_15</netbeans.hint.jdkPlatform> - </properties> -</project-shared-configuration> diff --git a/pom.xml b/pom.xml index 4970220..93583c8 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ <artifactId>vospace-oats</artifactId> <version>0.0.1-SNAPSHOT</version> <name>vospace-oats</name> - <description>Demo project for Spring Boot</description> + <description>VOSpace REST service</description> <properties> <java.version>11</java.version> @@ -36,28 +36,12 @@ <artifactId>spring-boot-starter-web</artifactId> </dependency> - <!-- JAXB dependency --> - <dependency> - <groupId>javax.xml.bind</groupId> - <artifactId>jaxb-api</artifactId> - </dependency> - - <dependency> - <groupId>org.glassfish.jaxb</groupId> - <artifactId>jaxb-runtime</artifactId> - </dependency> - <!-- Jackson-JAXB compatibility --> <dependency> <groupId>com.fasterxml.jackson.module</groupId> <artifactId>jackson-module-jaxb-annotations</artifactId> </dependency> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-actuator</artifactId> - </dependency> - <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-devtools</artifactId> @@ -81,36 +65,46 @@ <groupId>it.oats.inaf</groupId> <artifactId>vospace-datamodel</artifactId> <version>1.0-SNAPSHOT</version> + <exclusions> + <!-- Transitive dependency excluded to avoid duplicated dependency issues. + We want to use always the version provided by Spring Boot --> + <exclusion> + <groupId>com.fasterxml.jackson.module</groupId> + <artifactId>jackson-module-jaxb-annotations</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>it.inaf.ia2</groupId> <artifactId>auth-lib</artifactId> <version>2.0.0-SNAPSHOT</version> - </dependency> - + </dependency> + + <!-- Embedded PostgreSQL: --> <dependency> - <groupId>it.inaf.ia2</groupId> - <artifactId>rap-client</artifactId> - <version>1.0-SNAPSHOT</version> - </dependency> - + <groupId>com.opentable.components</groupId> + <artifactId>otj-pg-embedded</artifactId> + <version>0.13.3</version> + <scope>test</scope> + </dependency> <dependency> - <groupId>it.inaf.ia2</groupId> - <artifactId>gms-client</artifactId> - <version>1.0-SNAPSHOT</version> + <groupId>io.zonky.test.postgres</groupId> + <artifactId>embedded-postgres-binaries-linux-amd64</artifactId> + <version>12.5.0</version> + <scope>test</scope> </dependency> - + </dependencies> - <repositories> - <repository> - <id>ia2-snapshots</id> - <name>your custom repo</name> - <url>http://repo.ia2.inaf.it/maven/repository/snapshots</url> - </repository> - </repositories> + <repositories> + <repository> + <id>ia2-snapshots</id> + <name>your custom repo</name> + <url>http://repo.ia2.inaf.it/maven/repository/snapshots</url> + </repository> + </repositories> <build> <plugins> diff --git a/src/main/java/it/inaf/oats/vospace/ListNodeController.java b/src/main/java/it/inaf/oats/vospace/ListNodeController.java index 1d6a906..ed04f65 100644 --- a/src/main/java/it/inaf/oats/vospace/ListNodeController.java +++ b/src/main/java/it/inaf/oats/vospace/ListNodeController.java @@ -1,38 +1,43 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - package it.inaf.oats.vospace; -import java.util.List; - import org.springframework.web.bind.annotation.RestController; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; import net.ivoa.xml.vospace.v2.Node; import it.inaf.oats.vospace.persistence.NodeDAO; +import javax.servlet.http.HttpServletRequest; +import org.springframework.http.MediaType; - @RestController public class ListNodeController { - + @Autowired private NodeDAO nodeDAO; - - @GetMapping(value="/{nodeName}") - public ResponseEntity<List<Node>>listNodes(@PathVariable("nodeName")String node_name) { - - // dal nome del nodo devo ricavarmi l'ivo_id - String node_ivo_id = ""; - return ResponseEntity.ok(nodeDAO.listNode(node_ivo_id)); - + + @GetMapping(value = {"/nodes", "/nodes/**"}, + produces = {MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE, MediaType.TEXT_XML_VALUE}) + public ResponseEntity<Node> listNode(HttpServletRequest request) { + String path = getPath(request); + return ResponseEntity.ok(nodeDAO.listNode(path)); + } + + /** + * Slash is a special character in defining REST endpoints and trying to + * define a PathVariable containing slashes doesn't work, so the endpoint + * has been defined using "/nodes/**" instead of "/nodes/{path}" and the + * path is extracted manually parsing the request URL. + */ + private String getPath(HttpServletRequest request) { + String requestURL = request.getRequestURL().toString(); + String[] split = requestURL.split("/nodes/"); + + String path = "/"; + if (split.length == 2) { + path += split[1]; + } + return path; } - - } diff --git a/src/main/java/it/inaf/oats/vospace/persistence/NodeDAO.java b/src/main/java/it/inaf/oats/vospace/persistence/NodeDAO.java index 38e6da8..9dc64e7 100644 --- a/src/main/java/it/inaf/oats/vospace/persistence/NodeDAO.java +++ b/src/main/java/it/inaf/oats/vospace/persistence/NodeDAO.java @@ -1,29 +1,17 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ - package it.inaf.oats.vospace.persistence; import net.ivoa.xml.vospace.v2.Node; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Types; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; import javax.sql.DataSource; +import net.ivoa.xml.vospace.v2.ContainerNode; +import net.ivoa.xml.vospace.v2.DataNode; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.stereotype.Component; import org.springframework.stereotype.Repository; -import org.springframework.http.ResponseEntity; /** * @@ -32,6 +20,8 @@ import org.springframework.http.ResponseEntity; @Repository public class NodeDAO { + @Value("${vospace-authority}") + private String authority; private final JdbcTemplate jdbcTemplate; @@ -41,8 +31,7 @@ public class NodeDAO { } public Node createNode(Node myNode) { - - + StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO "); sb.append("NodeProperty"); @@ -50,26 +39,74 @@ public class NodeDAO { sb.append(" WHERE NOT EXISTS (SELECT * FROM NodeProperty"); sb.append(" WHERE nodeID=? and propertyURI=?)"); String sqlQuery = sb.toString(); - + return myNode; } - - - public List<Node> listNode(String nodeIvoId) { + public Node listNode(String path) { - String sql = "SELECT * FROM Node WHERE ivo_id=?"; + String sql = "SELECT os.os_path, n.node_id, type, async_trans, owner_id, group_read, group_write, is_public, content_length, created_on, last_modified from node n\n" + + "JOIN node_os_path os ON n.node_id = os.node_id\n" + + "WHERE n.path ~ (" + getFirstLevelChildrenSelector(path) + ")::lquery\n" + + "OR os.os_path = ? ORDER BY os_path"; - return jdbcTemplate.query(conn -> { + List<Node> parentAndChildren = jdbcTemplate.query(conn -> { PreparedStatement ps = conn.prepareStatement(sql); - ps.setString(1, nodeIvoId); + ps.setString(1, path); + ps.setString(2, path); return ps; }, (row, index) -> { - Node newNode = new Node(); - newNode.setUri(row.getString("ivo_id")); - return newNode; + return getNodeFromResultSet(row); }); + // Query returns parent as first node + Node node = parentAndChildren.get(0); + + // Fill children + if (node instanceof ContainerNode && parentAndChildren.size() > 1) { + ContainerNode parent = (ContainerNode) node; + for (int i = 1; i < parentAndChildren.size(); i++) { + parent.getNodes().add(parentAndChildren.get(i)); + } + } + + return node; + } + + private String getFirstLevelChildrenSelector(String path) { + String select = "(SELECT path FROM node WHERE node_id = (SELECT node_id FROM node_os_path WHERE os_path = ?))::varchar || '"; + + if (!"/".equals(path)) { + select += "."; + } + select += "*{1}'"; + return select; } + private Node getNodeFromResultSet(ResultSet rs) throws SQLException { + + Node node = getTypedNode(rs.getString("type")); + node.setUri(getUri(rs.getString("os_path"))); + + return node; + } + + private Node getTypedNode(String type) { + Node node; + switch (type) { + case "container": + node = new ContainerNode(); + break; + case "data": + node = new DataNode(); + break; + default: + throw new UnsupportedOperationException("Node type " + type + " not supported yet"); + } + return node; + } + + private String getUri(String path) { + return "vos://" + authority + path; + } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 5754698..8b26974 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -19,9 +19,9 @@ server.servlet.context-path=/vospace # For development only: spring.profiles.active=dev -spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/vospacedb -spring.datasource.username=vospaceusr -spring.datasource.password=Peper0ne +spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/vospace_testdb +spring.datasource.username=postgres +spring.datasource.password=postgres # enable debug logging # this is equivalent to passing '--debug' as command line argument @@ -32,3 +32,5 @@ logging.level.org.springframework.web=TRACE #logging.level.org.springframework.boot=DEBUG # log to file (absolute/relative path of log file) #logging.file=path/to/log/file.log + +vospace-authority=example.com!vospace diff --git a/src/test/java/it/inaf/oats/vospace/ListNodeControllerTest.java b/src/test/java/it/inaf/oats/vospace/ListNodeControllerTest.java new file mode 100644 index 0000000..0b8b5f3 --- /dev/null +++ b/src/test/java/it/inaf/oats/vospace/ListNodeControllerTest.java @@ -0,0 +1,57 @@ +package it.inaf.oats.vospace; + +import it.inaf.oats.vospace.persistence.NodeDAO; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import static org.mockito.ArgumentMatchers.eq; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +@ExtendWith(MockitoExtension.class) +public class ListNodeControllerTest { + + @Mock + private NodeDAO dao; + + @InjectMocks + private ListNodeController controller; + + private MockMvc mockMvc; + + @BeforeEach + public void init() { + mockMvc = MockMvcBuilders.standaloneSetup(controller).build(); + } + + @Test + public void testRootXml() throws Exception { + + mockMvc.perform(get("/nodes") + .accept(MediaType.APPLICATION_XML)) + .andDo(print()) + .andExpect(status().isOk()); + + verify(dao, times(1)).listNode(eq("/")); + } + + @Test + public void testNodeXml() throws Exception { + + mockMvc.perform(get("/nodes/mynode") + .accept(MediaType.APPLICATION_XML)) + .andDo(print()) + .andExpect(status().isOk()); + + verify(dao, times(1)).listNode(eq("/mynode")); + } +} diff --git a/src/test/java/it/inaf/oats/vospace/persistence/DataSourceConfig.java b/src/test/java/it/inaf/oats/vospace/persistence/DataSourceConfig.java new file mode 100644 index 0000000..e933955 --- /dev/null +++ b/src/test/java/it/inaf/oats/vospace/persistence/DataSourceConfig.java @@ -0,0 +1,117 @@ +package it.inaf.oats.vospace.persistence; + +import com.opentable.db.postgres.embedded.EmbeddedPostgres; +import com.opentable.db.postgres.embedded.PgBinaryResolver; +import com.opentable.db.postgres.embedded.UncompressBundleDirectoryResolver; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.sql.DataSource; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.Scope; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.ClassPathResource; +import org.springframework.jdbc.datasource.init.ScriptUtils; + +/** + * Generates a DataSource that can be used for testing DAO classes. It loads an + * embedded Postgres database and fills it using the data from + * vospace-transfer-service repository (folder must exists; it location can be + * configured using the init_database_scripts_path in test.properties). + */ +@TestConfiguration +public class DataSourceConfig { + + @Value("${init_database_scripts_path}") + private String scriptPath; + + /** + * Using the prototype scope we are generating a different database in each + * test. + */ + @Bean + @Scope("prototype") + @Primary + public DataSource dataSource() throws Exception { + DataSource embeddedPostgresDS = EmbeddedPostgres.builder() + .setPgDirectoryResolver(new UncompressBundleDirectoryResolver(new CustomPostgresBinaryResolver())) + .start().getPostgresDatabase(); + + initDatabase(embeddedPostgresDS); + + return embeddedPostgresDS; + } + + private class CustomPostgresBinaryResolver implements PgBinaryResolver { + + /** + * Loads specific embedded Postgres version. + */ + @Override + public InputStream getPgBinary(String system, String architecture) throws IOException { + ClassPathResource resource = new ClassPathResource(String.format("postgres-%s-%s.txz", system.toLowerCase(), architecture)); + return resource.getInputStream(); + } + } + + /** + * Loads SQL scripts for database initialization from + * vospace-transfer-service repo directory. + */ + private void initDatabase(DataSource dataSource) throws Exception { + try ( Connection conn = dataSource.getConnection()) { + + File currentDir = new File(DataSourceConfig.class.getClassLoader().getResource(".").getFile()); + Path scriptDir = currentDir.toPath().resolve(scriptPath); + + List<String> scripts = Arrays.asList("00-init.sql", "01-pgsql_path.sql", "03-indexes.sql", "05-data.sql", "06-os_path_view.sql"); + + for (String script : scripts) { + ByteArrayResource scriptResource = replaceDollarQuoting(scriptDir.resolve(script)); + ScriptUtils.executeSqlScript(conn, scriptResource); + } + } + } + + /** + * It seems that dollar quoting (used in UDF) is broken in JDBC. Replacing + * it with single quotes solves the problem. We replace the quoting here + * instead of inside the original files because dollar quoting provides a + * better visibility. + */ + private ByteArrayResource replaceDollarQuoting(Path sqlScriptPath) throws Exception { + + String scriptContent = Files.readString(sqlScriptPath); + + if (scriptContent.contains("$func$")) { + + String func = extractFunctionDefinition(scriptContent); + + String originalFunction = "$func$" + func + "$func$"; + String newFunction = "'" + func.replaceAll("'", "''") + "'"; + + scriptContent = scriptContent.replace(originalFunction, newFunction); + } + + return new ByteArrayResource(scriptContent.getBytes()); + } + + private String extractFunctionDefinition(String scriptContent) { + Pattern pattern = Pattern.compile("\\$func\\$(.*?)\\$func\\$", Pattern.DOTALL); + Matcher matcher = pattern.matcher(scriptContent); + if (matcher.find()) { + return matcher.group(1); + } + throw new IllegalArgumentException(scriptContent + " doesn't contain $func$"); + } +} diff --git a/src/test/java/it/inaf/oats/vospace/persistence/NodeDAOTest.java b/src/test/java/it/inaf/oats/vospace/persistence/NodeDAOTest.java new file mode 100644 index 0000000..cdc1820 --- /dev/null +++ b/src/test/java/it/inaf/oats/vospace/persistence/NodeDAOTest.java @@ -0,0 +1,33 @@ +package it.inaf.oats.vospace.persistence; + +import javax.sql.DataSource; +import net.ivoa.xml.vospace.v2.ContainerNode; +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = {DataSourceConfig.class}) +@TestPropertySource(locations = "classpath:test.properties") +public class NodeDAOTest { + + @Autowired + private DataSource dataSource; + private NodeDAO dao; + + @BeforeEach + public void init() { + dao = new NodeDAO(dataSource); + } + + @Test + public void testListNode() { + ContainerNode root = (ContainerNode) dao.listNode("/"); + assertEquals(1, root.getNodes().size()); + } +} diff --git a/src/test/resources/test.properties b/src/test/resources/test.properties new file mode 100644 index 0000000..c633d73 --- /dev/null +++ b/src/test/resources/test.properties @@ -0,0 +1 @@ +init_database_scripts_path=../../../vospace-transfer-service/file_catalog -- GitLab