From a5974e7fcb5a8a42aba83e88b0b1a971bd0153be Mon Sep 17 00:00:00 2001
From: Sonia Zorba <sonia.zorba@inaf.it>
Date: Mon, 14 Jun 2021 16:25:27 +0200
Subject: [PATCH] Used multiple targets for transfers instead of creating
temporary 'list-of-files' node
---
.../vospace/ui/controller/JobController.java | 109 +-----------------
.../ui/controller/NodesController.java | 5 +-
.../ui/controller/UploadController.java | 2 +-
.../ui/service/MainNodesHtmlGenerator.java | 5 -
.../inaf/ia2/vospace/ui/service/NodeInfo.java | 21 ----
.../vospace/ui/client/VOSpaceClientTest.java | 7 +-
.../ui/controller/JobControllerTest.java | 2 +-
7 files changed, 12 insertions(+), 139 deletions(-)
diff --git a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/JobController.java b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/JobController.java
index 4afb13a..3c560e5 100644
--- a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/JobController.java
+++ b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/JobController.java
@@ -8,34 +8,23 @@ package it.inaf.ia2.vospace.ui.controller;
import it.inaf.ia2.vospace.ui.client.VOSpaceClient;
import it.inaf.ia2.vospace.ui.data.Job;
import it.inaf.ia2.vospace.ui.exception.BadRequestException;
-import java.util.ArrayList;
import java.util.List;
-import java.util.UUID;
+import java.util.stream.Collectors;
import net.ivoa.xml.uws.v1.ErrorType;
import net.ivoa.xml.uws.v1.ExecutionPhase;
import net.ivoa.xml.uws.v1.JobSummary;
-import net.ivoa.xml.vospace.v2.Param;
import net.ivoa.xml.vospace.v2.Protocol;
-import net.ivoa.xml.vospace.v2.StructuredDataNode;
import net.ivoa.xml.vospace.v2.Transfer;
-import net.ivoa.xml.vospace.v2.View;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
-import org.springframework.core.io.ByteArrayResource;
-import org.springframework.http.HttpEntity;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
-import org.springframework.util.LinkedMultiValueMap;
-import org.springframework.util.MultiValueMap;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
-import org.springframework.web.client.RestTemplate;
@RestController
public class JobController extends BaseController {
@@ -48,9 +37,6 @@ public class JobController extends BaseController {
@Autowired
private VOSpaceClient client;
- @Autowired
- private RestTemplate restTemplate;
-
@PostMapping(value = "/recall", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<Job> startRecallFromTapeJob(@RequestBody List<String> paths) {
@@ -58,23 +44,13 @@ public class JobController extends BaseController {
throw new BadRequestException("Received empty list of nodes");
}
- String target;
- if (paths.size() == 1) {
- target = "vos://" + authority + paths.get(0);
- } else {
- target = createTempListOfFilesNode(paths);
- }
+ List<String> targets = paths.stream().map(p -> "vos://" + authority + p).collect(Collectors.toList());
Transfer transfer = new Transfer();
transfer.setDirection("pullToVoSpace");
- transfer.setTarget(target);
+ transfer.setTarget(targets);
Protocol protocol = new Protocol();
- Param param = new Param();
- param.setUri("ia2:node-type");
- param.setValue(paths.size() == 1 ? "single" : "list");
- protocol.getParam().add(param);
-
protocol.setUri("ia2:async-recall");
transfer.getProtocols().add(protocol);
@@ -110,85 +86,6 @@ public class JobController extends BaseController {
throw new RuntimeException(errorMessage);
}
- private String createTempListOfFilesNode(List<String> paths) {
-
- StructuredDataNode dataNode = createStructuredDataNode(paths);
- client.createNode(dataNode);
- String uploadEndpoint = getTempFileEndpoint(dataNode.getUri());
-
- String content = String.join("\n", paths);
-
- upload(uploadEndpoint, content);
-
- return dataNode.getUri();
- }
-
- private StructuredDataNode createStructuredDataNode(List<String> paths) {
-
- List<View> views = new ArrayList<>();
- View view = new View();
- view.setUri("urn:list-of-files");
- views.add(view);
-
- StructuredDataNode dataNode = new StructuredDataNode();
-
- String parentPath = getParentPath(paths);
- String newTempFile = ".tmp-" + UUID.randomUUID().toString().replace("-", "") + ".txt";
-
- dataNode.setUri("vos://" + authority + parentPath + "/" + newTempFile);
-
- dataNode.setAccepts(views);
- dataNode.setProvides(views);
-
- return dataNode;
- }
-
- private String getParentPath(List<String> paths) {
- // All the paths have the same parent, we can choose the first for extracting the path
- String firstPath = paths.get(0);
- return firstPath.substring(0, firstPath.lastIndexOf("/"));
- }
-
- private String getTempFileEndpoint(String target) {
-
- Transfer transfer = new Transfer();
- transfer.setDirection("pushToVoSpace");
- transfer.setTarget(target);
- Protocol protocol = new Protocol();
- protocol.setUri("ivo://ivoa.net/vospace/core#httpput");
- transfer.getProtocols().add(protocol);
-
- return client.getFileServiceEndpoint(transfer);
- }
-
- private void upload(String endpoint, String content) {
-
- HttpHeaders headers = new HttpHeaders();
- headers.setContentType(MediaType.MULTIPART_FORM_DATA);
-
- MultiValueMap<String, Object> parts = new LinkedMultiValueMap<>();
- parts.add("file", new MultipartFileResource(content, "list.txt"));
-
- HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(parts, headers);
-
- restTemplate.exchange(endpoint, HttpMethod.PUT, requestEntity, Void.class);
- }
-
- private class MultipartFileResource extends ByteArrayResource {
-
- private final String fileName;
-
- public MultipartFileResource(String content, String fileName) {
- super(content.getBytes());
- this.fileName = fileName;
- }
-
- @Override
- public String getFilename() {
- return this.fileName;
- }
- }
-
@GetMapping(value = "/jobs", produces = MediaType.APPLICATION_JSON_VALUE)
public List<Job> getJobs() {
return client.getJobs();
diff --git a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/NodesController.java b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/NodesController.java
index ad4d2e9..0b44715 100644
--- a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/NodesController.java
+++ b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/NodesController.java
@@ -11,6 +11,7 @@ import it.inaf.ia2.vospace.ui.data.ListNodeData;
import it.inaf.ia2.vospace.ui.service.MainNodesHtmlGenerator;
import it.inaf.ia2.vospace.ui.service.MoveNodeModalHtmlGenerator;
import it.inaf.oats.vospace.datamodel.NodeUtils;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -92,7 +93,7 @@ public class NodesController extends BaseController {
Transfer transfer = new Transfer();
transfer.setDirection("pullFromVoSpace");
- transfer.setTarget("vos://" + authority + path);
+ transfer.setTarget(Arrays.asList("vos://" + authority + path));
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpget");
@@ -159,7 +160,7 @@ public class NodesController extends BaseController {
String direction = getRequiredParam(params, "direction");
Transfer transfer = new Transfer();
- transfer.setTarget("vos://" + authority + target);
+ transfer.setTarget(Arrays.asList("vos://" + authority + target));
transfer.setDirection("vos://" + authority + direction);
JobSummary job = client.startTransferJob(transfer);
diff --git a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/UploadController.java b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/UploadController.java
index ea2cab9..3508f3f 100644
--- a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/UploadController.java
+++ b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/UploadController.java
@@ -94,7 +94,7 @@ public class UploadController extends BaseController {
Transfer transfer = new Transfer();
transfer.setDirection("pushToVoSpace");
- transfer.setTarget(uri);
+ transfer.setTarget(Arrays.asList(uri));
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpput");
diff --git a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/MainNodesHtmlGenerator.java b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/MainNodesHtmlGenerator.java
index f20357d..5aaef51 100644
--- a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/MainNodesHtmlGenerator.java
+++ b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/MainNodesHtmlGenerator.java
@@ -31,11 +31,6 @@ public class MainNodesHtmlGenerator extends NodesHtmlGenerator {
protected void addChild(Node child, Element containerElement) {
NodeInfo nodeInfo = new NodeInfo(child, user, authority);
- if (nodeInfo.isListOfFiles()) {
- // hidden file
- return;
- }
-
Element row = containerElement.appendElement("tr");
addSelectionCell(nodeInfo, row);
diff --git a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/NodeInfo.java b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/NodeInfo.java
index e6d43a5..68e5108 100644
--- a/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/NodeInfo.java
+++ b/vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/NodeInfo.java
@@ -35,7 +35,6 @@ public class NodeInfo {
private final boolean asyncTrans;
private final boolean sticky;
private final boolean busy;
- private final boolean listOfFiles;
private final boolean writable;
private final boolean deletable;
@@ -52,7 +51,6 @@ public class NodeInfo {
this.asyncTrans = isAsyncTrans(node);
this.sticky = isSticky(node);
this.busy = isBusy(node);
- this.listOfFiles = isListOfFiles(node);
this.writable = NodeUtils.checkIfWritable(node, user.getName(), user.getGroups()) && !busy;
this.deletable = writable && !sticky && !asyncTrans;
}
@@ -98,21 +96,6 @@ public class NodeInfo {
return node instanceof DataNode && ((DataNode) node).isBusy();
}
- private boolean isListOfFiles(Node node) {
- if (node instanceof DataNode) {
- DataNode dataNode = (DataNode) node;
- List<View> provides = dataNode.getProvides();
- if (provides != null) {
- for (View provide : provides) {
- if ("urn:list-of-files".equals(provide.getUri())) {
- return true;
- }
- }
- }
- }
- return false;
- }
-
private Optional<String> getProperty(Node node, String uri) {
if (node.getProperties() != null && node.getProperties() != null) {
for (Property property : node.getProperties()) {
@@ -197,10 +180,6 @@ public class NodeInfo {
return busy;
}
- public boolean isListOfFiles() {
- return listOfFiles;
- }
-
public boolean isWritable() {
return writable;
}
diff --git a/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/client/VOSpaceClientTest.java b/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/client/VOSpaceClientTest.java
index efb8197..0a3a110 100644
--- a/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/client/VOSpaceClientTest.java
+++ b/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/client/VOSpaceClientTest.java
@@ -14,6 +14,7 @@ import java.net.http.HttpClient;
import java.net.http.HttpHeaders;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import javax.servlet.http.HttpServletRequest;
@@ -115,7 +116,7 @@ public class VOSpaceClientTest {
Transfer transfer = new Transfer();
transfer.setDirection("pushToVoSpace");
- transfer.setTarget("vos://ia2.inaf.it!vospace/mynode");
+ transfer.setTarget(Arrays.asList("vos://ia2.inaf.it!vospace/mynode"));
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpput");
@@ -132,7 +133,7 @@ public class VOSpaceClientTest {
Transfer transfer = new Transfer();
transfer.setDirection("pushToVoSpace");
- transfer.setTarget("vos://ia2.inaf.it!vospace/mynode");
+ transfer.setTarget(Arrays.asList("vos://ia2.inaf.it!vospace/mynode"));
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpput");
@@ -161,7 +162,7 @@ public class VOSpaceClientTest {
Transfer transfer = new Transfer();
transfer.setDirection("pushToVoSpace");
- transfer.setTarget("vos://ia2.inaf.it!vospace/mynode");
+ transfer.setTarget(Arrays.asList("vos://ia2.inaf.it!vospace/mynode"));
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpput");
diff --git a/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/JobControllerTest.java b/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/JobControllerTest.java
index 72302f3..4b6f675 100644
--- a/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/JobControllerTest.java
+++ b/vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/JobControllerTest.java
@@ -63,7 +63,7 @@ public class JobControllerTest {
job.setPhase(ExecutionPhase.PENDING);
when(client.startTransferJob(argThat(transfer -> {
- return transfer.getTarget().startsWith("vos://example.com!vospace/path/to/.tmp-");
+ return transfer.getTarget().size() > 1;
}))).thenReturn(job);
when(client.getFileServiceEndpoint(any())).thenReturn("http://file-service/path/to/file");
--
GitLab