Skip to content
Snippets Groups Projects
Commit a5974e7f authored by Sonia Zorba's avatar Sonia Zorba
Browse files

Used multiple targets for transfers instead of creating temporary 'list-of-files' node

parent f2417395
No related branches found
No related tags found
No related merge requests found
Pipeline #1972 passed
......@@ -8,34 +8,23 @@ package it.inaf.ia2.vospace.ui.controller;
import it.inaf.ia2.vospace.ui.client.VOSpaceClient;
import it.inaf.ia2.vospace.ui.data.Job;
import it.inaf.ia2.vospace.ui.exception.BadRequestException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
import net.ivoa.xml.uws.v1.ErrorType;
import net.ivoa.xml.uws.v1.ExecutionPhase;
import net.ivoa.xml.uws.v1.JobSummary;
import net.ivoa.xml.vospace.v2.Param;
import net.ivoa.xml.vospace.v2.Protocol;
import net.ivoa.xml.vospace.v2.StructuredDataNode;
import net.ivoa.xml.vospace.v2.Transfer;
import net.ivoa.xml.vospace.v2.View;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
@RestController
public class JobController extends BaseController {
......@@ -48,9 +37,6 @@ public class JobController extends BaseController {
@Autowired
private VOSpaceClient client;
@Autowired
private RestTemplate restTemplate;
@PostMapping(value = "/recall", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<Job> startRecallFromTapeJob(@RequestBody List<String> paths) {
......@@ -58,23 +44,13 @@ public class JobController extends BaseController {
throw new BadRequestException("Received empty list of nodes");
}
String target;
if (paths.size() == 1) {
target = "vos://" + authority + paths.get(0);
} else {
target = createTempListOfFilesNode(paths);
}
List<String> targets = paths.stream().map(p -> "vos://" + authority + p).collect(Collectors.toList());
Transfer transfer = new Transfer();
transfer.setDirection("pullToVoSpace");
transfer.setTarget(target);
transfer.setTarget(targets);
Protocol protocol = new Protocol();
Param param = new Param();
param.setUri("ia2:node-type");
param.setValue(paths.size() == 1 ? "single" : "list");
protocol.getParam().add(param);
protocol.setUri("ia2:async-recall");
transfer.getProtocols().add(protocol);
......@@ -110,85 +86,6 @@ public class JobController extends BaseController {
throw new RuntimeException(errorMessage);
}
private String createTempListOfFilesNode(List<String> paths) {
StructuredDataNode dataNode = createStructuredDataNode(paths);
client.createNode(dataNode);
String uploadEndpoint = getTempFileEndpoint(dataNode.getUri());
String content = String.join("\n", paths);
upload(uploadEndpoint, content);
return dataNode.getUri();
}
private StructuredDataNode createStructuredDataNode(List<String> paths) {
List<View> views = new ArrayList<>();
View view = new View();
view.setUri("urn:list-of-files");
views.add(view);
StructuredDataNode dataNode = new StructuredDataNode();
String parentPath = getParentPath(paths);
String newTempFile = ".tmp-" + UUID.randomUUID().toString().replace("-", "") + ".txt";
dataNode.setUri("vos://" + authority + parentPath + "/" + newTempFile);
dataNode.setAccepts(views);
dataNode.setProvides(views);
return dataNode;
}
private String getParentPath(List<String> paths) {
// All the paths have the same parent, we can choose the first for extracting the path
String firstPath = paths.get(0);
return firstPath.substring(0, firstPath.lastIndexOf("/"));
}
private String getTempFileEndpoint(String target) {
Transfer transfer = new Transfer();
transfer.setDirection("pushToVoSpace");
transfer.setTarget(target);
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpput");
transfer.getProtocols().add(protocol);
return client.getFileServiceEndpoint(transfer);
}
private void upload(String endpoint, String content) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.MULTIPART_FORM_DATA);
MultiValueMap<String, Object> parts = new LinkedMultiValueMap<>();
parts.add("file", new MultipartFileResource(content, "list.txt"));
HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(parts, headers);
restTemplate.exchange(endpoint, HttpMethod.PUT, requestEntity, Void.class);
}
private class MultipartFileResource extends ByteArrayResource {
private final String fileName;
public MultipartFileResource(String content, String fileName) {
super(content.getBytes());
this.fileName = fileName;
}
@Override
public String getFilename() {
return this.fileName;
}
}
@GetMapping(value = "/jobs", produces = MediaType.APPLICATION_JSON_VALUE)
public List<Job> getJobs() {
return client.getJobs();
......
......@@ -11,6 +11,7 @@ import it.inaf.ia2.vospace.ui.data.ListNodeData;
import it.inaf.ia2.vospace.ui.service.MainNodesHtmlGenerator;
import it.inaf.ia2.vospace.ui.service.MoveNodeModalHtmlGenerator;
import it.inaf.oats.vospace.datamodel.NodeUtils;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
......@@ -92,7 +93,7 @@ public class NodesController extends BaseController {
Transfer transfer = new Transfer();
transfer.setDirection("pullFromVoSpace");
transfer.setTarget("vos://" + authority + path);
transfer.setTarget(Arrays.asList("vos://" + authority + path));
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpget");
......@@ -159,7 +160,7 @@ public class NodesController extends BaseController {
String direction = getRequiredParam(params, "direction");
Transfer transfer = new Transfer();
transfer.setTarget("vos://" + authority + target);
transfer.setTarget(Arrays.asList("vos://" + authority + target));
transfer.setDirection("vos://" + authority + direction);
JobSummary job = client.startTransferJob(transfer);
......
......@@ -94,7 +94,7 @@ public class UploadController extends BaseController {
Transfer transfer = new Transfer();
transfer.setDirection("pushToVoSpace");
transfer.setTarget(uri);
transfer.setTarget(Arrays.asList(uri));
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpput");
......
......@@ -31,11 +31,6 @@ public class MainNodesHtmlGenerator extends NodesHtmlGenerator {
protected void addChild(Node child, Element containerElement) {
NodeInfo nodeInfo = new NodeInfo(child, user, authority);
if (nodeInfo.isListOfFiles()) {
// hidden file
return;
}
Element row = containerElement.appendElement("tr");
addSelectionCell(nodeInfo, row);
......
......@@ -35,7 +35,6 @@ public class NodeInfo {
private final boolean asyncTrans;
private final boolean sticky;
private final boolean busy;
private final boolean listOfFiles;
private final boolean writable;
private final boolean deletable;
......@@ -52,7 +51,6 @@ public class NodeInfo {
this.asyncTrans = isAsyncTrans(node);
this.sticky = isSticky(node);
this.busy = isBusy(node);
this.listOfFiles = isListOfFiles(node);
this.writable = NodeUtils.checkIfWritable(node, user.getName(), user.getGroups()) && !busy;
this.deletable = writable && !sticky && !asyncTrans;
}
......@@ -98,21 +96,6 @@ public class NodeInfo {
return node instanceof DataNode && ((DataNode) node).isBusy();
}
private boolean isListOfFiles(Node node) {
if (node instanceof DataNode) {
DataNode dataNode = (DataNode) node;
List<View> provides = dataNode.getProvides();
if (provides != null) {
for (View provide : provides) {
if ("urn:list-of-files".equals(provide.getUri())) {
return true;
}
}
}
}
return false;
}
private Optional<String> getProperty(Node node, String uri) {
if (node.getProperties() != null && node.getProperties() != null) {
for (Property property : node.getProperties()) {
......@@ -197,10 +180,6 @@ public class NodeInfo {
return busy;
}
public boolean isListOfFiles() {
return listOfFiles;
}
public boolean isWritable() {
return writable;
}
......
......@@ -14,6 +14,7 @@ import java.net.http.HttpClient;
import java.net.http.HttpHeaders;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import javax.servlet.http.HttpServletRequest;
......@@ -115,7 +116,7 @@ public class VOSpaceClientTest {
Transfer transfer = new Transfer();
transfer.setDirection("pushToVoSpace");
transfer.setTarget("vos://ia2.inaf.it!vospace/mynode");
transfer.setTarget(Arrays.asList("vos://ia2.inaf.it!vospace/mynode"));
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpput");
......@@ -132,7 +133,7 @@ public class VOSpaceClientTest {
Transfer transfer = new Transfer();
transfer.setDirection("pushToVoSpace");
transfer.setTarget("vos://ia2.inaf.it!vospace/mynode");
transfer.setTarget(Arrays.asList("vos://ia2.inaf.it!vospace/mynode"));
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpput");
......@@ -161,7 +162,7 @@ public class VOSpaceClientTest {
Transfer transfer = new Transfer();
transfer.setDirection("pushToVoSpace");
transfer.setTarget("vos://ia2.inaf.it!vospace/mynode");
transfer.setTarget(Arrays.asList("vos://ia2.inaf.it!vospace/mynode"));
Protocol protocol = new Protocol();
protocol.setUri("ivo://ivoa.net/vospace/core#httpput");
......
......@@ -63,7 +63,7 @@ public class JobControllerTest {
job.setPhase(ExecutionPhase.PENDING);
when(client.startTransferJob(argThat(transfer -> {
return transfer.getTarget().startsWith("vos://example.com!vospace/path/to/.tmp-");
return transfer.getTarget().size() > 1;
}))).thenReturn(job);
when(client.getFileServiceEndpoint(any())).thenReturn("http://file-service/path/to/file");
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment