diff --git a/src/main/java/it/inaf/ia2/transfer/service/PutFileService.java b/src/main/java/it/inaf/ia2/transfer/service/PutFileService.java
index 83e0f0baae47c79b590cfd192157b8e64a9634cd..0b0c4883c24c427301563d3ee92a27b3039105cd 100644
--- a/src/main/java/it/inaf/ia2/transfer/service/PutFileService.java
+++ b/src/main/java/it/inaf/ia2/transfer/service/PutFileService.java
@@ -12,7 +12,10 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Objects;
@@ -126,16 +129,16 @@ public class PutFileService {
LOG.warn("Destination file {} size mismatch with source", destinationFile.toPath().toString());
}
- if (sourceFileInfo.getContentType() != null &&
- !sourceFileInfo.getContentType().equals(destinationFileInfo.getContentType())) {
+ if (sourceFileInfo.getContentType() != null
+ && !sourceFileInfo.getContentType().equals(destinationFileInfo.getContentType())) {
LOG.warn("Destination file {} content type mismatch with source {} {}", destinationFile.toPath().toString(),
destinationFileInfo.getContentType(), sourceFileInfo.getContentType());
}
- if (sourceFileInfo.getContentMd5() != null &&
- !sourceFileInfo.getContentMd5().equals(md5Checksum)) {
+ if (sourceFileInfo.getContentMd5() != null
+ && !sourceFileInfo.getContentMd5().equals(md5Checksum)) {
LOG.warn("Destination file {} md5 mismatch with source {} {}", destinationFile.toPath().toString(),
- destinationFileInfo.getContentMd5(), sourceFileInfo.getContentMd5() );
+ destinationFileInfo.getContentMd5(), sourceFileInfo.getContentMd5());
}
}
@@ -189,7 +192,27 @@ public class PutFileService {
private String makeMD5Checksum(File file) throws NoSuchAlgorithmException, IOException {
MessageDigest md = MessageDigest.getInstance("MD5");
- md.update(Files.readAllBytes(file.toPath()));
+
+ // We can't update the MessageDigest object in a single step using
+ // Files.readAllBytes because we want to handle also big files and that
+ // method loads all the content in memory (OutOfMemoryError has been
+ // noticed for files bigger than 1GB). So multiple updates using
+ // FileChannel and ByteBuffer are used
+ try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) {
+ // creating buffer (by default it is in write mode). FileChannel will write into buffer
+ ByteBuffer buffer = ByteBuffer.allocate(1024);
+ int bytesRead;
+ while ((bytesRead = channel.read(buffer)) != -1) {
+ if (bytesRead > 0) {
+ // switch from write to read mode
+ buffer.flip();
+ md.update(buffer);
+ }
+ // using rewind the buffer can be filled again
+ buffer.rewind();
+ }
+ }
+
byte[] digest = md.digest();
String checksum = DatatypeConverter.printHexBinary(digest);
return checksum;
diff --git a/src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java b/src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java
index c3e51bd42aa8d9a111e780480f176e047687dc73..23a2f031d608c0c2ad13a59bd8645b5601eaaee9 100644
--- a/src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java
+++ b/src/test/java/it/inaf/ia2/transfer/controller/PutFileControllerTest.java
@@ -25,6 +25,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.eq;
import org.mockito.Mockito;
import static org.mockito.Mockito.mock;
@@ -77,7 +78,7 @@ public class PutFileControllerTest {
public void putGenericFile() throws Exception {
when(fileDao.getRemainingQuota(any())).thenReturn(null);
-
+
String randomFileName = UUID.randomUUID().toString();
createBaseFileInfo(randomFileName);
@@ -92,9 +93,12 @@ public class PutFileControllerTest {
assertTrue(file.exists());
assertEquals("content", Files.contentOf(file, StandardCharsets.UTF_8));
+
+ verify(fileDao, times(1)).updateFileAttributes(anyInt(), eq("text/plain"), any(), eq(7l), eq("9A0364B9E99BB480DD25E1F0284C8555"));
+
assertTrue(file.delete());
}
-
+
@Test
public void putGenericFileWithNameConflictExtension() throws Exception {
putGenericFileWithNameConflict("test.txt", "test-1.txt", "test-2.txt");
@@ -108,7 +112,7 @@ public class PutFileControllerTest {
private void putGenericFileWithNameConflict(String name1, String name2, String name3) throws Exception {
when(fileDao.getRemainingQuota(any())).thenReturn(null);
-
+
createBaseFileInfo(name1);
MockMultipartFile fakeFile = new MockMultipartFile("file", "test.txt", "text/plain", "content".getBytes());
@@ -154,7 +158,7 @@ public class PutFileControllerTest {
public void putGenericFileWithJobId() throws Exception {
when(fileDao.getRemainingQuota(any())).thenReturn(null);
-
+
when(jobDAO.isJobExisting("pippo10")).thenReturn(false);
when(jobDAO.isJobExisting("pippo5")).thenReturn(true);
@@ -171,7 +175,7 @@ public class PutFileControllerTest {
verify(jobDAO, times(1)).isJobExisting(eq("pippo10"));
- // Retry with valid jobid
+ // Retry with valid jobid
mockMvc.perform(putMultipart("/path/to/test.txt")
.file(fakeFile).param("jobId", "pippo5"))
.andDo(print())
@@ -233,7 +237,7 @@ public class PutFileControllerTest {
verify(jobDAO, times(1)).setJobError(eq("abcdef"), any());
}
-
+
@Test
public void testQuotaExceededMultipart() throws Exception {
@@ -291,16 +295,16 @@ public class PutFileControllerTest {
String randomFileName = UUID.randomUUID().toString();
return createBaseFileInfo(randomFileName);
}
-
- private FileInfo createBaseFileInfo(String fileName) {
+
+ private FileInfo createBaseFileInfo(String fileName) {
FileInfo fileInfo = new FileInfo();
fileInfo.setOsPath(getTestFilePath(fileName));
fileInfo.setVirtualPath("/path/to/" + fileName);
fileInfo.setPublic(false);
-
+
when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo));
when(fileDao.setBusy(any(), any())).thenReturn(1);
-
+
return fileInfo;
}