Skip to content
Snippets Groups Projects
Commit dde5448c authored by Wei Liu's avatar Wei Liu
Browse files

Handle the access exception when accessing file size

parent 54e98784
Branches
Tags v0.7.3
No related merge requests found
Hide whitespace changes
Inline Side-by-side
pom.xml
+ 2
− 2
View file @ dde5448c
......@@ -5,7 +5,7 @@
<groupId>au.edu.unimelb.mf</groupId>
<artifactId>unimelb-mf-clients</artifactId>
<version>0.7.2</version>
<version>0.7.3</version>
<packaging>jar</packaging>
<name>unimelb-mf-clients</name>
<url>https://gitlab.unimelb.edu.au/resplat-mediaflux/unimelb-mf-clients</url>
......@@ -44,7 +44,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<version>4.13.1</version>
<scope>test</scope>
</dependency>
</dependencies>
......
src/main/java/unimelb/mf/client/sync/task/FileSetUploadTask.java
+ 72
− 63
View file @ dde5448c
......@@ -7,6 +7,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.logging.Level;
import java.util.logging.Logger;
import arc.xml.XmlDoc;
......@@ -106,23 +107,33 @@ public class FileSetUploadTask extends AbstractMFTask {
boolean exists = ee.booleanValue();
if (!exists) {
Path file = _assetFiles.get(assetPath);
boolean isSymbolicLink = Files.isSymbolicLink(file);
if (isSymbolicLink && !_followSymlinks) {
_workers.submit(new SymlinkUploadTask(session(), logger(), file, assetPath, _ul));
} else {
long fileSize = Files.size(file);
if (_aggregateThreshold > 0 && fileSize <= _aggregateThreshold) {
aggregateUploadFiles.put(assetPath, file);
} else if (_workers.getMaximumPoolSize() > 1 && fileSize >= _splitThreshold) {
new SlicedFileUploadTask(session(), logger(), file, _workers.getMaximumPoolSize(), assetPath,
_dstStore, _csumCheck, _worm, _wormCanAddVersions, _wormCanMove, _maxNumberOfRetries,
_ul, _workers).execute();
try {
if (file == null) {
throw new AssertionError("Could not find file matching asset: '" + assetPath + "'");
}
boolean isSymbolicLink = Files.isSymbolicLink(file);
if (isSymbolicLink && !_followSymlinks) {
_workers.submit(new SymlinkUploadTask(session(), logger(), file, assetPath, _ul));
} else {
_workers.submit(new FileUploadTask(session(), logger(), file, assetPath, _dstStore, _csumCheck,
_worm, _wormCanAddVersions, _wormCanMove, _saveFileAttrs, _maxNumberOfRetries, _retryWaitTime, _ul));
long fileSize = Files.size(file);
if (_aggregateThreshold > 0 && fileSize <= _aggregateThreshold) {
aggregateUploadFiles.put(assetPath, file);
} else if (_workers.getMaximumPoolSize() > 1 && fileSize >= _splitThreshold) {
new SlicedFileUploadTask(session(), logger(), file, _workers.getMaximumPoolSize(), assetPath,
_dstStore, _csumCheck, _worm, _wormCanAddVersions, _wormCanMove, _maxNumberOfRetries,
_ul, _workers).execute();
} else {
_workers.submit(new FileUploadTask(session(), logger(), file, assetPath, _dstStore, _csumCheck,
_worm, _wormCanAddVersions, _wormCanMove, _saveFileAttrs, _maxNumberOfRetries, _retryWaitTime, _ul));
}
}
incCompletedOperations();
} catch (Throwable e) {
if (_ul != null) {
_ul.transferFailed(file, assetPath);
}
logError("accessing " + file + ": " + e.getMessage(), e);
}
incCompletedOperations();
} else {
w2.add("id", "path=" + assetPath);
nbExists++;
......@@ -134,47 +145,25 @@ public class FileSetUploadTask extends AbstractMFTask {
String assetPath = ae.value("path");
Path file = _assetFiles.get(assetPath);
if (file == null) {
throw new AssertionError("Could not find file matching asset: '" + assetPath + "'");
}
boolean isSymbolicLink = Files.isSymbolicLink(file);
if (isSymbolicLink && !_followSymlinks) {
// update symlink asset.
_workers.submit(new SymlinkUploadTask(session(), logger(), file, assetPath, _ul));
} else {
// update file asset.
Long contentSize = ae.longValue("content/size", null);
String assetCSumStr = ae.value("content/csum[@base='16']");
Long assetCSum = assetCSumStr == null ? null : Long.parseLong(assetCSumStr, 16);
Long fileSize = null;
try {
fileSize = Files.size(file);
} catch (Throwable e) {
logError("Failed to read file size. File: '" + file + "'", e);
if (_ul != null) {
_ul.transferFailed(file, assetPath);
}
continue;
try {
if (file == null) {
throw new AssertionError("Could not find file matching asset: '" + assetPath + "'");
}
boolean isSymbolicLink = Files.isSymbolicLink(file);
if (isSymbolicLink && !_followSymlinks) {
// update symlink asset.
_workers.submit(new SymlinkUploadTask(session(), logger(), file, assetPath, _ul));
} else {
// update file asset.
Long contentSize = ae.longValue("content/size", null);
String assetCSumStr = ae.value("content/csum[@base='16']");
Long assetCSum = assetCSumStr == null ? null : Long.parseLong(assetCSumStr, 16);
long fileSize = Files.size(file);
boolean fileSizesMatch = contentSize != null && (contentSize.equals(fileSize));
boolean fileSizesMatch = contentSize != null && (contentSize.equals(fileSize));
if (!fileSizesMatch) {
// upload file if size does not match
if (_aggregateThreshold > 0 && fileSize <= _aggregateThreshold) {
aggregateUploadFiles.put(assetPath, file);
} else if (_workers.getMaximumPoolSize() > 1 && fileSize >= _splitThreshold) {
new SlicedFileUploadTask(session(), logger(), file, _workers.getMaximumPoolSize(),
assetPath, _dstStore, _csumCheck, _worm, _wormCanAddVersions, _wormCanMove,
_maxNumberOfRetries, _ul, _workers).execute();
} else {
_workers.submit(new FileUploadTask(session(), logger(), file, assetPath, _dstStore,
_csumCheck, _worm, _wormCanAddVersions, _wormCanMove, _saveFileAttrs,
_maxNumberOfRetries, _retryWaitTime, _ul));
}
} else if (_csumCheck) {
if (assetCSum == null) {
// no asset checksum available. Upload file regardless.
if (!fileSizesMatch) {
// upload file if size does not match
if (_aggregateThreshold > 0 && fileSize <= _aggregateThreshold) {
aggregateUploadFiles.put(assetPath, file);
} else if (_workers.getMaximumPoolSize() > 1 && fileSize >= _splitThreshold) {
......@@ -186,21 +175,41 @@ public class FileSetUploadTask extends AbstractMFTask {
_csumCheck, _worm, _wormCanAddVersions, _wormCanMove, _saveFileAttrs,
_maxNumberOfRetries, _retryWaitTime, _ul));
}
} else if (_csumCheck) {
if (assetCSum == null) {
// no asset checksum available. Upload file regardless.
if (_aggregateThreshold > 0 && fileSize <= _aggregateThreshold) {
aggregateUploadFiles.put(assetPath, file);
} else if (_workers.getMaximumPoolSize() > 1 && fileSize >= _splitThreshold) {
new SlicedFileUploadTask(session(), logger(), file, _workers.getMaximumPoolSize(),
assetPath, _dstStore, _csumCheck, _worm, _wormCanAddVersions, _wormCanMove,
_maxNumberOfRetries, _ul, _workers).execute();
} else {
_workers.submit(new FileUploadTask(session(), logger(), file, assetPath, _dstStore,
_csumCheck, _worm, _wormCanAddVersions, _wormCanMove, _saveFileAttrs,
_maxNumberOfRetries, _retryWaitTime, _ul));
}
} else {
// compare crc32 then decide
_workers.submit(new FileCRC32Task(session(), logger(), file, assetPath, _dstStore,
assetCSum, _worm, _wormCanAddVersions, _wormCanMove, _saveFileAttrs,
_maxNumberOfRetries, _retryWaitTime, _ul));
}
} else {
// compare crc32 then decide
_workers.submit(new FileCRC32Task(session(), logger(), file, assetPath, _dstStore,
assetCSum, _worm, _wormCanAddVersions, _wormCanMove, _saveFileAttrs,
_maxNumberOfRetries, _retryWaitTime, _ul));
}
} else {
// skip
if (_ul != null) {
_ul.transferSkipped(file, assetPath);
// skip
if (_ul != null) {
_ul.transferSkipped(file, assetPath);
}
logger().info("Asset '" + assetPath + "' already exists. Skipped.");
}
logger().info("Asset '" + assetPath + "' already exists. Skipped.");
}
incCompletedOperations();
} catch (Throwable e) {
logError("accessing " + file + ": " + e.getMessage(), e);
if (_ul != null) {
_ul.transferFailed(file, assetPath);
}
}
incCompletedOperations();
// @formatter:off
// PosixAttributes contentAttrs = ae.elementExists("meta/" + PosixAttributes.DOC_TYPE)
......
unimelb-mf-clients.iml deleted 100644 → 0
+ 0
− 20
View file @ 54e98784
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: com.arcitecta:mfclient:4.13.033" level="project" />
<orderEntry type="library" name="Maven: org.jline:jline:3.9.0" level="project" />
<orderEntry type="library" name="Maven: info.picocli:picocli:4.6.1" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
</component>
</module>
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment