Skip to content
Snippets Groups Projects
Commit 06166fe9 authored by Wei Liu's avatar Wei Liu
Browse files

added --preserve-modified-time option for unimelb-mf-upload

parent 71331aab
Branches
Tags
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 748 additions and 698 deletions
src/main/java/unimelb/mf/client/file/PosixAttributes.java
+ 0
− 1
View file @ 06166fe9
......@@ -12,7 +12,6 @@ import java.nio.file.attribute.PosixFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
......
src/main/java/unimelb/mf/client/sync/MFSyncApp.java
+ 6
− 4
View file @ 06166fe9
......@@ -794,8 +794,8 @@ public abstract class MFSyncApp extends AbstractMFApp<unimelb.mf.client.sync.set
settings().wormCanAddVersions(), settings().wormCanMove(),
settings().saveFileAttributes(),
settings().aggregateUploadThreshold(), settings().splitThreshold(),
settings().maxNumberOfRetries(), settings().retryWaitTime(), _ul,
_workers));
settings().maxNumberOfRetries(), settings().retryWaitTime(),
settings().preserveModifiedTime(), _ul, _workers));
files.clear();
}
}
......@@ -843,7 +843,8 @@ public abstract class MFSyncApp extends AbstractMFApp<unimelb.mf.client.sync.set
settings().followSymlinks(), duj, settings().csumCheck(), settings().worm(),
settings().wormCanAddVersions(), settings().wormCanMove(), settings().saveFileAttributes(),
settings().aggregateUploadThreshold(), settings().splitThreshold(),
settings().maxNumberOfRetries(), settings().retryWaitTime(), _ul, _workers));
settings().maxNumberOfRetries(), settings().retryWaitTime(), settings().preserveModifiedTime(),
_ul, _workers));
files.clear();
}
} else if (job instanceof FileSetUploadJob) {
......@@ -866,7 +867,8 @@ public abstract class MFSyncApp extends AbstractMFApp<unimelb.mf.client.sync.set
settings().followSymlinks(), uj, settings().csumCheck(), settings().worm(),
settings().wormCanAddVersions(), settings().wormCanMove(), settings().saveFileAttributes(),
settings().aggregateUploadThreshold(), settings().splitThreshold(),
settings().maxNumberOfRetries(), settings().retryWaitTime(), _ul, _workers));
settings().maxNumberOfRetries(), settings().retryWaitTime(),
settings().preserveModifiedTime(), _ul, _workers));
}
}
......
src/main/java/unimelb/mf/client/sync/cli/MFUpload.java
+ 4
− 0
View file @ 06166fe9
......@@ -82,6 +82,7 @@ public class MFUpload extends MFSyncApp {
System.out.println(" --worm-can-add-versions Allow to add new versions of metadata and content when in the WORM state. Ignored if --worm option is not specified.");
System.out.println(" --worm-no-move Disallow to move the uploaded assets. Ignored if --worm option is not specified.");
System.out.println(" --save-file-attrs If specified, save local file attributes, such as owner uid, gid, ctime and ACls, into Mediaflux asset metadata.");
System.out.println(" --preserve-modified-time If specified, preserve the local file's modified time as corresponding asset modified time in Mediaflux.");
System.out.println(" --quiet Do not print progress messages.");
System.out.println(" --help Prints usage.");
System.out.println(" --version Prints version.");
......@@ -449,6 +450,9 @@ public class MFUpload extends MFSyncApp {
} else if ("--save-file-attrs".equalsIgnoreCase(args[i])) {
settings().setSaveFileAttributes(true);
return 1;
} else if ("--preserve-modified-time".equalsIgnoreCase(args[i])) {
settings().setPreserveModifiedTime(true);
return 1;
} else if ("--quiet".equalsIgnoreCase(args[i])) {
settings().setVerbose(false);
return 1;
......
src/main/java/unimelb/mf/client/sync/settings/Settings.java
+ 698
− 681
View file @ 06166fe9
This diff is collapsed.
src/main/java/unimelb/mf/client/sync/task/FileCRC32Task.java
+ 8
− 3
View file @ 06166fe9
......@@ -21,11 +21,14 @@ public class FileCRC32Task extends AbstractMFTask {
private final boolean _saveFileAttrs;
private final int _maxNumberOfRetries;
private final long _retryWaitTime;
private final boolean _preserveModifiedTime;
private final DataTransferListener<Path, String> _ul;
public FileCRC32Task(MFSession session, Logger logger, Path file, String assetPath, boolean dstIsNamespace,
String store, long assetCRC32, boolean worm, boolean wormCanAddVersions, boolean wormCanMove,
boolean saveFileAttrs, int maxNumberOfRetries, long retryWaitTime, DataTransferListener<Path, String> ul) {
boolean saveFileAttrs, int maxNumberOfRetries, long retryWaitTime, boolean preserveModifiedTime,
DataTransferListener<Path, String> ul) {
super(session, logger);
_file = file;
_assetPath = assetPath;
......@@ -38,6 +41,7 @@ public class FileCRC32Task extends AbstractMFTask {
_saveFileAttrs = saveFileAttrs;
_maxNumberOfRetries = maxNumberOfRetries;
_retryWaitTime = retryWaitTime;
_preserveModifiedTime = preserveModifiedTime;
_ul = ul;
}
......@@ -61,8 +65,9 @@ public class FileCRC32Task extends AbstractMFTask {
// NOTE: run upload task on the current thread instead of resubmit to the
// workers queue, because the workers queue may have already been shutdown and
// no longer accept new tasks.
new FileUploadTask(session(), logger(), _file, _assetPath, _dstIsNamespace, _dstStore, true, _worm, _wormCanAddVersions,
_wormCanMove, _saveFileAttrs, _maxNumberOfRetries, _retryWaitTime, _ul).execute();
new FileUploadTask(session(), logger(), _file, _assetPath, _dstIsNamespace, _dstStore, true, _worm,
_wormCanAddVersions, _wormCanMove, _saveFileAttrs, _maxNumberOfRetries, _retryWaitTime, _preserveModifiedTime, _ul)
.execute();
} else {
if (_ul != null) {
_ul.transferSkipped(_file, _assetPath);
......
src/main/java/unimelb/mf/client/sync/task/FileSetUploadTask.java
+ 11
− 5
View file @ 06166fe9
......@@ -35,6 +35,7 @@ public class FileSetUploadTask extends AbstractMFTask {
private int _maxNumberOfRetries = 0;
private long _retryWaitTime = 0;
private final boolean _preserveModifiedTime;
private final ThreadPoolExecutor _workers;
private final DataTransferListener<Path, String> _ul;
......@@ -53,7 +54,8 @@ public class FileSetUploadTask extends AbstractMFTask {
public FileSetUploadTask(MFSession session, Logger logger, List<Path> files, boolean followSymlinks, UploadJob job,
boolean csumCheck, boolean worm, boolean wormCanAddVersions, boolean wormCanMove, boolean saveFileAttrs,
long aggregateThreshold, long splitThreshold, int maxNumberOfRetries, long retryWaitTime,
DataTransferListener<Path, String> ul, ThreadPoolExecutor workers) throws Throwable {
boolean preserveModifiedTime, DataTransferListener<Path, String> ul, ThreadPoolExecutor workers)
throws Throwable {
super(session, logger);
_workers = workers;
_csumCheck = csumCheck;
......@@ -66,6 +68,8 @@ public class FileSetUploadTask extends AbstractMFTask {
_maxNumberOfRetries = maxNumberOfRetries;
_retryWaitTime = retryWaitTime;
_preserveModifiedTime = preserveModifiedTime;
_ul = ul;
_fileAssets = new LinkedHashMap<Path, String>();
_assetFiles = new LinkedHashMap<String, Path>();
......@@ -127,7 +131,7 @@ public class FileSetUploadTask extends AbstractMFTask {
} else {
_workers.submit(new FileUploadTask(session(), logger(), file, assetPath, _dstIsNamespace,
_dstStore, _csumCheck, _worm, _wormCanAddVersions, _wormCanMove, _saveFileAttrs,
_maxNumberOfRetries, _retryWaitTime, _ul));
_maxNumberOfRetries, _retryWaitTime, _preserveModifiedTime, _ul));
}
}
incCompletedOperations();
......@@ -177,7 +181,8 @@ public class FileSetUploadTask extends AbstractMFTask {
} else {
_workers.submit(new FileUploadTask(session(), logger(), file, assetPath,
_dstIsNamespace, _dstStore, _csumCheck, _worm, _wormCanAddVersions,
_wormCanMove, _saveFileAttrs, _maxNumberOfRetries, _retryWaitTime, _ul));
_wormCanMove, _saveFileAttrs, _maxNumberOfRetries, _retryWaitTime,
_preserveModifiedTime, _ul));
}
} else if (_csumCheck) {
if (assetCSum == null) {
......@@ -192,13 +197,14 @@ public class FileSetUploadTask extends AbstractMFTask {
} else {
_workers.submit(new FileUploadTask(session(), logger(), file, assetPath,
_dstIsNamespace, _dstStore, _csumCheck, _worm, _wormCanAddVersions,
_wormCanMove, _saveFileAttrs, _maxNumberOfRetries, _retryWaitTime, _ul));
_wormCanMove, _saveFileAttrs, _maxNumberOfRetries, _retryWaitTime,
_preserveModifiedTime, _ul));
}
} else {
// compare crc32 then decide
_workers.submit(new FileCRC32Task(session(), logger(), file, assetPath, _dstIsNamespace,
_dstStore, assetCSum, _worm, _wormCanAddVersions, _wormCanMove, _saveFileAttrs,
_maxNumberOfRetries, _retryWaitTime, _ul));
_maxNumberOfRetries, _retryWaitTime, _preserveModifiedTime, _ul));
}
} else {
// skip
......
src/main/java/unimelb/mf/client/sync/task/FileUploadTask.java
+ 21
− 4
View file @ 06166fe9
package unimelb.mf.client.sync.task;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
......@@ -36,6 +37,8 @@ public class FileUploadTask extends AbstractMFTask {
long _bytesUploaded = 0L;
private final DataTransferListener<Path, String> _ul;
private long _crc32;
private final boolean _preserveModifiedTime;
private PosixAttributes _fileAttrs;
/**
* @param session server session
......@@ -54,7 +57,8 @@ public class FileUploadTask extends AbstractMFTask {
*/
public FileUploadTask(MFSession session, Logger logger, Path file, String assetPath, boolean parentIsNamespace,
String store, boolean csumCheck, boolean worm, boolean wormCanAddVersions, boolean wormCanMove,
boolean saveFileAttrs, int maxNumberOfRetries, long retryWaitTime, DataTransferListener<Path, String> ul) {
boolean saveFileAttrs, int maxNumberOfRetries, long retryWaitTime, boolean preserveModifiedTime,
DataTransferListener<Path, String> ul) {
super(session, logger);
_file = file;
_assetPath = assetPath;
......@@ -68,6 +72,7 @@ public class FileUploadTask extends AbstractMFTask {
_maxNumberOfRetries = maxNumberOfRetries;
_retryWaitTime = retryWaitTime;
_bytesUploaded = 0;
_preserveModifiedTime = preserveModifiedTime;
_ul = ul;
}
......@@ -93,15 +98,21 @@ public class FileUploadTask extends AbstractMFTask {
w2.add("store", _store);
}
if (_saveFileAttrs) {
PosixAttributes fileAttrs = PosixAttributes.read(_file);
if (_saveFileAttrs && getFileAttributes() != null) {
w2.push("meta", new String[] { "action", "replace" });
fileAttrs.save(w2);
getFileAttributes().save(w2);
w2.pop();
}
w2.pop();
if (_preserveModifiedTime && getFileAttributes() != null && getFileAttributes().mtime() != null) {
w2.push("service", new String[] { "name", "asset.modify.time.set" });
w2.add("id", "path=" + _assetPath);
w2.add("time-in-millisec", _fileAttrs.mtime());
w2.pop();
}
w2.push("service", new String[] { "name", "asset.get" });
w2.add("id", "path=" + _assetPath);
w2.pop();
......@@ -197,4 +208,10 @@ public class FileUploadTask extends AbstractMFTask {
_crc32 = 0;
}
private PosixAttributes getFileAttributes() throws IOException {
if (_fileAttrs == null) {
_fileAttrs = PosixAttributes.read(_file);
}
return _fileAttrs;
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment