diff --git a/src/main/java/unimelb/mf/client/archive/MFArchive.java b/src/main/java/unimelb/mf/client/archive/MFArchive.java
index ccc81b59bfe7090e79198298bebf8ce574e9bfc1..f94aa40df01fabd97d8143363e2c3e8ae1a69a40 100644
--- a/src/main/java/unimelb/mf/client/archive/MFArchive.java
+++ b/src/main/java/unimelb/mf/client/archive/MFArchive.java
@@ -52,7 +52,7 @@ public class MFArchive {
if (ext != null) {
Type[] vs = values();
for (Type v : vs) {
- if ("tgz".equalsIgnoreCase(ext)) {
+ if ("tgz".equalsIgnoreCase(ext) || "gtar".equalsIgnoreCase(ext) || "tar.gz".equalsIgnoreCase(ext)) {
return GZIPPED_TAR;
} else if (v.extension.equalsIgnoreCase(ext)) {
return v;
@@ -76,7 +76,8 @@ public class MFArchive {
public static Type fromFileName(String fileName) {
if (fileName != null) {
- String ext = PathUtils.getFileExtension(fileName);
+ String ext = fileName.toLowerCase().endsWith(".tar.gz") ? "tar.gz"
+ : PathUtils.getFileExtension(fileName);
return fromFileExtension(ext);
}
return null;
diff --git a/src/main/java/unimelb/mf/client/sync/cli/MFImportArchive.java b/src/main/java/unimelb/mf/client/sync/cli/MFImportArchive.java
index 5f0fa51b2d64e3b16d8aab2455c36018e0b2071e..99d69121e57b919aef5d281261952ead5e72a50c 100644
--- a/src/main/java/unimelb/mf/client/sync/cli/MFImportArchive.java
+++ b/src/main/java/unimelb/mf/client/sync/cli/MFImportArchive.java
@@ -1,55 +1,96 @@
package unimelb.mf.client.sync.cli;
import java.nio.file.Path;
-import java.util.List;
+import java.util.Arrays;
import java.util.concurrent.Callable;
+import java.util.logging.Level;
+import java.util.logging.Logger;
-import unimelb.mf.client.archive.MFArchive;
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.Parameters;
+import unimelb.mf.client.session.MFConfigBuilder;
+import unimelb.mf.client.session.MFSession;
+import unimelb.mf.client.sync.task.FileSetArchiveCreateTask;
+import unimelb.utils.LoggingUtils;
+@Command(name = "mf-import-archive", abbreviateSynopsis = true, usageHelpWidth = 120, synopsisHeading = "\nUSAGE:\n ", descriptionHeading = "\nDESCRIPTION:\n ", description = "Import local files or directory into Mediaflux as an archive asset. The destination archive file format can be .zip, .tar or .aar.", parameterListHeading = "\nPARAMETERS:\n", optionListHeading = "\nOPTIONS:\n", sortOptions = false, version = MFImportArchive.VERSION, separator = " ")
public class MFImportArchive implements Callable<Integer> {
- public static class Options {
- private MFArchive.Type archiveType = MFArchive.Type.ZIP;
- private int compressLevel = 6;
- private boolean analyze = false;
+ public static final String VERSION = "0.0.1";
- public Options(MFArchive.Type archiveType, int compressLevel, boolean analyze) {
- this.archiveType = archiveType;
- this.compressLevel = compressLevel;
- this.analyze = analyze;
- }
+ @Option(names = { "-c",
+ "--config" }, required = false, paramLabel = "<mflux.cfg>", description = "Mediaflux configuration file. If not specified, it will try $HOME/.Arcitecta/mflux.cfg on MacOS/Linux or %%USERPROFILE%%\\.Arcitecta\\mflux.cfg on Windows.")
+ private Path cfgFile;
- public String archiveMimeType() {
- if (this.archiveType != null) {
- return this.archiveType.mimeType;
- }
- return null;
- }
+ @Option(names = { "-a",
+ "--asset-path" }, required = true, paramLabel = "<asset-path>", description = "Path to the destination archive asset in Mediaflux.")
+ private String assetPath;
+
+ @Option(names = {
+ "--log-dir" }, required = false, paramLabel = "<log-directory>", description = "The log directory. Logging is disabled unless this is specified.")
+ private Path logDir;
+
+ @Option(names = {
+ "--compress" }, required = false, negatable = true, description = "Compress the destination archive. By default, the archive is not compressed.")
+ private boolean compress = false;
+
+ @Option(names = {
+ "--analyze" }, required = false, negatable = true, description = "Analyze the asset content after the archive asset is created.")
+ private boolean analyze = false;
+
+ @Option(names = { "-h", "--help" }, usageHelp = true, description = "Print usage information")
+ private boolean printHelp;
+
+ @Option(names = { "--version" }, versionHelp = true, description = "Print version information")
+ private boolean printVersion;
+
+ @Option(names = { "--quiet" }, defaultValue = "false", description = "Quiet mode. ")
+ private boolean quiet;
+
+ @Parameters(description = "Source files or directories to import into Mediaflux.", index = "0", arity = "1..", paramLabel = "SRC_FILES")
+ private Path[] srcFiles;
- public String archiveFileExtension() {
- if (this.archiveType != null) {
- return this.archiveType.extension;
+ @Override
+ public Integer call() throws Exception {
+ try {
+ MFConfigBuilder mfconfig = new MFConfigBuilder().setConsoleLogon(true);
+ mfconfig.findAndLoadFromConfigFile();
+ mfconfig.loadFromSystemEnv();
+ if (this.cfgFile != null) {
+ mfconfig.loadFromConfigFile(cfgFile);
}
- return null;
- }
+ mfconfig.setApp(MFUpload.PROG);
- public int compressLevel() {
- return this.compressLevel;
- }
+ MFSession session = MFSession.create(mfconfig);
+
+ Logger logger = createLogger(this.logDir, "mf-import-archive", this.quiet ? Level.WARNING : Level.ALL,
+ 100000000, 2);
- public boolean analyze() {
- return this.analyze;
+ new FileSetArchiveCreateTask(session, logger, Arrays.asList(this.srcFiles), this.assetPath,
+ this.compress ? 6 : 0, this.analyze).execute();
+
+ return 0;
+ } catch (Throwable e) {
+ throw (e instanceof Exception) ? (Exception) e : new RuntimeException(e);
}
}
- public MFImportArchive(List<Path> files, String assetPath) {
-
+ private static Logger createLogger(Path logDir, String logFileName, Level logLevel, int logFileSize,
+ int logFileCount) throws Throwable {
+ Logger logger = logDir == null ? LoggingUtils.createConsoleLogger()
+ : LoggingUtils.createFileAndConsoleLogger(logDir, logFileName, logFileSize, logFileCount);
+ if (logLevel != null) {
+ logger.setLevel(logLevel);
+ } else {
+ logger.setLevel(Level.WARNING);
+ }
+ return logger;
}
- @Override
- public Integer call() throws Exception {
- // TODO Auto-generated method stub
- return null;
+ public static void main(String[] args) {
+ System.exit(new CommandLine(new MFImportArchive()).execute(args));
}
-}
+}
\ No newline at end of file
diff --git a/src/main/java/unimelb/mf/client/sync/task/AggregatedUploadTask.java b/src/main/java/unimelb/mf/client/sync/task/AggregatedUploadTask.java
index 7e7ea29fabdf7db3234f6cb37600d9434d186e0c..b1c5ee394ae0d81b16bd70f3dc6f8a9353133928 100644
--- a/src/main/java/unimelb/mf/client/sync/task/AggregatedUploadTask.java
+++ b/src/main/java/unimelb/mf/client/sync/task/AggregatedUploadTask.java
@@ -89,8 +89,8 @@ public class AggregatedUploadTask extends AbstractMFTask {
private void doAggregateUpload() throws Throwable {
- ServerClient.Input input = new MFGeneratedInput(MFArchive.TYPE_AAR, MFArchive.Type.AAR.extension,
- null, -1, _store) {
+ ServerClient.Input input = new MFGeneratedInput(MFArchive.TYPE_AAR, MFArchive.Type.AAR.extension, null, -1,
+ _store == null ? null : ("asset:" + _store)) {
@Override
protected void consume(OutputStream out, AbortCheck ac) throws Throwable {
Archive.declareSupportForAllTypes();
diff --git a/src/main/java/unimelb/mf/client/sync/task/FileSetArchiveCreateTask.java b/src/main/java/unimelb/mf/client/sync/task/FileSetArchiveCreateTask.java
index 74f7b2dc7059a8d2958ec59c19a63913caf28aec..530d0818c9478b89aeac81740b24d9ee3409772d 100644
--- a/src/main/java/unimelb/mf/client/sync/task/FileSetArchiveCreateTask.java
+++ b/src/main/java/unimelb/mf/client/sync/task/FileSetArchiveCreateTask.java
@@ -1,49 +1,193 @@
package unimelb.mf.client.sync.task;
+import java.io.IOException;
import java.io.OutputStream;
+import java.nio.file.FileVisitOption;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.EnumSet;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.logging.Level;
import java.util.logging.Logger;
+import arc.archive.ArchiveOutput;
import arc.archive.ArchiveRegistry;
import arc.mf.client.ServerClient;
import arc.mf.client.archive.Archive;
import arc.streams.StreamCopy.AbortCheck;
-import arc.utils.Path;
+import arc.xml.XmlStringWriter;
import unimelb.mf.client.archive.MFArchive;
+import unimelb.mf.client.session.MFGeneratedInput;
import unimelb.mf.client.session.MFSession;
import unimelb.mf.client.task.AbstractMFTask;
import unimelb.mf.client.util.AssetNamespaceUtils;
+import unimelb.mf.client.util.AssetUtils;
+import unimelb.utils.PathUtils;
public class FileSetArchiveCreateTask extends AbstractMFTask {
- private List<Path> _files;
- private String _dstAssetPath;
- private boolean _createNS;
- private int _compressLevel;
+ private Set<Path> _files;
+ private MFArchive.Type _archiveType;
+ private String _assetPath;
+ private int _compressionLevel = 6;
+ private boolean _analyze = false;
- protected FileSetArchiveCreateTask(MFSession session, Logger logger, List<Path> files, String dstAssetPath, boolean createNS, int compressLevel) {
+ public FileSetArchiveCreateTask(MFSession session, Logger logger, List<Path> files, String assetPath,
+ int compressionLevel, boolean analyze) {
super(session, logger);
+ _files = new LinkedHashSet<>();
+ if (files != null && !files.isEmpty()) {
+ _files.addAll(files);
+ }
+ _archiveType = archiveTypeFromPath(assetPath, MFArchive.Type.ZIP);
+ _assetPath = assetPath.toLowerCase().endsWith(_archiveType.extension) ? assetPath
+ : (assetPath + "." + _archiveType.extension);
+ _compressionLevel = compressionLevel;
+ _analyze = analyze;
+ }
- // TODO Auto-generated constructor stub
+ private static MFArchive.Type archiveTypeFromPath(String filePath, MFArchive.Type defaultType) {
+ if (filePath != null) {
+ String lcFileName = PathUtils.getFileName(filePath).toLowerCase();
+ if (lcFileName.endsWith(".tar.gz") || lcFileName.endsWith(".gtar") || lcFileName.endsWith(".tgz")) {
+ return MFArchive.Type.GZIPPED_TAR;
+ } else if (lcFileName.endsWith(".zip")) {
+ return MFArchive.Type.ZIP;
+ } else if (lcFileName.endsWith(".tar")) {
+ return MFArchive.Type.TAR;
+ } else if (lcFileName.endsWith(".jar")) {
+ return MFArchive.Type.JAR;
+ } else if (lcFileName.endsWith(".aar")) {
+ return MFArchive.Type.AAR;
+ }
+ }
+ return defaultType;
}
@Override
public void execute() throws Throwable {
- // if (_files == null || _files.isEmpty()) {
- // return;
- // }
- // String store = AssetNamespaceUtils.getStore(session(), _dstAssetNS);
- // String source = _files.size() == 1 ? _files.get(0).path() : null;
- // Archive.declareSupportForAllTypes();
- // ServerClient.Input input = new ServerClient.GeneratedInput(_arcType.mimeType, _arcType.extension, source,
- // _compressLevel, store) {
+ Archive.declareSupportForAllTypes();
+ Path baseDir = _files.size() == 1 ? _files.iterator().next().getParent()
+ : PathUtils.getLongestCommonParent(_files);
+ String source = _files.size() == 1 ? _files.iterator().next().toString() : null;
+ String store = getAssetStoreFor(session(), _assetPath);
+ store = store == null ? null : ("asset:" + store);
+ ServerClient.Input input = new MFGeneratedInput(_archiveType.mimeType, _archiveType.extension, source, -1,
+ store) {
+
+ @Override
+ protected void consume(OutputStream out, AbortCheck ac) throws Throwable {
+ ArchiveOutput ao = ArchiveRegistry.createOutput(out, _archiveType.mimeType, _compressionLevel, null);
+ try {
+ for (Path f : _files) {
+ add(ao, f, baseDir);
+ }
+ } finally {
+ ao.close();
+ }
+ }
+ };
+
+ XmlStringWriter w = new XmlStringWriter();
+ w.add("id", "path=" + _assetPath);
+ w.add("create", true);
+ w.add("analyze", _analyze);
+ session().execute("asset.set", w.document(), input);
+ String assetId = AssetUtils.getAssetMetaByPath(session(), _assetPath).value("@id");
+ System.out.println("Imported Mediaflux asset(id=" + assetId + "): " + _assetPath);
+ }
+
+ private void add(ArchiveOutput ao, Path f, Path baseDir) throws Throwable {
+ if (!Files.isDirectory(f)) {
+ addFile(ao, f, baseDir);
+ } else {
+ addDirectory(ao, f, baseDir);
+ }
+ }
+
+ private void addFile(ArchiveOutput ao, Path f, Path baseDir) throws Throwable {
+ String name;
+ if (baseDir == null) {
+ name = f.getFileName().toString();
+ } else {
+ String filePath = f.toString();
+ String baseDirPath = baseDir.toString();
+ if (filePath.startsWith(baseDirPath)) {
+ name = filePath.substring(baseDirPath.length() + 1);
+ } else {
+ baseDir = PathUtils.getLongestCommonParent(f, baseDir);
+ baseDirPath = baseDir == null ? null : baseDir.toString();
+ name = baseDir == null ? filePath : filePath.substring(baseDirPath.length() + 1);
+ }
+ }
+ logger().info("Adding '" + f + "'");
+ ao.add(null, name, f.toFile());
+ }
+
+ private void addDirectory(ArchiveOutput ao, Path dir, Path baseDir) throws Throwable {
+ Files.walkFileTree(dir, EnumSet.noneOf(FileVisitOption.class), Integer.MAX_VALUE,
+ new SimpleFileVisitor<Path>() {
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+ try {
+ addFile(ao, file, baseDir);
+ } catch (Throwable e) {
+ logger().log(Level.SEVERE, e.getMessage(), e);
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFileFailed(Path file, IOException ioe) {
+ logger().log(Level.SEVERE, "Failed to access file: " + file, ioe);
+ return FileVisitResult.CONTINUE;
+ }
- // @Override
- // protected void copyTo(OutputStream arg0, AbortCheck arg1) throws Throwable {
- // // TODO Auto-generated method stub
+ @Override
+ public FileVisitResult postVisitDirectory(Path dir, IOException ioe) {
+ if (ioe != null) {
+ logger().log(Level.SEVERE, "[postVisitDirectory] " + dir + ": " + ioe.getMessage(), ioe);
+ }
+ return FileVisitResult.CONTINUE;
+ }
- // }
- // };
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+ return super.preVisitDirectory(dir, attrs);
+ }
+ });
}
+ private static String getAssetStoreFor(MFSession session, String assetPath) throws Throwable {
+ String closestExistingAssetNamespace = getClosestExistingAssetNamespace(session, assetPath);
+ if (closestExistingAssetNamespace != null) {
+ String store = AssetNamespaceUtils.getStore(session, closestExistingAssetNamespace);
+ return store;
+ }
+ return null;
+ }
+
+ private static String getClosestExistingAssetNamespace(MFSession session, String assetPath) throws Throwable {
+ if (assetPath != null) {
+ Path path = Paths.get(assetPath.startsWith("/") ? assetPath : ("/" + assetPath));
+ Path parentPath = path.getParent();
+ while (parentPath != null) {
+ boolean exists = AssetNamespaceUtils.assetNamespaceExists(session, parentPath.toString());
+ if (exists) {
+ return parentPath.toString();
+ } else {
+ parentPath = parentPath.getParent();
+ }
+ }
+ }
+ return null;
+ }
}
diff --git a/src/main/java/unimelb/utils/PathUtils.java b/src/main/java/unimelb/utils/PathUtils.java
index 578a3889770eb324dac4d0335f7ee8b629189123..6dd147db394b73411bfc055660e3c1066e22a2a1 100644
--- a/src/main/java/unimelb/utils/PathUtils.java
+++ b/src/main/java/unimelb/utils/PathUtils.java
@@ -3,6 +3,8 @@ package unimelb.utils;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.Iterator;
public class PathUtils {
@@ -271,6 +273,70 @@ public class PathUtils {
return false;
}
+ public static Path getLongestCommonParent(Path... files) {
+ if (files == null || files.length < 2) {
+ return null;
+ }
+ Path commonParent = null;
+ /**
+ * compare the first two files...
+ */
+ if (files[0] != null && files[1] != null) {
+ Path rp = files[0].relativize(files[1]).normalize();
+ while (rp != null && !rp.endsWith("..")) {
+ rp = rp.getParent();
+ }
+ if (rp != null) {
+ commonParent = files[0].resolve(rp).normalize();
+ commonParent = (commonParent.toString() == null || commonParent.toString().isBlank()) ? null
+ : commonParent;
+ }
+ }
+
+ // return if there're only two files
+ if (files.length == 2) {
+ return commonParent;
+ }
+ // compare the rest files
+ for (int i = 2; i < files.length; i += 1) {
+ commonParent = getLongestCommonParent(files[i], commonParent);
+ }
+ return commonParent;
+ }
+
+ public static Path getLongestCommonParent(Collection<Path> files) {
+ if (files == null || files.isEmpty()) {
+ return null;
+ }
+
+ Iterator<Path> iterator = files.iterator();
+ Path file1 = iterator.next();
+ if (!iterator.hasNext()) {
+ return null;
+ }
+ Path file2 = iterator.next();
+
+ Path commonParent = null;
+ // get common parent of the first two files..
+ if (file1 != null && file2 != null) {
+ Path rp = file1.relativize(file2).normalize();
+ while (rp != null && !rp.endsWith("..")) {
+ rp = rp.getParent();
+ }
+ if (rp != null) {
+ commonParent = file1.resolve(rp).normalize();
+ commonParent = (commonParent.toString() == null || commonParent.toString().isBlank()) ? null
+ : commonParent;
+ }
+ }
+
+ // compare the rest files
+ while (iterator.hasNext()) {
+ commonParent = getLongestCommonParent(iterator.next(), commonParent);
+ }
+ return commonParent;
+ }
+
public static void main(String[] args) {
// System.out.println(joinSystemIndependent("ab/c", "\\d\\\\e\\",
// "eee//ddd"));
diff --git a/src/main/scripts/unix/unimelb-mf-import-archive b/src/main/scripts/unix/unimelb-mf-import-archive
new file mode 100644
index 0000000000000000000000000000000000000000..9a011bad744e72e6a1e71097e96cd71cd89dcf32
--- /dev/null
+++ b/src/main/scripts/unix/unimelb-mf-import-archive
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# ${ROOT}/bin/
+BIN=$(dirname ${BASH_SOURCE[0]})
+
+# current directory
+CWD=$(pwd)
+
+# ${ROOT}/
+ROOT=$(cd ${BIN}/../../ && pwd && cd ${CWD})
+
+# ${ROOT}/lib/
+LIB=${ROOT}/lib
+
+# ${ROOT}/lib/unimelb-mf-clients.jar
+JAR=${LIB}/unimelb-mf-clients.jar
+
+# check if unimelb-mf-clients.jar exists
+[[ ! -f $JAR ]] && echo "${JAR} is not found." >&2 && exit 2
+
+#export JAVA_HOME=${ROOT}/@JAVA_HOME@
+#export PATH=${JAVA_HOME}/bin:${PATH}
+
+# check if java exists
+[[ -z $(which java) ]] && echo "Java is not found." >&2 && exit 1
+
+# execute the command
+java -XX:+UseG1GC -XX:+UseStringDeduplication -Xmx1g -cp "${JAR}" unimelb.mf.client.sync.cli.MFImportArchive ${1+"$@"}
diff --git a/src/main/scripts/windows/unimelb-mf-import-archive.cmd b/src/main/scripts/windows/unimelb-mf-import-archive.cmd
new file mode 100644
index 0000000000000000000000000000000000000000..ecb93e4e4935bb899dbc9c3136b4b1cb494f3259
--- /dev/null
+++ b/src/main/scripts/windows/unimelb-mf-import-archive.cmd
@@ -0,0 +1,12 @@
+@echo off
+
+pushd %~dp0..\..\
+set ROOT=%cd%
+popd
+
+@REM set JAVA_HOME=%ROOT%\@JAVA_HOME@
+@REM set PATH=%JAVA_HOME%\bin;%PATH%
+
+set JAR=%ROOT%\lib\unimelb-mf-clients.jar
+
+java -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+UseStringDeduplication -Xmx1g -cp "%JAR%" unimelb.mf.client.sync.cli.MFImportArchive %*