zip ang zip are now multi threaded

build.gradle

@@ -15,6 +15,7 @@ minecraft {
 
 repositories{
 	maven { url 'http://server.bbkr.space:8081/artifactory/libs-release' }
+	maven { url 'https://jitpack.io' }
 }
 
 
@@ -33,8 +34,13 @@ dependencies {
 	include "io.github.cottonmc.cotton:cotton-logging:1.0.0-rc.4"
 	include "io.github.cottonmc.cotton:cotton-config:1.0.0-rc.7"
 
-	modCompile "org.apache.commons:commons-compress:1.20"
+	modCompile "org.apache.commons:commons-compress:1.13"
-	include "org.apache.commons:commons-compress:1.20"
+	include "org.apache.commons:commons-compress:1.13"
+
+	modCompile "org.tukaani:xz:1.8"
+	include "org.tukaani:xz:1.8"
+
+	modCompile 'com.github.shevek:parallelgzip:master-SNAPSHOT'
 }
 
 processResources {

gradle.properties

@@ -9,6 +9,6 @@ loader_version=0.8.2+build.194
 fabric_version=0.5.1+build.294-1.15
 
 # Mod Properties
-	mod_version = 1.2.0B-1.15
+mod_version = 1.2.0S-1.15
-	maven_group = net.szum123321
+maven_group = net.szum123321
-	archives_base_name = textile_backup
+archives_base_name = textile_backup

src/Copyright_Notice.txt

@@ -0,0 +1,8 @@
+This project uses third party libraries as its dependencies and includes them in jar. Those are :
+	Apache Commons Compress version 1.20 licensed under Apache License Version 2.0 which can be found at http://www.apache.org/licenses/
+	Cotton config, Cotton logging, and Jankson-Fabric all by Cotton team licensed under MIT license which can be found at https://github.com/CottonMC/Cotton
+	XZ for Java by Tukaani released as public domain. https://tukaani.org/xz/java.html
+
+Some code was partially or fully inspired by:
+    Parallel zip compression: https://stackoverflow.com/questions/54624695/how-to-implement-parallel-zip-creation-with-scatterzipoutputstream-with-zip64-su
+        answer by: https://stackoverflow.com/users/2987755/dkb

src/main/java/net/szum123321/textile_backup/ConfigHandler.java

@@ -59,9 +59,9 @@ public class ConfigHandler {
 
     @Comment(value = "\nAvailable formats are:\n" +
                     "ZIP - normal zip archive using standard deflate compression\n" +
-                    "BZIP2 - tar.bz2 archive using bzip2 compression\n" +
                     "GIZP - tar.gz using gzip compression\n" +
-                    "LZ4 - tar.lz using lz4 compression\n")
+                    "BZIP2 - tar.bz2 archive using bzip2 compression\n" +
+                    "LZMA - tar.xz using lzma compression\n")
     public final ArchiveFormat format = ArchiveFormat.ZIP;
 
     @Comment("\nPrint info to game out\n")
@@ -84,9 +84,9 @@ public class ConfigHandler {
 
     public enum ArchiveFormat {
         ZIP(".zip"),
-        BZIP2(".tar.bz2"),
         GZIP(".tar.gz"),
-        LZ4(".tar.lz4");
+        BZIP2(".tar.bz2"),
+        LZMA(".tar.xz");
 
         private final String extension;
 

src/main/java/net/szum123321/textile_backup/core/BackupHelper.java

@@ -128,9 +128,7 @@ public class BackupHelper {
 	}
 
 	private static String getFileExtension(File f) {
-		System.out.println(f.getName());
 		String[] parts = f.getName().split("\\.");
-		System.out.println(parts.length);
 
 		switch (parts[parts.length - 1]) {
 			case "zip":
@@ -139,8 +137,8 @@ public class BackupHelper {
 				return ConfigHandler.ArchiveFormat.BZIP2.getExtension();
 			case "gz":
 				return ConfigHandler.ArchiveFormat.GZIP.getExtension();
-			case "lz4":
+			case "xz":
-				return ConfigHandler.ArchiveFormat.LZ4.getExtension();
+				return ConfigHandler.ArchiveFormat.LZMA.getExtension();
 
 			default:
 				return null;
@@ -149,7 +147,7 @@ public class BackupHelper {
 
 
 	public static File getBackupRootPath(String worldName) {
-		File path = new File(TextileBackup.config.path);
+		File path = new File(TextileBackup.config.path).getAbsoluteFile();
 
 		if (TextileBackup.config.perWorldBackup)
 			path = path.toPath().resolve(worldName).toFile();

src/main/java/net/szum123321/textile_backup/core/MakeBackupThread.java

@@ -23,10 +23,11 @@ import net.minecraft.server.command.ServerCommandSource;
 import net.minecraft.world.dimension.DimensionType;
 import net.szum123321.textile_backup.TextileBackup;
 import net.szum123321.textile_backup.core.compressors.GenericTarCompressor;
-import net.szum123321.textile_backup.core.compressors.ZipCompressor;
+import net.szum123321.textile_backup.core.compressors.ParallelZipCompressor;
+import org.anarres.parallelgzip.ParallelGZIPOutputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
-import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorOutputStream;
+import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
 
 import java.io.File;
 import java.io.IOException;
@@ -67,7 +68,7 @@ public class MakeBackupThread implements Runnable {
 
         switch (TextileBackup.config.format) {
             case ZIP:
-                ZipCompressor.createArchive(world, outFile, ctx);
+                ParallelZipCompressor.createArchive(world, outFile, ctx);
                 break;
 
             case BZIP2:
@@ -75,16 +76,16 @@ public class MakeBackupThread implements Runnable {
                 break;
 
             case GZIP:
-                GenericTarCompressor.createArchive(world, outFile, GzipCompressorOutputStream.class, ctx);
+                GenericTarCompressor.createArchive(world, outFile, ParallelGZIPOutputStream.class, ctx);
                 break;
 
-            case LZ4:
+            case LZMA:
-                GenericTarCompressor.createArchive(world, outFile, FramedLZ4CompressorOutputStream.class, ctx);
+                GenericTarCompressor.createArchive(world, outFile, XZCompressorOutputStream.class, ctx);
                 break;
 
             default:
                 Utilities.log("Error! No correct compression format specified! using default compressor!", ctx);
-                ZipCompressor.createArchive(world, outFile, ctx);
+                ParallelZipCompressor.createArchive(world, outFile, ctx);
                 break;
         }
 

src/main/java/net/szum123321/textile_backup/core/compressors/GenericTarCompressor.java

@@ -8,24 +8,26 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.compress.compressors.CompressorOutputStream;
 import org.apache.commons.compress.utils.IOUtils;
 
+
 import java.io.*;
 import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Files;
 
 public class GenericTarCompressor {
-	public static void createArchive(File in, File out, Class<? extends CompressorOutputStream> CompressorStreamClass, ServerCommandSource ctx) {
+	public static void createArchive(File in, File out, Class<? extends OutputStream> CompressorStreamClass, ServerCommandSource ctx) {
 		Utilities.log("Starting compression...", ctx);
 
+		long start = System.nanoTime();
+
 		try (FileOutputStream outStream = new FileOutputStream(out);
 			 BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream);
-			 CompressorOutputStream compressorStream = CompressorStreamClass.getDeclaredConstructor(OutputStream.class).newInstance(bufferedOutputStream);// CompressorStreamClass.getConstructor().newInstance(bufferedOutputStream);
+			 OutputStream compressorStream = CompressorStreamClass.getDeclaredConstructor(OutputStream.class).newInstance(bufferedOutputStream);// CompressorStreamClass.getConstructor().newInstance(bufferedOutputStream);
 			 TarArchiveOutputStream arc = new TarArchiveOutputStream(compressorStream)) {
 
 			arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
 			arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
 
 			File input = in.getCanonicalFile();
-			int rootPathLength = input.toString().length() + 1;
 
 			Files.walk(input.toPath()).filter(
 					path -> !path.equals(input.toPath()) &&
@@ -36,7 +38,7 @@ public class GenericTarCompressor {
 
 				try (FileInputStream fin = new FileInputStream(file);
 					 BufferedInputStream bfin = new BufferedInputStream(fin)){
-					ArchiveEntry entry = arc.createArchiveEntry(file, file.getAbsolutePath().substring(rootPathLength));
+					ArchiveEntry entry = arc.createArchiveEntry(file, input.toPath().relativize(path).toString());
 
 					arc.putArchiveEntry(entry);
 					IOUtils.copy(bfin, arc);
@@ -46,11 +48,14 @@ public class GenericTarCompressor {
 					TextileBackup.logger.error(e.getMessage());
 				}
 			});
+
 			arc.finish();
 		} catch (IOException | IllegalAccessException | NoSuchMethodException | InstantiationException | InvocationTargetException e) {
 			TextileBackup.logger.error(e.toString());
 		}
 
-		Utilities.log("Compression finished", ctx);
+		long end = System.nanoTime();
+
+		Utilities.log("Compression took: " + ((end - start) / 1000000000.0) + "s", ctx);
 	}
 }

src/main/java/net/szum123321/textile_backup/core/compressors/ParallelZipCompressor.java

@@ -0,0 +1,83 @@
+package net.szum123321.textile_backup.core.compressors;
+
+import net.minecraft.server.command.ServerCommandSource;
+import net.szum123321.textile_backup.TextileBackup;
+import net.szum123321.textile_backup.core.Utilities;
+import org.apache.commons.compress.archivers.zip.*;
+import org.apache.commons.compress.parallel.InputStreamSupplier;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.util.concurrent.*;
+import java.util.zip.ZipEntry;
+
+/*
+	This part of code is based on:
+	https://stackoverflow.com/questions/54624695/how-to-implement-parallel-zip-creation-with-scatterzipoutputstream-with-zip64-su
+	answer by:
+	https://stackoverflow.com/users/2987755/dkb
+*/
+
+public class ParallelZipCompressor {
+	public static void createArchive(File in, File out, ServerCommandSource ctx) {
+		Utilities.log("Starting compression...", ctx);
+
+		long start = System.nanoTime();;
+
+		try (FileOutputStream fileOutputStream = new FileOutputStream(out);
+			 BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream);
+			 ZipArchiveOutputStream arc = new ZipArchiveOutputStream(bufferedOutputStream)) {
+
+			ParallelScatterZipCreator scatterZipCreator = new ParallelScatterZipCreator();
+
+			arc.setMethod(ZipArchiveOutputStream.DEFLATED);
+			arc.setUseZip64(Zip64Mode.AsNeeded);
+			arc.setLevel(TextileBackup.config.compression);
+			arc.setComment("Created on: " + Utilities.getDateTimeFormatter().format(LocalDateTime.now()));
+
+			File input = in.getCanonicalFile();
+
+			Files.walk(input.toPath()).filter(
+					path -> !path.equals(input.toPath()) &&
+							path.toFile().isFile() &&
+							!Utilities.isBlacklisted(input.toPath().relativize(path))
+			).forEach(p -> {
+					ZipArchiveEntry entry = new ZipArchiveEntry(input.toPath().relativize(p).toString());
+					entry.setMethod(ZipEntry.DEFLATED);
+					FileInputStreamSupplier supplier = new FileInputStreamSupplier(p);
+					scatterZipCreator.addArchiveEntry(entry, supplier);
+			});
+
+			scatterZipCreator.writeTo(arc);
+
+			arc.finish();
+		} catch (IOException | InterruptedException | ExecutionException e) {
+			TextileBackup.logger.error(e.getMessage());
+		}
+
+		long end = System.nanoTime();
+
+		Utilities.log("Compression took: " + ((end - start) / 1000000000.0) + "s", ctx);
+	}
+
+	static class FileInputStreamSupplier implements InputStreamSupplier {
+		private final Path sourceFile;
+		private InputStream stream;
+
+		FileInputStreamSupplier(Path sourceFile) {
+			this.sourceFile = sourceFile;
+		}
+
+		public InputStream get() {
+			try {
+				System.out.println("Creating: " + sourceFile);
+				stream = Files.newInputStream(sourceFile);
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			return stream;
+		}
+	}
+}

src/main/java/net/szum123321/textile_backup/core/compressors/ZipCompressor.java

@@ -1,76 +0,0 @@
-/*
-    A simple backup mod for Fabric
-    Copyright (C) 2020  Szum123321
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
-package net.szum123321.textile_backup.core.compressors;
-
-import net.minecraft.server.command.ServerCommandSource;
-import net.szum123321.textile_backup.TextileBackup;
-import net.szum123321.textile_backup.core.Utilities;
-import org.apache.commons.compress.archivers.zip.Zip64Mode;
-import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
-import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
-import org.apache.commons.compress.utils.IOUtils;
-
-import java.io.*;
-import java.nio.file.Files;
-import java.time.LocalDateTime;
-
-public class ZipCompressor {
-	public static void createArchive(File in, File out, ServerCommandSource ctx) {
-		Utilities.log("Starting compression...", ctx);
-
-		try (FileOutputStream fileOutputStream = new FileOutputStream(out);
-			 BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream);
-			 ZipArchiveOutputStream arc = new ZipArchiveOutputStream(bufferedOutputStream)) {
-
-			arc.setMethod(ZipArchiveOutputStream.DEFLATED);
-			arc.setUseZip64(Zip64Mode.AsNeeded);
-			arc.setLevel(TextileBackup.config.compression);
-			arc.setComment("Created on: " + Utilities.getDateTimeFormatter().format(LocalDateTime.now()));
-
-			File input = in.getCanonicalFile();
-			int rootPathLength = input.toString().length() + 1;
-
-			Files.walk(input.toPath()).filter(
-					path -> !path.equals(input.toPath()) &&
-							path.toFile().isFile() &&
-							!Utilities.isBlacklisted(input.toPath().relativize(path))
-			).forEach(path -> {
-				File file = path.toAbsolutePath().toFile();
-
-				try (FileInputStream fin = new FileInputStream(file);
-					 BufferedInputStream bfin = new BufferedInputStream(fin)) {
-					ZipArchiveEntry entry = new ZipArchiveEntry(file, file.getAbsolutePath().substring(rootPathLength));
-
-					arc.putArchiveEntry(entry);
-					IOUtils.copy(bfin, arc);
-
-					arc.closeArchiveEntry();
-				} catch (IOException e) {
-					TextileBackup.logger.error(e.getMessage());
-				}
-			});
-
-			arc.finish();
-		} catch (IOException e) {
-			TextileBackup.logger.error(e.getMessage());
-		}
-
-		Utilities.log("Compression finished", ctx);
-	}
-}