From 5f74d33e32ff24dc8c3ece3decf1661ab5f27bfe Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sun, 24 May 2020 12:44:58 +0200 Subject: [PATCH] I think i repaired https://github.com/Szum123321/textile_backup/issues/12 --- gradle.properties | 10 ++-- .../textile_backup/ConfigHandler.java | 3 + .../textile_backup/core/MakeBackupThread.java | 22 ++++--- .../compressors/GenericTarCompressor.java | 2 +- .../compressors/ParallelBZip2Compressor.java | 4 +- .../compressors/ParallelGzipCompressor.java | 59 +++++++++++++++++++ .../compressors/ParallelZipCompressor.java | 7 ++- 7 files changed, 88 insertions(+), 19 deletions(-) create mode 100644 src/main/java/net/szum123321/textile_backup/core/compressors/ParallelGzipCompressor.java diff --git a/gradle.properties b/gradle.properties index ca8e893..3e30579 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,14 +1,14 @@ # Done to increase the memory available to gradle. org.gradle.jvmargs=-Xmx1G -minecraft_version=20w19a -yarn_mappings=20w19a+build.6 -loader_version=0.8.2+build.194 +minecraft_version=20w21a +yarn_mappings=20w21a+build.15 +loader_version=0.8.4+build.198 #Fabric api -fabric_version=0.10.7+build.344-1.16 +fabric_version=0.10.10+build.347-1.16 # Mod Properties -mod_version = 1.2.0-20w19a +mod_version = 1.2.1-20w21a maven_group = net.szum123321 archives_base_name = textile_backup \ No newline at end of file diff --git a/src/main/java/net/szum123321/textile_backup/ConfigHandler.java b/src/main/java/net/szum123321/textile_backup/ConfigHandler.java index 360fadf..7bf95dc 100644 --- a/src/main/java/net/szum123321/textile_backup/ConfigHandler.java +++ b/src/main/java/net/szum123321/textile_backup/ConfigHandler.java @@ -64,6 +64,9 @@ public class ConfigHandler { "LZMA - tar.xz using lzma compression\n") public ArchiveFormat format = ArchiveFormat.ZIP; + @Comment("\nLimit how many cores can be used for compression.\n") + public int compressionCoreCountLimit = 0; + @Comment("\nPrint info to game out\n") public boolean log = true; diff --git a/src/main/java/net/szum123321/textile_backup/core/MakeBackupThread.java b/src/main/java/net/szum123321/textile_backup/core/MakeBackupThread.java index 7c4a8cb..9c75598 100644 --- a/src/main/java/net/szum123321/textile_backup/core/MakeBackupThread.java +++ b/src/main/java/net/szum123321/textile_backup/core/MakeBackupThread.java @@ -18,18 +18,16 @@ package net.szum123321.textile_backup.core; -import net.minecraft.class_5218; import net.minecraft.server.MinecraftServer; import net.minecraft.server.command.ServerCommandSource; import net.minecraft.world.dimension.DimensionType; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.core.compressors.GenericTarCompressor; import net.szum123321.textile_backup.core.compressors.ParallelBZip2Compressor; +import net.szum123321.textile_backup.core.compressors.ParallelGzipCompressor; import net.szum123321.textile_backup.core.compressors.ParallelZipCompressor; import net.szum123321.textile_backup.mixin.MinecraftServerSessionAccessor; -import org.anarres.parallelgzip.ParallelGZIPOutputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; -import org.at4j.comp.bzip2.BZip2OutputStream; import java.io.File; import java.io.IOException; @@ -67,26 +65,34 @@ public class MakeBackupThread implements Runnable { return; } + int coreCount; + + if(TextileBackup.config.compressionCoreCountLimit <= 0) { + coreCount = Runtime.getRuntime().availableProcessors(); + } else { + coreCount = Math.min(TextileBackup.config.compressionCoreCountLimit, Runtime.getRuntime().availableProcessors()); + } + switch (TextileBackup.config.format) { case ZIP: - ParallelZipCompressor.createArchive(world, outFile, ctx); + ParallelZipCompressor.createArchive(world, outFile, ctx, coreCount); break; case BZIP2: - ParallelBZip2Compressor.createArchive(world, outFile, ctx); + ParallelBZip2Compressor.createArchive(world, outFile, ctx, coreCount); break; case GZIP: - GenericTarCompressor.createArchive(world, outFile, ParallelGZIPOutputStream.class, ctx); + ParallelGzipCompressor.createArchive(world, outFile, ctx, coreCount); break; case LZMA: - GenericTarCompressor.createArchive(world, outFile, XZCompressorOutputStream.class, ctx); + GenericTarCompressor.createArchive(world, outFile, XZCompressorOutputStream.class, ctx, coreCount); break; default: Utilities.log("Error! No correct compression format specified! using default compressor!", ctx); - ParallelZipCompressor.createArchive(world, outFile, ctx); + ParallelZipCompressor.createArchive(world, outFile, ctx, coreCount); break; } diff --git a/src/main/java/net/szum123321/textile_backup/core/compressors/GenericTarCompressor.java b/src/main/java/net/szum123321/textile_backup/core/compressors/GenericTarCompressor.java index 85dffb7..8b96f31 100644 --- a/src/main/java/net/szum123321/textile_backup/core/compressors/GenericTarCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/compressors/GenericTarCompressor.java @@ -13,7 +13,7 @@ import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; public class GenericTarCompressor { - public static void createArchive(File in, File out, Class CompressorStreamClass, ServerCommandSource ctx) { + public static void createArchive(File in, File out, Class CompressorStreamClass, ServerCommandSource ctx, int coreLimit) { Utilities.log("Starting compression...", ctx); long start = System.nanoTime(); diff --git a/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelBZip2Compressor.java b/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelBZip2Compressor.java index bc0c3c2..eda9e5c 100644 --- a/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelBZip2Compressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelBZip2Compressor.java @@ -13,10 +13,10 @@ import java.io.*; import java.nio.file.Files; public class ParallelBZip2Compressor { - public static void createArchive(File in, File out, ServerCommandSource ctx) { + public static void createArchive(File in, File out, ServerCommandSource ctx, int coreLimit) { Utilities.log("Starting compression...", ctx); - BZip2OutputStreamSettings settings = new BZip2OutputStreamSettings().setNumberOfEncoderThreads(Runtime.getRuntime().availableProcessors()); + BZip2OutputStreamSettings settings = new BZip2OutputStreamSettings().setNumberOfEncoderThreads(coreLimit); long start = System.nanoTime(); diff --git a/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelGzipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelGzipCompressor.java new file mode 100644 index 0000000..ed52f08 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelGzipCompressor.java @@ -0,0 +1,59 @@ +package net.szum123321.textile_backup.core.compressors; + +import net.minecraft.server.command.ServerCommandSource; +import net.szum123321.textile_backup.TextileBackup; +import net.szum123321.textile_backup.core.Utilities; +import org.anarres.parallelgzip.ParallelGZIPOutputStream; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.utils.IOUtils; + +import java.io.*; +import java.nio.file.Files; + +public class ParallelGzipCompressor { + public static void createArchive(File in, File out, ServerCommandSource ctx, int coreLimit) { + Utilities.log("Starting compression...", ctx); + + long start = System.nanoTime(); + + try (FileOutputStream outStream = new FileOutputStream(out); + BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream); + ParallelGZIPOutputStream gzipOutputStream = new ParallelGZIPOutputStream(bufferedOutputStream, coreLimit); + TarArchiveOutputStream arc = new TarArchiveOutputStream(gzipOutputStream)) { + + arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); + + File input = in.getCanonicalFile(); + + Files.walk(input.toPath() + ).filter(path -> !path.equals(input.toPath()) && + path.toFile().isFile() && + !Utilities.isBlacklisted(input.toPath().relativize(path)) + ).forEach(path -> { + File file = path.toAbsolutePath().toFile(); + + try (FileInputStream fin = new FileInputStream(file); + BufferedInputStream bfin = new BufferedInputStream(fin)) { + ArchiveEntry entry = arc.createArchiveEntry(file, input.toPath().relativize(path).toString()); + + arc.putArchiveEntry(entry); + IOUtils.copy(bfin, arc); + + arc.closeArchiveEntry(); + } catch (IOException e) { + TextileBackup.logger.error(e.getMessage()); + } + }); + + arc.finish(); + } catch (IOException e) { + TextileBackup.logger.error(e.toString()); + } + + long end = System.nanoTime(); + + Utilities.log("Compression took: " + ((end - start) / 1000000000.0) + "s", ctx); + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelZipCompressor.java index 29c48c6..15108bd 100644 --- a/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelZipCompressor.java @@ -11,6 +11,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; import java.util.concurrent.*; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; /* @@ -21,7 +22,7 @@ import java.util.zip.ZipEntry; */ public class ParallelZipCompressor { - public static void createArchive(File in, File out, ServerCommandSource ctx) { + public static void createArchive(File in, File out, ServerCommandSource ctx, int coreLimit) { Utilities.log("Starting compression...", ctx); long start = System.nanoTime(); @@ -30,7 +31,7 @@ public class ParallelZipCompressor { BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream); ZipArchiveOutputStream arc = new ZipArchiveOutputStream(bufferedOutputStream)) { - ParallelScatterZipCreator scatterZipCreator = new ParallelScatterZipCreator(); + ParallelScatterZipCreator scatterZipCreator = new ParallelScatterZipCreator(Executors.newFixedThreadPool(coreLimit)); arc.setMethod(ZipArchiveOutputStream.DEFLATED); arc.setUseZip64(Zip64Mode.AsNeeded); @@ -43,7 +44,7 @@ public class ParallelZipCompressor { ).filter(path -> !path.equals(input.toPath()) && path.toFile().isFile() && !Utilities.isBlacklisted(input.toPath().relativize(path)) - ).forEach(p -> { + ).collect(Collectors.toList()).parallelStream().forEach(p -> { ZipArchiveEntry entry = new ZipArchiveEntry(input.toPath().relativize(p).toString()); entry.setMethod(ZipEntry.DEFLATED); FileInputStreamSupplier supplier = new FileInputStreamSupplier(p);