Szum123321 2020-05-24 12:44:58 +02:00
parent 5d1bdf5e9e
commit 5f74d33e32
7 changed files with 88 additions and 19 deletions

View File

@ -1,14 +1,14 @@
# Done to increase the memory available to gradle. # Done to increase the memory available to gradle.
org.gradle.jvmargs=-Xmx1G org.gradle.jvmargs=-Xmx1G
minecraft_version=20w19a minecraft_version=20w21a
yarn_mappings=20w19a+build.6 yarn_mappings=20w21a+build.15
loader_version=0.8.2+build.194 loader_version=0.8.4+build.198
#Fabric api #Fabric api
fabric_version=0.10.7+build.344-1.16 fabric_version=0.10.10+build.347-1.16
# Mod Properties # Mod Properties
mod_version = 1.2.0-20w19a mod_version = 1.2.1-20w21a
maven_group = net.szum123321 maven_group = net.szum123321
archives_base_name = textile_backup archives_base_name = textile_backup

View File

@ -64,6 +64,9 @@ public class ConfigHandler {
"LZMA - tar.xz using lzma compression\n") "LZMA - tar.xz using lzma compression\n")
public ArchiveFormat format = ArchiveFormat.ZIP; public ArchiveFormat format = ArchiveFormat.ZIP;
@Comment("\nLimit how many cores can be used for compression.\n")
public int compressionCoreCountLimit = 0;
@Comment("\nPrint info to game out\n") @Comment("\nPrint info to game out\n")
public boolean log = true; public boolean log = true;

View File

@ -18,18 +18,16 @@
package net.szum123321.textile_backup.core; package net.szum123321.textile_backup.core;
import net.minecraft.class_5218;
import net.minecraft.server.MinecraftServer; import net.minecraft.server.MinecraftServer;
import net.minecraft.server.command.ServerCommandSource; import net.minecraft.server.command.ServerCommandSource;
import net.minecraft.world.dimension.DimensionType; import net.minecraft.world.dimension.DimensionType;
import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileBackup;
import net.szum123321.textile_backup.core.compressors.GenericTarCompressor; import net.szum123321.textile_backup.core.compressors.GenericTarCompressor;
import net.szum123321.textile_backup.core.compressors.ParallelBZip2Compressor; import net.szum123321.textile_backup.core.compressors.ParallelBZip2Compressor;
import net.szum123321.textile_backup.core.compressors.ParallelGzipCompressor;
import net.szum123321.textile_backup.core.compressors.ParallelZipCompressor; import net.szum123321.textile_backup.core.compressors.ParallelZipCompressor;
import net.szum123321.textile_backup.mixin.MinecraftServerSessionAccessor; import net.szum123321.textile_backup.mixin.MinecraftServerSessionAccessor;
import org.anarres.parallelgzip.ParallelGZIPOutputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
import org.at4j.comp.bzip2.BZip2OutputStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
@ -67,26 +65,34 @@ public class MakeBackupThread implements Runnable {
return; return;
} }
int coreCount;
if(TextileBackup.config.compressionCoreCountLimit <= 0) {
coreCount = Runtime.getRuntime().availableProcessors();
} else {
coreCount = Math.min(TextileBackup.config.compressionCoreCountLimit, Runtime.getRuntime().availableProcessors());
}
switch (TextileBackup.config.format) { switch (TextileBackup.config.format) {
case ZIP: case ZIP:
ParallelZipCompressor.createArchive(world, outFile, ctx); ParallelZipCompressor.createArchive(world, outFile, ctx, coreCount);
break; break;
case BZIP2: case BZIP2:
ParallelBZip2Compressor.createArchive(world, outFile, ctx); ParallelBZip2Compressor.createArchive(world, outFile, ctx, coreCount);
break; break;
case GZIP: case GZIP:
GenericTarCompressor.createArchive(world, outFile, ParallelGZIPOutputStream.class, ctx); ParallelGzipCompressor.createArchive(world, outFile, ctx, coreCount);
break; break;
case LZMA: case LZMA:
GenericTarCompressor.createArchive(world, outFile, XZCompressorOutputStream.class, ctx); GenericTarCompressor.createArchive(world, outFile, XZCompressorOutputStream.class, ctx, coreCount);
break; break;
default: default:
Utilities.log("Error! No correct compression format specified! using default compressor!", ctx); Utilities.log("Error! No correct compression format specified! using default compressor!", ctx);
ParallelZipCompressor.createArchive(world, outFile, ctx); ParallelZipCompressor.createArchive(world, outFile, ctx, coreCount);
break; break;
} }

View File

@ -13,7 +13,7 @@ import java.lang.reflect.InvocationTargetException;
import java.nio.file.Files; import java.nio.file.Files;
public class GenericTarCompressor { public class GenericTarCompressor {
public static void createArchive(File in, File out, Class<? extends OutputStream> CompressorStreamClass, ServerCommandSource ctx) { public static void createArchive(File in, File out, Class<? extends OutputStream> CompressorStreamClass, ServerCommandSource ctx, int coreLimit) {
Utilities.log("Starting compression...", ctx); Utilities.log("Starting compression...", ctx);
long start = System.nanoTime(); long start = System.nanoTime();

View File

@ -13,10 +13,10 @@ import java.io.*;
import java.nio.file.Files; import java.nio.file.Files;
public class ParallelBZip2Compressor { public class ParallelBZip2Compressor {
public static void createArchive(File in, File out, ServerCommandSource ctx) { public static void createArchive(File in, File out, ServerCommandSource ctx, int coreLimit) {
Utilities.log("Starting compression...", ctx); Utilities.log("Starting compression...", ctx);
BZip2OutputStreamSettings settings = new BZip2OutputStreamSettings().setNumberOfEncoderThreads(Runtime.getRuntime().availableProcessors()); BZip2OutputStreamSettings settings = new BZip2OutputStreamSettings().setNumberOfEncoderThreads(coreLimit);
long start = System.nanoTime(); long start = System.nanoTime();

View File

@ -0,0 +1,59 @@
package net.szum123321.textile_backup.core.compressors;
import net.minecraft.server.command.ServerCommandSource;
import net.szum123321.textile_backup.TextileBackup;
import net.szum123321.textile_backup.core.Utilities;
import org.anarres.parallelgzip.ParallelGZIPOutputStream;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import java.io.*;
import java.nio.file.Files;
public class ParallelGzipCompressor {
public static void createArchive(File in, File out, ServerCommandSource ctx, int coreLimit) {
Utilities.log("Starting compression...", ctx);
long start = System.nanoTime();
try (FileOutputStream outStream = new FileOutputStream(out);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream);
ParallelGZIPOutputStream gzipOutputStream = new ParallelGZIPOutputStream(bufferedOutputStream, coreLimit);
TarArchiveOutputStream arc = new TarArchiveOutputStream(gzipOutputStream)) {
arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
File input = in.getCanonicalFile();
Files.walk(input.toPath()
).filter(path -> !path.equals(input.toPath()) &&
path.toFile().isFile() &&
!Utilities.isBlacklisted(input.toPath().relativize(path))
).forEach(path -> {
File file = path.toAbsolutePath().toFile();
try (FileInputStream fin = new FileInputStream(file);
BufferedInputStream bfin = new BufferedInputStream(fin)) {
ArchiveEntry entry = arc.createArchiveEntry(file, input.toPath().relativize(path).toString());
arc.putArchiveEntry(entry);
IOUtils.copy(bfin, arc);
arc.closeArchiveEntry();
} catch (IOException e) {
TextileBackup.logger.error(e.getMessage());
}
});
arc.finish();
} catch (IOException e) {
TextileBackup.logger.error(e.toString());
}
long end = System.nanoTime();
Utilities.log("Compression took: " + ((end - start) / 1000000000.0) + "s", ctx);
}
}

View File

@ -11,6 +11,7 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.concurrent.*; import java.util.concurrent.*;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
/* /*
@ -21,7 +22,7 @@ import java.util.zip.ZipEntry;
*/ */
public class ParallelZipCompressor { public class ParallelZipCompressor {
public static void createArchive(File in, File out, ServerCommandSource ctx) { public static void createArchive(File in, File out, ServerCommandSource ctx, int coreLimit) {
Utilities.log("Starting compression...", ctx); Utilities.log("Starting compression...", ctx);
long start = System.nanoTime(); long start = System.nanoTime();
@ -30,7 +31,7 @@ public class ParallelZipCompressor {
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream); BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream);
ZipArchiveOutputStream arc = new ZipArchiveOutputStream(bufferedOutputStream)) { ZipArchiveOutputStream arc = new ZipArchiveOutputStream(bufferedOutputStream)) {
ParallelScatterZipCreator scatterZipCreator = new ParallelScatterZipCreator(); ParallelScatterZipCreator scatterZipCreator = new ParallelScatterZipCreator(Executors.newFixedThreadPool(coreLimit));
arc.setMethod(ZipArchiveOutputStream.DEFLATED); arc.setMethod(ZipArchiveOutputStream.DEFLATED);
arc.setUseZip64(Zip64Mode.AsNeeded); arc.setUseZip64(Zip64Mode.AsNeeded);
@ -43,7 +44,7 @@ public class ParallelZipCompressor {
).filter(path -> !path.equals(input.toPath()) && ).filter(path -> !path.equals(input.toPath()) &&
path.toFile().isFile() && path.toFile().isFile() &&
!Utilities.isBlacklisted(input.toPath().relativize(path)) !Utilities.isBlacklisted(input.toPath().relativize(path))
).forEach(p -> { ).collect(Collectors.toList()).parallelStream().forEach(p -> {
ZipArchiveEntry entry = new ZipArchiveEntry(input.toPath().relativize(p).toString()); ZipArchiveEntry entry = new ZipArchiveEntry(input.toPath().relativize(p).toString());
entry.setMethod(ZipEntry.DEFLATED); entry.setMethod(ZipEntry.DEFLATED);
FileInputStreamSupplier supplier = new FileInputStreamSupplier(p); FileInputStreamSupplier supplier = new FileInputStreamSupplier(p);