dugulp
/
textile_backup_chinese
mirror of https://github.com/dugulingping/textile_backup_chinese.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Some minor improvements to compressors

2.x-1.16
szymon 2020-08-11 09:43:19 +02:00
parent 1ea2dc2969
commit 786666f827
2 changed files with 35 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractTarCompressor.java
View File

@ -18,27 +18,28 @@
package net.szum123321.textile_backup.core.create.compressors; package net.szum123321.textile_backup.core.create.compressors;
import net.minecraft.server.command.ServerCommandSource;
import net.szum123321.textile_backup.Statics; import net.szum123321.textile_backup.Statics;
import net.szum123321.textile_backup.core.Utilities; import net.szum123321.textile_backup.core.Utilities;
import net.szum123321.textile_backup.core.create.BackupContext;
import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.compress.utils.IOUtils;
import java.io.*; import java.io.*;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration; import java.time.Duration;
import java.time.Instant; import java.time.Instant;
public abstract class AbstractTarCompressor { public abstract class AbstractTarCompressor {
protected abstract OutputStream openCompressorStream(OutputStream outputStream, int coreCountLimit) throws IOException; protected abstract OutputStream openCompressorStream(OutputStream outputStream, int coreCountLimit) throws IOException;
public void createArchive(File inputFile, File out, ServerCommandSource ctx, int coreLimit) { public void createArchive(File inputFile, File outputFile, BackupContext ctx, int coreLimit) {
Statics.LOGGER.sendInfo(ctx, "Starting compression..."); Statics.LOGGER.sendInfo(ctx, "Starting compression...");
Instant start = Instant.now(); Instant start = Instant.now();
try (FileOutputStream outStream = new FileOutputStream(out); try (FileOutputStream outStream = new FileOutputStream(outputFile);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream); BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream);
OutputStream compressorOutputStream = openCompressorStream(bufferedOutputStream, coreLimit); OutputStream compressorOutputStream = openCompressorStream(bufferedOutputStream, coreLimit);
TarArchiveOutputStream arc = new TarArchiveOutputStream(compressorOutputStream)) { TarArchiveOutputStream arc = new TarArchiveOutputStream(compressorOutputStream)) {
@ -46,32 +47,27 @@ public abstract class AbstractTarCompressor {
arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
Files.walk(inputFile.toPath()) Files.walk(inputFile.toPath())
.filter(path -> !path.equals(inputFile.toPath()))
.filter(path -> path.toFile().isFile())
.filter(path -> !Utilities.isBlacklisted(inputFile.toPath().relativize(path))) .filter(path -> !Utilities.isBlacklisted(inputFile.toPath().relativize(path)))
.forEach(path -> { .map(Path::toFile)
File file = path.toAbsolutePath().toFile(); .filter(File::isFile)
.forEach(file -> {
try (FileInputStream fileInputStream = new FileInputStream(file); try (FileInputStream fileInputStream = new FileInputStream(file)){
BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) { ArchiveEntry entry = arc.createArchiveEntry(file, inputFile.toPath().relativize(file.toPath()).toString());
ArchiveEntry entry = arc.createArchiveEntry(file, inputFile.toPath().relativize(path).toString());
arc.putArchiveEntry(entry); arc.putArchiveEntry(entry);
IOUtils.copy(bufferedInputStream, arc);
IOUtils.copy(fileInputStream, arc);
arc.closeArchiveEntry(); arc.closeArchiveEntry();
} catch (IOException e) { } catch (IOException e) {
Statics.LOGGER.error("An exception occurred while trying to compress: {}", path.getFileName(), e); Statics.LOGGER.error("An exception occurred while trying to compress: {}", file.getName(), e);
Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!"); Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
} }
}); });
arc.finish();
} catch (IOException e) { } catch (IOException e) {
Statics.LOGGER.error("An exception occurred!", e); Statics.LOGGER.error("An exception occurred!", e);
Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!"); Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
} finally {
Statics.LOGGER.sendInfo(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now())));
} }
Statics.LOGGER.sendInfo(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now())));
} }
} }

39
src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java
View File

@ -18,9 +18,9 @@
package net.szum123321.textile_backup.core.create.compressors; package net.szum123321.textile_backup.core.create.compressors;
import net.minecraft.server.command.ServerCommandSource;
import net.szum123321.textile_backup.Statics; import net.szum123321.textile_backup.Statics;
import net.szum123321.textile_backup.core.Utilities; import net.szum123321.textile_backup.core.Utilities;
import net.szum123321.textile_backup.core.create.BackupContext;
import org.apache.commons.compress.archivers.zip.*; import org.apache.commons.compress.archivers.zip.*;
import org.apache.commons.compress.parallel.InputStreamSupplier; import org.apache.commons.compress.parallel.InputStreamSupplier;
@ -40,12 +40,14 @@ import java.util.zip.ZipEntry;
https://stackoverflow.com/users/2987755/dkb https://stackoverflow.com/users/2987755/dkb
*/ */
public class ParallelZipCompressor { public class ParallelZipCompressor {
public static void createArchive(File in, File out, ServerCommandSource ctx, int coreLimit) { public static void createArchive(File inputFile, File outputFile, BackupContext ctx, int coreLimit) {
Statics.LOGGER.sendInfo(ctx, "Starting compression..."); Statics.LOGGER.sendInfo(ctx, "Starting compression...");
Instant start = Instant.now(); Instant start = Instant.now();
try (FileOutputStream fileOutputStream = new FileOutputStream(out); Path rootPath = inputFile.toPath();
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream); BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream);
ZipArchiveOutputStream arc = new ZipArchiveOutputStream(bufferedOutputStream)) { ZipArchiveOutputStream arc = new ZipArchiveOutputStream(bufferedOutputStream)) {
@ -56,22 +58,23 @@ public class ParallelZipCompressor {
arc.setLevel(Statics.CONFIG.compression); arc.setLevel(Statics.CONFIG.compression);
arc.setComment("Created on: " + Utilities.getDateTimeFormatter().format(LocalDateTime.now())); arc.setComment("Created on: " + Utilities.getDateTimeFormatter().format(LocalDateTime.now()));
File input = in.getCanonicalFile(); Files.walk(inputFile.toPath())
.filter(path -> !Utilities.isBlacklisted(inputFile.toPath().relativize(path)))
.map(Path::toFile)
.filter(File::isFile)
.forEach(file -> {
try { //IOException gets thrown only when arc is closed
ZipArchiveEntry entry = (ZipArchiveEntry)arc.createArchiveEntry(file, rootPath.relativize(file.toPath()).toString());
Files.walk(input.toPath()) entry.setMethod(ZipEntry.DEFLATED);
.filter(path -> !path.equals(input.toPath())) scatterZipCreator.addArchiveEntry(entry, new FileInputStreamSupplier(file));
.filter(path -> path.toFile().isFile()) } catch (IOException e) {
.filter(path -> !Utilities.isBlacklisted(input.toPath().relativize(path))) Statics.LOGGER.error("An exception occurred while trying to compress: {}", file.getName(), e);
.forEach(p -> { Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
ZipArchiveEntry entry = new ZipArchiveEntry(input.toPath().relativize(p).toString()); }
entry.setMethod(ZipEntry.DEFLATED);
FileInputStreamSupplier supplier = new FileInputStreamSupplier(p);
scatterZipCreator.addArchiveEntry(entry, supplier);
}); });
scatterZipCreator.writeTo(arc); scatterZipCreator.writeTo(arc);
arc.finish();
} catch (IOException | InterruptedException | ExecutionException e) { } catch (IOException | InterruptedException | ExecutionException e) {
Statics.LOGGER.error("An exception occurred!", e); Statics.LOGGER.error("An exception occurred!", e);
Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!"); Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
@ -81,16 +84,16 @@ public class ParallelZipCompressor {
} }
static class FileInputStreamSupplier implements InputStreamSupplier { static class FileInputStreamSupplier implements InputStreamSupplier {
private final Path sourceFile; private final File sourceFile;
private InputStream stream; private InputStream stream;
FileInputStreamSupplier(Path sourceFile) { FileInputStreamSupplier(File sourceFile) {
this.sourceFile = sourceFile; this.sourceFile = sourceFile;
} }
public InputStream get() { public InputStream get() {
try { try {
stream = Files.newInputStream(sourceFile); stream = new BufferedInputStream(new FileInputStream(sourceFile));
} catch (IOException e) { } catch (IOException e) {
Statics.LOGGER.error("An exception occurred while trying to create input stream!", e); Statics.LOGGER.error("An exception occurred while trying to create input stream!", e);
} }