diff --git a/build.gradle b/build.gradle index 610e8da..e31094b 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'fabric-loom' version '0.5-SNAPSHOT' + id 'fabric-loom' version '0.8-SNAPSHOT' id 'maven-publish' } @@ -14,7 +14,7 @@ minecraft { } repositories{ - maven { url 'http://server.bbkr.space:8081/artifactory/libs-release' } + maven { url 'https://server.bbkr.space/artifactory/libs-release' } maven { url 'https://jitpack.io' } } @@ -22,24 +22,24 @@ dependencies { //to change the versions see the gradle.properties file minecraft "com.mojang:minecraft:${project.minecraft_version}" mappings "net.fabricmc:yarn:${project.yarn_mappings}:v2" - modCompile "net.fabricmc:fabric-loader:${project.loader_version}" + modImplementation "net.fabricmc:fabric-loader:${project.loader_version}" // Fabric API. This is technically optional, but you probably want it anyway. - modCompile "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}" + modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}" - modCompile "io.github.cottonmc.cotton:cotton-config:1.0.0-rc.7" + modImplementation "io.github.cottonmc.cotton:cotton-config:1.0.0-rc.7" include "io.github.cottonmc:Jankson-Fabric:3.0.0+j1.2.0" include "io.github.cottonmc.cotton:cotton-logging:1.0.0-rc.4" include "io.github.cottonmc.cotton:cotton-config:1.0.0-rc.7" - modCompile "org.apache.commons:commons-compress:1.19" + modImplementation "org.apache.commons:commons-compress:1.19" include "org.apache.commons:commons-compress:1.19" - modCompile "org.tukaani:xz:1.8" + modImplementation "org.tukaani:xz:1.8" include "org.tukaani:xz:1.8" - modCompile 'com.github.shevek:parallelgzip:master-SNAPSHOT' + modImplementation 'com.github.shevek:parallelgzip:master-SNAPSHOT' include 'com.github.shevek:parallelgzip:master-SNAPSHOT' } diff --git a/gradle.properties b/gradle.properties index f7ab039..0276941 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,14 +1,14 @@ # Done to increase the memory available to gradle. org.gradle.jvmargs=-Xmx1G -minecraft_version=1.16.4 -yarn_mappings=1.16.4+build.7 -loader_version=0.10.8 +minecraft_version=1.17 +yarn_mappings=1.17+build.10 +loader_version=0.11.5 #Fabric api -fabric_version=0.26.3+1.16 +fabric_version=0.35.1+1.17 # Mod Properties -mod_version = 2.1.0-prev +mod_version = 2.1.0-prev5 maven_group = net.szum123321 archives_base_name = textile_backup \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 490fda8..e708b1c 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 622ab64..0f80bbf 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.0.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/net/szum123321/textile_backup/TextileBackup.java b/src/main/java/net/szum123321/textile_backup/TextileBackup.java index 3cca231..b1de95e 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileBackup.java +++ b/src/main/java/net/szum123321/textile_backup/TextileBackup.java @@ -45,6 +45,7 @@ import java.util.concurrent.Executors; public class TextileBackup implements ModInitializer { @Override public void onInitialize() { + //Statics.LOGGER.info("Tmpfs: {}", GetPropertyAction.privilegedGetProperty("java.io.tmpdir")); Statics.LOGGER.info("Starting Textile Backup by Szum123321."); Statics.CONFIG = ConfigManager.loadConfig(ConfigHandler.class); @@ -65,8 +66,7 @@ public class TextileBackup implements ModInitializer { } } - if(Statics.CONFIG.backupInterval > 0) - ServerTickEvents.END_SERVER_TICK.register(Statics.scheduler::tick); + if(Statics.CONFIG.backupInterval > 0) ServerTickEvents.END_SERVER_TICK.register(Statics.scheduler::tick); //Restart Executor Service in singleplayer ServerLifecycleEvents.SERVER_STARTING.register(ignored -> { diff --git a/src/main/java/net/szum123321/textile_backup/commands/FileSuggestionProvider.java b/src/main/java/net/szum123321/textile_backup/commands/FileSuggestionProvider.java index eb2534f..7f6851f 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/FileSuggestionProvider.java +++ b/src/main/java/net/szum123321/textile_backup/commands/FileSuggestionProvider.java @@ -28,8 +28,9 @@ import net.minecraft.entity.player.PlayerEntity; import net.minecraft.server.command.ServerCommandSource; import net.szum123321.textile_backup.Statics; import net.szum123321.textile_backup.core.restore.RestoreHelper; -import org.lwjgl.system.CallbackI; +import java.util.Collections; +import java.util.List; import java.util.concurrent.CompletableFuture; public final class FileSuggestionProvider implements SuggestionProvider { @@ -43,22 +44,22 @@ public final class FileSuggestionProvider implements SuggestionProvider getSuggestions(CommandContext ctx, SuggestionsBuilder builder) throws CommandSyntaxException { String remaining = builder.getRemaining(); - for (RestoreHelper.RestoreableFile file : RestoreHelper.getAvailableBackups(ctx.getSource().getMinecraftServer())) { + List list = RestoreHelper.getAvailableBackups(ctx.getSource().getMinecraftServer()); + + Collections.sort(list); + Collections.reverse(list); + + for (RestoreHelper.RestoreableFile file : list) { String formattedCreationTime = file.getCreationTime().format(Statics.defaultDateTimeFormatter); if (formattedCreationTime.startsWith(remaining)) { - if (ctx.getSource().getEntity() instanceof PlayerEntity) { //was typed by player - if (file.getComment() != null) { + if (file.getComment() != null) { + if(ctx.getSource().getEntity() instanceof PlayerEntity) builder.suggest(formattedCreationTime, new LiteralMessage("Comment: " + file.getComment())); - } else { - builder.suggest(formattedCreationTime); - } - } else { //was typed from server console - if (file.getComment() != null) { + else builder.suggest(file.getCreationTime() + "#" + file.getComment()); - } else { - builder.suggest(formattedCreationTime); - } + } else { + builder.suggest(formattedCreationTime); } } } diff --git a/src/main/java/net/szum123321/textile_backup/commands/restore/RestoreBackupCommand.java b/src/main/java/net/szum123321/textile_backup/commands/restore/RestoreBackupCommand.java index ad516f3..c57c70c 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/restore/RestoreBackupCommand.java +++ b/src/main/java/net/szum123321/textile_backup/commands/restore/RestoreBackupCommand.java @@ -66,7 +66,7 @@ public class RestoreBackupCommand { } private static int execute(String file, @Nullable String comment, ServerCommandSource source) throws CommandSyntaxException { - if(Statics.restoreAwaitThread == null || (Statics.restoreAwaitThread != null && !Statics.restoreAwaitThread.isAlive())) { + if(Statics.restoreAwaitThread == null || !Statics.restoreAwaitThread.isAlive()) { LocalDateTime dateTime; try { diff --git a/src/main/java/net/szum123321/textile_backup/core/Utilities.java b/src/main/java/net/szum123321/textile_backup/core/Utilities.java index 5a72687..3ec2e51 100644 --- a/src/main/java/net/szum123321/textile_backup/core/Utilities.java +++ b/src/main/java/net/szum123321/textile_backup/core/Utilities.java @@ -45,7 +45,7 @@ public class Utilities { public static File getWorldFolder(MinecraftServer server) { return ((MinecraftServerSessionAccessor)server) .getSession() - .getWorldDirectory(RegistryKey.of(Registry.DIMENSION, DimensionType.OVERWORLD_REGISTRY_KEY.getValue())); + .getWorldDirectory(RegistryKey.of(Registry.WORLD_KEY, DimensionType.OVERWORLD_REGISTRY_KEY.getValue())); } public static File getBackupRootPath(String worldName) { diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index 0231a06..49fefde 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -26,8 +26,6 @@ import net.szum123321.textile_backup.core.create.compressors.tar.AbstractTarArch import net.szum123321.textile_backup.core.create.compressors.tar.LZMACompressor; import net.szum123321.textile_backup.core.create.compressors.tar.ParallelBZip2Compressor; import net.szum123321.textile_backup.core.create.compressors.tar.ParallelGzipCompressor; -import net.szum123321.textile_backup.core.create.compressors.ParallelZipCompressor; - import java.io.File; import java.io.IOException; import java.io.OutputStream; diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java index dc6fd84..77611a0 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java @@ -20,13 +20,19 @@ package net.szum123321.textile_backup.core.create.compressors; import net.szum123321.textile_backup.Statics; import net.szum123321.textile_backup.core.create.BackupContext; +import net.szum123321.textile_backup.core.create.compressors.ParallelZipCompressor.FileInputStreamSupplier; +import net.szum123321.textile_backup.core.create.compressors.parallel_zip_fix.FailsafeScatterGatherBackingStore; import org.apache.commons.compress.archivers.zip.*; import org.apache.commons.compress.parallel.InputStreamSupplier; +import org.apache.commons.compress.parallel.ScatterGatherBackingStore; +import org.apache.commons.compress.parallel.ScatterGatherBackingStoreSupplier; +import sun.security.action.GetPropertyAction; import java.io.*; -import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.concurrent.*; -import java.util.zip.CRC32; +import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.ZipEntry; /* @@ -37,14 +43,17 @@ import java.util.zip.ZipEntry; */ public class ParallelZipCompressor extends ZipCompressor { private ParallelScatterZipCreator scatterZipCreator; + private ScatterZipOutputStream dirs; public static ParallelZipCompressor getInstance() { return new ParallelZipCompressor(); } @Override - protected OutputStream createArchiveOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) { - scatterZipCreator = new ParallelScatterZipCreator(Executors.newFixedThreadPool(coreLimit)); + protected OutputStream createArchiveOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) throws IOException { + dirs = ScatterZipOutputStream.fileBased(File.createTempFile("scatter-dirs", "tmp")); + scatterZipCreator = new ParallelScatterZipCreator(Executors.newFixedThreadPool(coreLimit), new CatchingBackingStoreSupplier()); + return super.createArchiveOutputStream(stream, ctx, coreLimit); } @@ -52,20 +61,25 @@ public class ParallelZipCompressor extends ZipCompressor { protected void addEntry(File file, String entryName, OutputStream arc) throws IOException { ZipArchiveEntry entry = (ZipArchiveEntry)((ZipArchiveOutputStream)arc).createArchiveEntry(file, entryName); - if(ZipCompressor.isDotDat(file.getName())) { - entry.setMethod(ZipArchiveOutputStream.STORED); - entry.setSize(file.length()); - entry.setCompressedSize(file.length()); - entry.setCrc(getCRC(file)); - } else entry.setMethod(ZipEntry.DEFLATED); + if(entry.isDirectory() && !entry.isUnixSymlink()) { + dirs.addArchiveEntry( + ZipArchiveEntryRequest.createZipArchiveEntryRequest(entry, new FileInputStreamSupplier(file)) + ); + } else { + if (ZipCompressor.isDotDat(file.getName())) { + entry.setMethod(ZipArchiveOutputStream.STORED); + entry.setCompressedSize(entry.getSize()); + entry.setCrc(getCRC(file)); + } else entry.setMethod(ZipEntry.DEFLATED); - entry.setTime(System.currentTimeMillis()); - - scatterZipCreator.addArchiveEntry(entry, new FileInputStreamSupplier(file)); + scatterZipCreator.addArchiveEntry(entry, new FileInputStreamSupplier(file)); + } } @Override protected void finish(OutputStream arc) throws InterruptedException, ExecutionException, IOException { + dirs.writeTo((ZipArchiveOutputStream) arc); + dirs.close(); scatterZipCreator.writeTo((ZipArchiveOutputStream) arc); } @@ -86,4 +100,14 @@ public class ParallelZipCompressor extends ZipCompressor { return null; } } + + private static class CatchingBackingStoreSupplier implements ScatterGatherBackingStoreSupplier { + final AtomicInteger storeNum = new AtomicInteger(0); + + @Override + public ScatterGatherBackingStore get() throws IOException { + //final File tempFile = File.createTempFile("catchngparallelscatter", "n" + storeNum.incrementAndGet()); + return new FailsafeScatterGatherBackingStore(storeNum.incrementAndGet(), Paths.get(GetPropertyAction.privilegedGetProperty("java.io.tmpdir"))); + } + } } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/CompositeScatterGatherBackingStore.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/CompositeScatterGatherBackingStore.java new file mode 100644 index 0000000..44f7deb --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/CompositeScatterGatherBackingStore.java @@ -0,0 +1,86 @@ +package net.szum123321.textile_backup.core.create.compressors.parallel_zip_fix; + +import org.apache.commons.compress.parallel.ScatterGatherBackingStore; +import sun.security.action.GetPropertyAction; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.atomic.AtomicInteger; + +public class CompositeScatterGatherBackingStore implements ScatterGatherBackingStore { + private final static String NO_SPACE_LEFT_ON_DEVICE_EXCEPTION_MESSAGE = "No space left on device"; + private final static Path mainTmpPath = Paths.get(GetPropertyAction.privilegedGetProperty("java.io.tmpdir")); + + private final static AtomicInteger TMP_FILE_COUNTER = new AtomicInteger(0); + private static Path localTmpPath; + + private final File mainTarget; + private long mainBytesWritten = 0; + private File localTarget = null; + private OutputStream os; + + public CompositeScatterGatherBackingStore(Path localTmpPath) throws IOException { + this.localTmpPath = localTmpPath; + + mainTarget = mainTmpPath.resolve("scatter_storage_" + TMP_FILE_COUNTER.getAndIncrement()).toFile(); + mainTarget.createNewFile(); + //mainTmpFile.deleteOnExit(); + + os = Files.newOutputStream(mainTarget.toPath()); + } + + @Override + public InputStream getInputStream() throws IOException { + if(localTarget == null) + return new SequenceInputStream( + new SizeLimitedInputStream((int) mainBytesWritten, Files.newInputStream(mainTarget.toPath())), + Files.newInputStream(localTarget.toPath()) + ); + + return Files.newInputStream(mainTarget.toPath()); + } + + @Override + public void writeOut(byte[] data, int offset, int length) throws IOException { + try { + os.write(data, offset, length); + if(localTarget == null) mainBytesWritten += length; + } catch (IOException e) { + if(e.getMessage().equals(NO_SPACE_LEFT_ON_DEVICE_EXCEPTION_MESSAGE)) { + if(localTarget == null) { + os.close(); + + localTarget = localTmpPath.resolve(mainTarget.getName()).toFile(); + localTarget.createNewFile(); + //localTmpFile.deleteOnExit(); + os = Files.newOutputStream(localTarget.toPath()); + os.write(data, offset, length); + } + } else { + throw e; + } + } + } + + @Override + public void closeForWriting() throws IOException { + os.close(); + } + + @Override + public void close() throws IOException { + if(mainTarget.exists() && !mainTarget.delete()) mainTarget.deleteOnExit(); + if(localTarget != null && localTarget.exists() && !localTarget.delete()) localTarget.deleteOnExit(); + } + /* + public static void setMemoryStorageSize(long size) { + MAX_MEMORY_STORAGE_SIZE = size; + } + + public static void resetMemoryStorage() { + MEMORY_SPACE_USED.set(MAX_MEMORY_STORAGE_SIZE); + } + */ +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/FailsafeScatterGatherBackingStore.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/FailsafeScatterGatherBackingStore.java new file mode 100644 index 0000000..9b455a6 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/FailsafeScatterGatherBackingStore.java @@ -0,0 +1,205 @@ +package net.szum123321.textile_backup.core.create.compressors.parallel_zip_fix; + +import org.apache.commons.compress.parallel.ScatterGatherBackingStore; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +/** + * So the main issue with the {@link org.apache.commons.compress.parallel.FileBasedScatterGatherBackingStore} is that it + * stores its results as files in tmpfs. In most cases it a good thing, as it allows for low system memory usage. + * Sadly some Minecraft Server Providers limit the size of the folder which causes software to fail. + * + * This {@link ScatterGatherBackingStore } implementation should overcome this issue by storing data in tmp files, and if that fails, + * it will switch to {@link MemoryBlockOutputStream}. This creates another issue as the system might run out of memory if too much data would be stored. + */ + +public class FailsafeScatterGatherBackingStore implements ScatterGatherBackingStore { + private final static String NO_SPACE_LEFT_ON_DEVICE_EXCEPTION_MESSAGE = "No space left on device"; + + private final Path tmpdir; + private final int id; + private final Deque> queue; + private int fileCounter; + private OutputStream os; + private boolean closed; + + public FailsafeScatterGatherBackingStore(int id, Path tmpdir) throws IOException { + this.tmpdir = tmpdir; + this.id = id; + queue = new ArrayDeque<>(); + //this.target = File.createTempFile("parallelscaterstore", String.valueOf(id), tmpdir.toFile()); + + if(!tryAddingNewFileToQueue()) { + queue.add(new MemoryBasedDataChunk()); + os = (OutputStream) queue.peek().getSource(); + } else { + os = Files.newOutputStream(((File)queue.peek().getSource()).toPath()); + } + + /*try { + os = Files.newOutputStream(target.toPath()); + } catch (IOException ex) { + if(ex.getMessage().equals(NO_SPACE_LEFT_ON_DEVICE_EXCEPTION_MESSAGE )) { + //Caught it! + state = State.Memory; + os = new MemoryBlockOutputStream(); + target.delete(); + } else { + //No need to stay backwards-compatible with Compress 1.13 + throw ex; + } + }*/ + } + + private boolean tryAddingNewFileToQueue() throws IOException { + try { + queue.add(new FileBasedDataChunk(File.createTempFile("parallescatterstore-" + id, String.valueOf(fileCounter++)))); + } catch (IOException e) { + if(e.getMessage().equals(NO_SPACE_LEFT_ON_DEVICE_EXCEPTION_MESSAGE)) { + return false; + } else { + throw e; + } + } + + return true; + } + + @Override + public InputStream getInputStream() throws IOException { + ArrayList list = new ArrayList<>(queue.size()); + for(DataChunk dataChunk: queue) list.add(dataChunk.getInputStream()); + return new SequenceInputStream(Collections.enumeration(list)); + /*if(state == State.MemoryBackup) { + return new SequenceInputStream( + new SizeLimitedInputStream(safelyWritten, Files.newInputStream(target.toPath())), + ((MemoryBlockOutputStream)os).getInputStream() + ); + } else if(state == State.Memory) { + return ((MemoryBlockOutputStream)os).getInputStream(); + } else { + return Files.newInputStream(target.toPath()); + }*/ + } + + @Override + public void writeOut(byte[] data, int offset, int length) throws IOException { + try { + os.write(data, offset, length); + queue.peekLast().size += length; + } catch (IOException e) { + if(e.getMessage().equals(NO_SPACE_LEFT_ON_DEVICE_EXCEPTION_MESSAGE)) { + //Caught it! + queue.add(new MemoryBasedDataChunk()); + os = (OutputStream) queue.peek().getSource(); + } else { + throw e; + } + } + /*try { + os.write(data, offset, length); + safelyWritten += length; + } catch (IOException e) { + if(e.getMessage().equals(NO_SPACE_LEFT_ON_DEVICE_EXCEPTION_MESSAGE )) { + //Caught it! + state = State.MemoryBackup; + os.close(); + os = new MemoryBlockOutputStream(); + os.write(data, offset, length); + } else { + throw e; + } + }*/ + } + + @Override + public void closeForWriting() throws IOException { + if (!closed) { + os.close(); + closed = true; + } + } + + @Override + public void close() throws IOException { + try { + closeForWriting(); + } finally { + queue.stream() + .filter(dataChunk -> dataChunk instanceof FileBasedDataChunk) + .map(dataChunk -> (File)dataChunk.getSource()) + .filter(file -> file.exists() && !file.delete()) + .forEach(File::deleteOnExit); + } + } + + private static abstract class DataChunk { + private long size; + + public DataChunk() { + this.size = 0; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + public abstract T getSource(); + + public abstract InputStream getInputStream() throws IOException; + public abstract OutputStream getOutputStream() throws IOException; + } + + private static class FileBasedDataChunk extends DataChunk { + private final File file; + + public FileBasedDataChunk(File file) { + this.file = file; + } + + @Override + public File getSource() { + return file; + } + + @Override + public InputStream getInputStream() throws IOException { + return Files.newInputStream(file.toPath()); + } + + @Override + public OutputStream getOutputStream() throws IOException { + return Files.newOutputStream(file.toPath()); + } + } + + private static class MemoryBasedDataChunk extends DataChunk { + private final MemoryBlockOutputStream memoryBlockOutputStream; + + public MemoryBasedDataChunk() { + memoryBlockOutputStream = new MemoryBlockOutputStream(); + } + + @Override + public MemoryBlockOutputStream getSource() { + return memoryBlockOutputStream; + } + + @Override + public InputStream getInputStream() { + return memoryBlockOutputStream.getInputStream(); + } + + @Override + public OutputStream getOutputStream() { + return memoryBlockOutputStream; + } + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/MemoryBlockOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/MemoryBlockOutputStream.java new file mode 100644 index 0000000..d6719e1 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/MemoryBlockOutputStream.java @@ -0,0 +1,123 @@ +package net.szum123321.textile_backup.core.create.compressors.parallel_zip_fix; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayDeque; +import java.util.Deque; + +/** + * Somewhat similar to ByteArrayOutputStream, except the data is stored in variable-size blocks. + * Blocks are created to be at least {@link MemoryBlockOutputStream#MIN_BLOCK_SIZE} in size. + * It is to limit object overhead + */ +public class MemoryBlockOutputStream extends OutputStream { + private static final int MIN_BLOCK_SIZE = 65536; //64K + private final Deque blockQueue; + + public MemoryBlockOutputStream() { + this.blockQueue = new ArrayDeque<>(); + } + + @Override + public void write(int b) throws IOException { + this.write(new byte[] {(byte)(b & 0xFF)}, 0, 1); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + while(len > 0) { + if(blockQueue.isEmpty() || blockQueue.peekLast().full()) blockQueue.add(new DataBlock(len)); + + //assert blockQueue.peekLast() != null; + int written = blockQueue.peekLast().write(b, off, len); + off += written; + len -= written; + } + } + + /** + * Warning! Returned InputStream will DESTROY data stored in the queue! + * @return {@link InputStream} to read data stored in queue buffer + */ + public InputStream getInputStream() { + return new InMemoryInputStream(blockQueue); + } + + private static class DataBlock { + private final byte[] block; + private final int size; + private int written = 0; + private int read = 0; + + public DataBlock(int size) { + this.size = Math.max(size, MIN_BLOCK_SIZE); + this.block = new byte[this.size]; + } + + public boolean full() { + return written == size; + } + + public boolean dataLeft() { + return read < size; + } + + public int write(byte[] b, int off, int len) { + int tbw = Math.min(len, size - written); + + System.arraycopy(b, off, block, written, tbw); + written += tbw; + return tbw; + } + + public int read(byte[] b, int off, int len) { + //if(!dataLeft()) return -1; + int tbr = Math.min(len, written - read); + + System.arraycopy(block, read, b, off, tbr); + read += tbr; + return tbr; + } + + public byte[] getBlock() { + return block; + } + } + + private static class InMemoryInputStream extends InputStream { + private final Deque blockQueue; + + public InMemoryInputStream(Deque blockQueue) { + this.blockQueue = blockQueue; + } + + @Override + public int read() { + byte[] buff = new byte[1]; + return (this.read(buff, 0, 1) == -1) ? -1 : buff[0]; + } + + @Override + public int read(byte[] b, int off, int len) { + if(blockQueue.isEmpty()) return -1; + + int totalRead = 0; + + while(len > 0 && !blockQueue.isEmpty()) { + if(!blockQueue.peek().dataLeft()) { + blockQueue.poll(); + continue; + } + + int read = blockQueue.peek().read(b, off, len); + + off += read; + len -= read; + totalRead += read; + } + + return totalRead; + } + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/SizeLimitedInputStream.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/SizeLimitedInputStream.java new file mode 100644 index 0000000..40fa252 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/parallel_zip_fix/SizeLimitedInputStream.java @@ -0,0 +1,41 @@ +package net.szum123321.textile_backup.core.create.compressors.parallel_zip_fix; + +import org.jetbrains.annotations.NotNull; + +import java.io.*; + +public class SizeLimitedInputStream extends FilterInputStream { + //private final int maxSize; + private int dataLeft; + + public SizeLimitedInputStream(int maxSize, InputStream inputStream) { + super(inputStream); + //this.maxSize = maxSize; + this.dataLeft = maxSize; + } + + @Override + public int read() throws IOException { + if(dataLeft == 0) return -1; + int read = super.read(); + + if(read != -1) dataLeft--; + return read; + } + + @Override + public int available() throws IOException { + return Math.min(dataLeft, super.available()); + } + + @Override + public int read(@NotNull byte[] b, int off, int len) throws IOException { + if(dataLeft == 0) return -1; + + int read = super.read(b, off, Math.min(dataLeft, len)); + + if(read != -1) dataLeft -= read; + + return read; + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index c9d71f3..7898b68 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -67,7 +67,11 @@ public class RestoreBackupRunnable implements Runnable { Set undeleted = deleteDirectory(worldFile); if(!undeleted.isEmpty()) { - Statics.LOGGER.error("Failed to delete {} files:\n {}",undeleted.size(), Arrays.toString(undeleted.toArray())); + Statics.LOGGER.error("Failed to delete {} file{}:\n {}", + undeleted.size(), + undeleted.size() > 1 ? "s" : "", + Arrays.toString(undeleted.toArray()) + ); } worldFile.mkdirs(); diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java index 1a5a19d..4440893 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java @@ -63,12 +63,6 @@ public class GenericTarDecompressor { } catch (IOException e) { Statics.LOGGER.error("An exception occurred when trying to create {}", file, e); } - /* - if (!parent.isDirectory() && !parent.mkdirs()) { - Statics.LOGGER.error("Failed to create {}", parent); - Statics.LOGGER.error("Skipping: {}", file); - continue; - }*/ try (OutputStream outputStream = Files.newOutputStream(file.toPath()); BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outputStream)) { @@ -85,6 +79,12 @@ public class GenericTarDecompressor { Statics.LOGGER.info("Decompression took {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now()))); } + /** + * This function handles uncompressed (.tar) streams + * @param inputStream File input stream + * @return Either { @link CompressorInputStream } that decompresses the file or the inputStream if it's tar + * @throws CompressorException when the file is neither a tar or other supported archive + */ private static InputStream getCompressorInputStream(InputStream inputStream) throws CompressorException { try { return new CompressorStreamFactory().createCompressorInputStream(inputStream); diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java index 14ef543..13f2009 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java @@ -21,25 +21,23 @@ package net.szum123321.textile_backup.core.restore.decompressors; import net.szum123321.textile_backup.Statics; import net.szum123321.textile_backup.core.Utilities; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipFile; import org.apache.commons.compress.utils.IOUtils; import java.io.*; import java.nio.file.Files; import java.time.Duration; import java.time.Instant; +import java.util.Enumeration; public class ZipDecompressor { public static void decompress(File inputFile, File target) { Instant start = Instant.now(); - try (FileInputStream fileInputStream = new FileInputStream(inputFile); - BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream); - ZipArchiveInputStream zipInputStream = new ZipArchiveInputStream((bufferedInputStream))) { - ZipArchiveEntry entry; - - while ((entry = zipInputStream.getNextZipEntry()) != null) { - if(!zipInputStream.canReadEntryData(entry)){ + try (ZipFile zipFile = new ZipFile(inputFile)) { + for(Enumeration enumeration = zipFile.getEntries(); enumeration.hasMoreElements();) { + ZipArchiveEntry entry = enumeration.nextElement(); + if(!zipFile.canReadEntryData(entry)) { Statics.LOGGER.error("Something when wrong while trying to decompress {}", entry.getName()); continue; } @@ -52,8 +50,6 @@ public class ZipDecompressor { } catch (IOException e) { Statics.LOGGER.error("An exception occurred when trying to create {}", file, e); } - //if(!file.isDirectory() && !file.mkdirs()) - // Statics.LOGGER.error("Failed to create: {}", file); } else { File parent = file.getParentFile(); @@ -65,7 +61,7 @@ public class ZipDecompressor { try (OutputStream outputStream = Files.newOutputStream(file.toPath()); BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outputStream)) { - IOUtils.copy(zipInputStream, bufferedOutputStream); + IOUtils.copy(zipFile.getInputStream(entry), bufferedOutputStream); } catch (IOException e) { Statics.LOGGER.error("An exception occurred while trying to decompress file: {}", file.getName(), e); } diff --git a/src/main/java/net/szum123321/textile_backup/mixin/MinecraftServerSessionAccessor.java b/src/main/java/net/szum123321/textile_backup/mixin/MinecraftServerSessionAccessor.java deleted file mode 100644 index 047c1fb..0000000 --- a/src/main/java/net/szum123321/textile_backup/mixin/MinecraftServerSessionAccessor.java +++ /dev/null @@ -1,12 +0,0 @@ -package net.szum123321.textile_backup.mixin; - -import net.minecraft.server.MinecraftServer; -import net.minecraft.world.level.storage.LevelStorage; -import org.spongepowered.asm.mixin.Mixin; -import org.spongepowered.asm.mixin.gen.Accessor; - -@Mixin(MinecraftServer.class) -public interface MinecraftServerSessionAccessor { - @Accessor - LevelStorage.Session getSession(); -}