diff --git a/core/src/main/java/com/boydti/fawe/config/BBC.java b/core/src/main/java/com/boydti/fawe/config/BBC.java index 981a6767..1331be7d 100644 --- a/core/src/main/java/com/boydti/fawe/config/BBC.java +++ b/core/src/main/java/com/boydti/fawe/config/BBC.java @@ -26,25 +26,25 @@ public enum BBC { SCHEMATIC_PASTING("&7The schematic is pasting. This cannot be undone.", "Info"), LIGHTING_PROPOGATE_SELECTION("&7Lighting has been propogated in %s0 chunks. (Note: To remove light use //removelight)", "Info"), UPDATED_LIGHTING_SELECTION("&7Lighting has been updated in %s0 chunks. (It may take a second for the packets to send)", "Info"), - SET_REGION("&7Selection set to your current WorldEdit region", "Info"), + SET_REGION("&7Selection set to your current allowed region", "Info"), WORLDEDIT_COMMAND_LIMIT("&7Please wait until your current action completes", "Info"), - WORLDEDIT_DELAYED("&7Please wait while we process your WorldEdit action...", "Info"), + WORLDEDIT_DELAYED("&7Please wait while we process your FAWE action...", "Info"), WORLDEDIT_RUN("&7Apologies for the delay. Now executing: %s", "Info"), - WORLDEDIT_COMPLETE("&7WorldEdit action completed.", "Info"), + WORLDEDIT_COMPLETE("&7Edit completed.", "Info"), REQUIRE_SELECTION_IN_MASK("&7%s of your selection is not within your mask. You can only make edits within allowed regions.", "Info"), WORLDEDIT_VOLUME("&7You cannot select a volume of %current%. The maximum volume you can modify is %max%.", "Info"), WORLDEDIT_ITERATIONS("&7You cannot iterate %current% times. The maximum number of iterations allowed is %max%.", "Info"), WORLDEDIT_UNSAFE("&7Access to that command has been blocked", "Info"), - WORLDEDIT_DANGEROUS_WORLDEDIT("&cProcessed unsafe WorldEdit at %s0 by %s1", "Info"), + WORLDEDIT_DANGEROUS_WORLDEDIT("&cProcessed unsafe edit at %s0 by %s1", "Info"), WORLDEDIT_BYPASS("&7&oTo bypass your restrictions use &c/wea", "Info"), - WORLDEDIT_EXTEND("&cYour WorldEdit may have extended outside your allowed region.", "Error"), - WORLDEDIT_TOGGLE_TIPS_ON("&7Disabled WorldEdit tips.", "Info"), - WORLDEDIT_TOGGLE_TIPS_OFF("&7Enabled WorldEdit tips.", "Info"), + WORLDEDIT_EXTEND("&cYour edit may have extended outside your allowed region.", "Error"), + WORLDEDIT_TOGGLE_TIPS_ON("&7Disabled FAWE tips.", "Info"), + WORLDEDIT_TOGGLE_TIPS_OFF("&7Enabled FAWE tips.", "Info"), - WORLDEDIT_BYPASSED("&7Currently bypassing WorldEdit restriction.", "Info"), - WORLDEDIT_UNMASKED("&6Your WorldEdit is now unrestricted.", "Info"), + WORLDEDIT_BYPASSED("&7Currently bypassing FAWE restriction.", "Info"), + WORLDEDIT_UNMASKED("&6Your FAWE edits are now unrestricted.", "Info"), - WORLDEDIT_RESTRICTED("&6Your WorldEdit is now restricted.", "Info"), + WORLDEDIT_RESTRICTED("&6Your FAWE edits are now restricted.", "Info"), WORLDEDIT_OOM_ADMIN("&cPossible options:\n&8 - &7//fast\n&8 - &7Do smaller edits\n&8 - &7Allocate more memory\n&8 - &7Disable `max-memory-percent`", "Info"), COMPRESSED("History compressed. Saved ~ %s0b (%s1x smaller)", "Info"), @@ -231,7 +231,7 @@ public enum BBC { NO_PERM("&cYou are lacking the permission node: %s0", "Error"), SETTING_DISABLE("&cLacking setting: %s0","Error"), SCHEMATIC_NOT_FOUND("&cSchematic not found: &7%s0", "Error"), - NO_REGION("&cYou have no current WorldEdit region", "Error"), + NO_REGION("&cYou have no current allowed region", "Error"), NO_MASK("&cYou have no current mask set", "Error"), NOT_PLAYER("&cYou must be a player to perform this action!", "Error"), PLAYER_NOT_FOUND("&cPlayer not found:&7 %s0", "Error"), diff --git a/core/src/main/java/com/boydti/fawe/util/MainUtil.java b/core/src/main/java/com/boydti/fawe/util/MainUtil.java index 6922ae5b..2f76a434 100644 --- a/core/src/main/java/com/boydti/fawe/util/MainUtil.java +++ b/core/src/main/java/com/boydti/fawe/util/MainUtil.java @@ -65,11 +65,12 @@ import java.util.zip.GZIPInputStream; import java.util.zip.Inflater; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import net.jpountz.lz4.LZ4BlockInputStream; +import net.jpountz.lz4.LZ4BlockOutputStream; import net.jpountz.lz4.LZ4Compressor; import net.jpountz.lz4.LZ4Factory; import net.jpountz.lz4.LZ4FastDecompressor; import net.jpountz.lz4.LZ4InputStream; -import net.jpountz.lz4.LZ4OutputStream; import net.jpountz.lz4.LZ4Utils; public class MainUtil { @@ -311,7 +312,7 @@ public class MainUtil { } public static FaweOutputStream getCompressedOS(OutputStream os, int amount, int buffer) throws IOException { - os.write((byte) -amount); + os.write((byte) 9 + amount); os = new BufferedOutputStream(os, buffer); if (amount == 0) { return new FaweOutputStream(os); @@ -324,14 +325,14 @@ public class MainUtil { LZ4Factory factory = LZ4Factory.fastestInstance(); int fastAmount = 1 + ((amount - 1) % 3); for (int i = 0; i < fastAmount; i++) { - os = new LZ4OutputStream(os, buffer, factory.fastCompressor()); + os = new LZ4BlockOutputStream(os, buffer, factory.fastCompressor()); } int highAmount = amount > 3 ? 1 : 0; for (int i = 0; i < highAmount; i++) { if (amount == 9) { - os = new LZ4OutputStream(os, buffer, factory.highCompressor(17)); + os = new LZ4BlockOutputStream(os, buffer, factory.highCompressor(17)); } else { - os = new LZ4OutputStream(os, buffer, factory.highCompressor()); + os = new LZ4BlockOutputStream(os, buffer, factory.highCompressor()); } } return new FaweOutputStream(os); @@ -342,14 +343,21 @@ public class MainUtil { } public static FaweInputStream getCompressedIS(InputStream is, int buffer) throws IOException { - int amount = (byte) is.read(); + int mode = (byte) is.read(); is = new BufferedInputStream(is, buffer); - if (amount == 0) { + if (mode == 0) { return new FaweInputStream(is); } - int amountAbs = Math.abs(amount); + boolean legacy; + if (mode > 9) { + legacy = false; + mode = -mode + 9; + } else { + legacy = true; + } + int amountAbs = Math.abs(mode); if (amountAbs > 6) { - if (amount > 0) { + if (mode > 0) { is = new BufferedInputStream(new GZIPInputStream(is, buffer)); } else { is = new ZstdInputStream(is); @@ -357,7 +365,11 @@ public class MainUtil { } amountAbs = (1 + ((amountAbs - 1) % 3)) + (amountAbs > 3 ? 1 : 0); for (int i = 0; i < amountAbs; i++) { - is = new LZ4InputStream(is); + if (legacy) { + is = new LZ4InputStream(is); + } else { + is = new LZ4BlockInputStream(is); + } } return new FaweInputStream(is); } diff --git a/core/src/main/java/com/sk89q/worldedit/command/ChunkCommands.java b/core/src/main/java/com/sk89q/worldedit/command/ChunkCommands.java index 3c901197..a212af46 100644 --- a/core/src/main/java/com/sk89q/worldedit/command/ChunkCommands.java +++ b/core/src/main/java/com/sk89q/worldedit/command/ChunkCommands.java @@ -94,7 +94,7 @@ public class ChunkCommands { @Command( aliases = { "delchunks" }, usage = "", - desc = "Delete chunks that your selection includes", + desc = "Deprecated, use anvil commands", min = 0, max = 0 ) @@ -114,7 +114,7 @@ public class ChunkCommands { out = new FileOutputStream("worldedit-delchunks.bat"); OutputStreamWriter writer = new OutputStreamWriter(out, "UTF-8"); writer.write("@ECHO off\r\n"); - writer.write("ECHO This batch file was generated by WorldEdit.\r\n"); + writer.write("ECHO This batch file was generated by FAWE.\r\n"); writer.write("ECHO It contains a list of chunks that were in the selected region\r\n"); writer.write("ECHO at the time that the /delchunks command was used. Run this file\r\n"); writer.write("ECHO in order to delete the chunk files listed in this file.\r\n"); @@ -145,7 +145,7 @@ public class ChunkCommands { out = new FileOutputStream("worldedit-delchunks.sh"); OutputStreamWriter writer = new OutputStreamWriter(out, "UTF-8"); writer.write("#!/bin/bash\n"); - writer.write("echo This shell file was generated by WorldEdit.\n"); + writer.write("echo This shell file was generated by FAWE.\n"); writer.write("echo It contains a list of chunks that were in the selected region\n"); writer.write("echo at the time that the /delchunks command was used. Run this file\n"); writer.write("echo in order to delete the chunk files listed in this file.\n"); diff --git a/core/src/main/java/com/sk89q/worldedit/command/GeneralCommands.java b/core/src/main/java/com/sk89q/worldedit/command/GeneralCommands.java index 17f4976a..38909c92 100644 --- a/core/src/main/java/com/sk89q/worldedit/command/GeneralCommands.java +++ b/core/src/main/java/com/sk89q/worldedit/command/GeneralCommands.java @@ -43,7 +43,7 @@ public class GeneralCommands { @Command( aliases = { "/tips", "tips" }, - desc = "Toggle WorldEdit tips" + desc = "Toggle FAWE tips" ) public void tips(Player player, LocalSession session) throws WorldEditException { FawePlayer fp = FawePlayer.wrap(player); @@ -57,7 +57,7 @@ public class GeneralCommands { @Command( aliases = { "/fast" }, usage = "[on|off]", - desc = "Toggles WorldEdit undo", + desc = "Toggles FAWE undo", min = 0, max = 1 ) diff --git a/core/src/main/java/com/sk89q/worldedit/command/SnapshotUtilCommands.java b/core/src/main/java/com/sk89q/worldedit/command/SnapshotUtilCommands.java index 591eff93..701e638b 100644 --- a/core/src/main/java/com/sk89q/worldedit/command/SnapshotUtilCommands.java +++ b/core/src/main/java/com/sk89q/worldedit/command/SnapshotUtilCommands.java @@ -95,10 +95,10 @@ public class SnapshotUtilCommands { File dir = config.snapshotRepo.getDirectory(); try { - logger.info("WorldEdit found no snapshots: looked in: " + logger.info("FAWE found no snapshots: looked in: " + dir.getCanonicalPath()); } catch (IOException e) { - logger.info("WorldEdit found no snapshots: looked in " + logger.info("FAWE found no snapshots: looked in " + "(NON-RESOLVABLE PATH - does it exist?): " + dir.getPath()); } diff --git a/core/src/main/java/com/sk89q/worldedit/command/WorldEditCommands.java b/core/src/main/java/com/sk89q/worldedit/command/WorldEditCommands.java index c6fdf48c..e980f3f5 100644 --- a/core/src/main/java/com/sk89q/worldedit/command/WorldEditCommands.java +++ b/core/src/main/java/com/sk89q/worldedit/command/WorldEditCommands.java @@ -66,22 +66,9 @@ public class WorldEditCommands { max = 0 ) public void version(Actor actor) throws WorldEditException { - actor.print(BBC.getPrefix() + "WorldEdit " + WorldEdit.getVersion()); - PlatformManager pm = we.getPlatformManager(); - actor.printDebug("------------------------------------"); - actor.printDebug("Platforms:"); - for (Platform platform : pm.getPlatforms()) { - actor.printDebug(String.format(" - %s (%s)", platform.getPlatformName(), platform.getPlatformVersion())); - } - actor.printDebug("Capabilities:"); - for (Capability capability : Capability.values()) { - Platform platform = pm.queryCapability(capability); - actor.printDebug(String.format(" - %s: %s", capability.name(), platform != null ? platform.getPlatformName() : "NONE")); - } - actor.printDebug("------------------------------------"); FaweVersion fVer = Fawe.get().getVersion(); String fVerStr = fVer == null ? "unknown" : fVer.year + "." + fVer.month + "." + fVer.day + "-" + Integer.toHexString(fVer.hash) + "-" + fVer.build; - actor.print(BBC.getPrefix() + "FAWE " + fVerStr); + actor.print(BBC.getPrefix() + "FAWE " + fVerStr + " by Empire92"); if (fVer != null) { actor.printDebug("------------------------------------"); FaweVersion version = Fawe.get().getVersion(); @@ -99,8 +86,21 @@ public class WorldEditCommands { actor.printDebug(" - UPDATES: Latest Version"); } actor.printDebug("------------------------------------"); - actor.printDebug("Wiki: " + "https://github.com/boy0001/FastAsyncWorldedit/wiki"); } + actor.print(BBC.getPrefix() + "WorldEdit " + WorldEdit.getVersion() + " by sk89q"); + PlatformManager pm = we.getPlatformManager(); + actor.printDebug("------------------------------------"); + actor.printDebug("Platforms:"); + for (Platform platform : pm.getPlatforms()) { + actor.printDebug(String.format(" - %s (%s)", platform.getPlatformName(), platform.getPlatformVersion())); + } + actor.printDebug("Capabilities:"); + for (Capability capability : Capability.values()) { + Platform platform = pm.queryCapability(capability); + actor.printDebug(String.format(" - %s: %s", capability.name(), platform != null ? platform.getPlatformName() : "NONE")); + } + actor.printDebug("------------------------------------"); + actor.printDebug("Wiki: " + "https://github.com/boy0001/FastAsyncWorldedit/wiki"); } @Command( @@ -202,7 +202,7 @@ public class WorldEditCommands { @Command( aliases = { "help" }, usage = "[]", - desc = "Displays help for WorldEdit commands", + desc = "Displays help for FAWE commands", min = 0, max = -1 ) diff --git a/core/src/main/java/com/sk89q/worldedit/extension/platform/CommandManager.java b/core/src/main/java/com/sk89q/worldedit/extension/platform/CommandManager.java index b3cb3791..3e13b485 100644 --- a/core/src/main/java/com/sk89q/worldedit/extension/platform/CommandManager.java +++ b/core/src/main/java/com/sk89q/worldedit/extension/platform/CommandManager.java @@ -236,7 +236,7 @@ public final class CommandManager { .registerMethods(new ToolCommands(worldEdit)) .registerMethods(new UtilityCommands(worldEdit)) .group("worldedit", "we", "fawe") - .describeAs("WorldEdit commands") + .describeAs("FAWE commands") .registerMethods(new WorldEditCommands(worldEdit)).parent().group("schematic", "schem", "/schematic", "/schem") .describeAs("Schematic commands for saving/loading areas") .registerMethods(new SchematicCommands(worldEdit)).parent().group("snapshot", "snap") diff --git a/core/src/main/java/net/jpountz/lz4/LZ4BlockInputStream.java b/core/src/main/java/net/jpountz/lz4/LZ4BlockInputStream.java new file mode 100644 index 00000000..89d7f94c --- /dev/null +++ b/core/src/main/java/net/jpountz/lz4/LZ4BlockInputStream.java @@ -0,0 +1,235 @@ +package net.jpountz.lz4; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.EOFException; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.Checksum; +import net.jpountz.util.SafeUtils; + + +import static net.jpountz.lz4.LZ4BlockOutputStream.*; + +/** + * {@link InputStream} implementation to decode data written with + * {@link LZ4BlockOutputStream}. This class is not thread-safe and does not + * support {@link #mark(int)}/{@link #reset()}. + * @see LZ4BlockOutputStream + */ +public final class LZ4BlockInputStream extends FilterInputStream { + + private final LZ4FastDecompressor decompressor; + private final Checksum checksum; + private byte[] buffer; + private byte[] compressedBuffer; + private int originalLen; + private int o; + private boolean finished; + + /** + * Create a new {@link InputStream}. + * + * @param in the {@link InputStream} to poll + * @param decompressor the {@link LZ4FastDecompressor decompressor} instance to + * use + * @param checksum the {@link Checksum} instance to use, must be + * equivalent to the instance which has been used to + * write the stream + */ + public LZ4BlockInputStream(InputStream in, LZ4FastDecompressor decompressor, Checksum checksum) { + super(in); + this.decompressor = decompressor; + this.checksum = checksum; + this.buffer = new byte[0]; + this.compressedBuffer = new byte[HEADER_LENGTH]; + o = originalLen = 0; + finished = false; + } + + public LZ4BlockInputStream(InputStream in, LZ4FastDecompressor decompressor) { + this(in, decompressor, null); + } + + /** + * Create a new instance which uses the fastest {@link LZ4FastDecompressor} available. + * @see LZ4Factory#fastestInstance() + * @see #LZ4BlockInputStream(InputStream, LZ4FastDecompressor) + */ + public LZ4BlockInputStream(InputStream in) { + this(in, LZ4Factory.fastestInstance().fastDecompressor()); + } + + @Override + public int available() throws IOException { + return originalLen - o; + } + + @Override + public int read() throws IOException { + if (finished) { + return -1; + } + if (o == originalLen) { + refill(); + } + if (finished) { + return -1; + } + return buffer[o++] & 0xFF; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + SafeUtils.checkRange(b, off, len); + if (finished) { + return -1; + } + if (o == originalLen) { + refill(); + } + if (finished) { + return -1; + } + len = Math.min(len, originalLen - o); + System.arraycopy(buffer, o, b, off, len); + o += len; + return len; + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public long skip(long n) throws IOException { + if (finished) { + return -1; + } + if (o == originalLen) { + refill(); + } + if (finished) { + return -1; + } + final int skipped = (int) Math.min(n, originalLen - o); + o += skipped; + return skipped; + } + + private void refill() throws IOException { + readFully(compressedBuffer, HEADER_LENGTH); + for (int i = 0; i < MAGIC_LENGTH; ++i) { + if (compressedBuffer[i] != MAGIC[i]) { + throw new IOException("Stream is corrupted"); + } + } + final int token = compressedBuffer[MAGIC_LENGTH] & 0xFF; + final int compressionMethod = token & 0xF0; + final int compressionLevel = COMPRESSION_LEVEL_BASE + (token & 0x0F); + if (compressionMethod != COMPRESSION_METHOD_RAW && compressionMethod != COMPRESSION_METHOD_LZ4) { + throw new IOException("Stream is corrupted"); + } + final int compressedLen = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 1); + originalLen = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 5); + final int check = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 9); + assert HEADER_LENGTH == MAGIC_LENGTH + 13; + if (originalLen > 1 << compressionLevel + || originalLen < 0 + || compressedLen < 0 + || (originalLen == 0 && compressedLen != 0) + || (originalLen != 0 && compressedLen == 0) + || (compressionMethod == COMPRESSION_METHOD_RAW && originalLen != compressedLen)) { + throw new IOException("Stream is corrupted"); + } + if (originalLen == 0 && compressedLen == 0) { + if (check != 0) { + throw new IOException("Stream is corrupted"); + } + finished = true; + return; + } + if (buffer.length < originalLen) { + buffer = new byte[Math.max(originalLen, buffer.length * 3 / 2)]; + } + switch (compressionMethod) { + case COMPRESSION_METHOD_RAW: + readFully(buffer, originalLen); + break; + case COMPRESSION_METHOD_LZ4: + if (compressedBuffer.length < originalLen) { + compressedBuffer = new byte[Math.max(compressedLen, compressedBuffer.length * 3 / 2)]; + } + readFully(compressedBuffer, compressedLen); + try { + final int compressedLen2 = decompressor.decompress(compressedBuffer, 0, buffer, 0, originalLen); + if (compressedLen != compressedLen2) { + throw new IOException("Stream is corrupted"); + } + } catch (LZ4Exception e) { + throw new IOException("Stream is corrupted", e); + } + break; + default: + throw new AssertionError(); + } + if (checksum != null) { + checksum.reset(); + checksum.update(buffer, 0, originalLen); + if ((int) checksum.getValue() != check) { + throw new IOException("Stream is corrupted"); + } + } + o = 0; + } + + private void readFully(byte[] b, int len) throws IOException { + int read = 0; + while (read < len) { + final int r = in.read(b, read, len - read); + if (r < 0) { + throw new EOFException("Stream ended prematurely"); + } + read += r; + } + assert len == read; + } + + @Override + public boolean markSupported() { + return false; + } + + @SuppressWarnings("sync-override") + @Override + public void mark(int readlimit) { + // unsupported + } + + @SuppressWarnings("sync-override") + @Override + public void reset() throws IOException { + throw new IOException("mark/reset not supported"); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(in=" + in + + ", decompressor=" + decompressor + ", checksum=" + checksum + ")"; + } + +} \ No newline at end of file diff --git a/core/src/main/java/net/jpountz/lz4/LZ4BlockOutputStream.java b/core/src/main/java/net/jpountz/lz4/LZ4BlockOutputStream.java new file mode 100644 index 00000000..ee36034b --- /dev/null +++ b/core/src/main/java/net/jpountz/lz4/LZ4BlockOutputStream.java @@ -0,0 +1,256 @@ +package net.jpountz.lz4; + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.zip.Checksum; + +import net.jpountz.util.SafeUtils; + +/** + * Streaming LZ4. + *

+ * This class compresses data into fixed-size blocks of compressed data. + * @see LZ4BlockInputStream + */ +public final class LZ4BlockOutputStream extends FilterOutputStream { + + static final byte[] MAGIC = new byte[] { 'L', 'Z', '4', 'B', 'l', 'o', 'c', 'k' }; + static final int MAGIC_LENGTH = MAGIC.length; + + static final int HEADER_LENGTH = + MAGIC_LENGTH // magic bytes + + 1 // token + + 4 // compressed length + + 4 // decompressed length + + 4; // checksum + + static final int COMPRESSION_LEVEL_BASE = 10; + static final int MIN_BLOCK_SIZE = 64; + static final int MAX_BLOCK_SIZE = 1 << (COMPRESSION_LEVEL_BASE + 0x0F); + + static final int COMPRESSION_METHOD_RAW = 0x10; + static final int COMPRESSION_METHOD_LZ4 = 0x20; + + static final int DEFAULT_SEED = 0x9747b28c; + + private static int compressionLevel(int blockSize) { + if (blockSize < MIN_BLOCK_SIZE) { + throw new IllegalArgumentException("blockSize must be >= " + MIN_BLOCK_SIZE + ", got " + blockSize); + } else if (blockSize > MAX_BLOCK_SIZE) { + throw new IllegalArgumentException("blockSize must be <= " + MAX_BLOCK_SIZE + ", got " + blockSize); + } + int compressionLevel = 32 - Integer.numberOfLeadingZeros(blockSize - 1); // ceil of log2 + assert (1 << compressionLevel) >= blockSize; + assert blockSize * 2 > (1 << compressionLevel); + compressionLevel = Math.max(0, compressionLevel - COMPRESSION_LEVEL_BASE); + assert compressionLevel >= 0 && compressionLevel <= 0x0F; + return compressionLevel; + } + + private final int blockSize; + private final int compressionLevel; + private final LZ4Compressor compressor; + private final Checksum checksum; + private final byte[] buffer; + private final byte[] compressedBuffer; + private final boolean syncFlush; + private boolean finished; + private int o; + + /** + * Create a new {@link OutputStream} with configurable block size. Large + * blocks require more memory at compression and decompression time but + * should improve the compression ratio. + * + * @param out the {@link OutputStream} to feed + * @param blockSize the maximum number of bytes to try to compress at once, + * must be >= 64 and <= 32 M + * @param compressor the {@link LZ4Compressor} instance to use to compress + * data + * @param checksum the {@link Checksum} instance to use to check data for + * integrity. + * @param syncFlush true if pending data should also be flushed on {@link #flush()} + */ + public LZ4BlockOutputStream(OutputStream out, int blockSize, LZ4Compressor compressor, Checksum checksum, boolean syncFlush) { + super(out); + this.blockSize = blockSize; + this.compressor = compressor; + this.checksum = checksum; + this.compressionLevel = compressionLevel(blockSize); + this.buffer = new byte[blockSize]; + final int compressedBlockSize = HEADER_LENGTH + compressor.maxCompressedLength(blockSize); + this.compressedBuffer = new byte[compressedBlockSize]; + this.syncFlush = syncFlush; + o = 0; + finished = false; + System.arraycopy(MAGIC, 0, compressedBuffer, 0, MAGIC_LENGTH); + } + + public LZ4BlockOutputStream(OutputStream out, int blockSize, LZ4Compressor compressor) { + this(out, blockSize, compressor, null, false); + } + + /** + * Create a new instance which compresses with the standard LZ4 compression + * algorithm. + * @see #LZ4BlockOutputStream(OutputStream, int, LZ4Compressor) + * @see LZ4Factory#fastCompressor() + */ + public LZ4BlockOutputStream(OutputStream out, int blockSize) { + this(out, blockSize, LZ4Factory.fastestInstance().fastCompressor()); + } + + /** + * Create a new instance which compresses into blocks of 64 KB. + * @see #LZ4BlockOutputStream(OutputStream, int) + */ + public LZ4BlockOutputStream(OutputStream out) { + this(out, 1 << 16); + } + + private void ensureNotFinished() { + if (finished) { + throw new IllegalStateException("This stream is already closed"); + } + } + + @Override + public void write(int b) throws IOException { + ensureNotFinished(); + if (o == blockSize) { + flushBufferedData(); + } + buffer[o++] = (byte) b; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + SafeUtils.checkRange(b, off, len); + ensureNotFinished(); + + while (o + len > blockSize) { + final int l = blockSize - o; + System.arraycopy(b, off, buffer, o, blockSize - o); + o = blockSize; + flushBufferedData(); + off += l; + len -= l; + } + System.arraycopy(b, off, buffer, o, len); + o += len; + } + + @Override + public void write(byte[] b) throws IOException { + ensureNotFinished(); + write(b, 0, b.length); + } + + @Override + public void close() throws IOException { + if (!finished) { + finish(); + } + if (out != null) { + out.close(); + out = null; + } + } + + private void flushBufferedData() throws IOException { + if (o == 0) { + return; + } + final int check; + if (checksum != null) { + checksum.reset(); + checksum.update(buffer, 0, o); + check = (int) checksum.getValue(); + } else { + check = 0; + } + int compressedLength = compressor.compress(buffer, 0, o, compressedBuffer, HEADER_LENGTH); + final int compressMethod; + if (compressedLength >= o) { + compressMethod = COMPRESSION_METHOD_RAW; + compressedLength = o; + System.arraycopy(buffer, 0, compressedBuffer, HEADER_LENGTH, o); + } else { + compressMethod = COMPRESSION_METHOD_LZ4; + } + + compressedBuffer[MAGIC_LENGTH] = (byte) (compressMethod | compressionLevel); + writeIntLE(compressedLength, compressedBuffer, MAGIC_LENGTH + 1); + writeIntLE(o, compressedBuffer, MAGIC_LENGTH + 5); + writeIntLE(check, compressedBuffer, MAGIC_LENGTH + 9); + assert MAGIC_LENGTH + 13 == HEADER_LENGTH; + out.write(compressedBuffer, 0, HEADER_LENGTH + compressedLength); + o = 0; + } + + /** + * Flush this compressed {@link OutputStream}. + * + * If the stream has been created with syncFlush=true, pending + * data will be compressed and appended to the underlying {@link OutputStream} + * before calling {@link OutputStream#flush()} on the underlying stream. + * Otherwise, this method just flushes the underlying stream, so pending + * data might not be available for reading until {@link #finish()} or + * {@link #close()} is called. + */ + @Override + public void flush() throws IOException { + if (out != null) { + if (syncFlush) { + flushBufferedData(); + } + out.flush(); + } + } + + /** + * Same as {@link #close()} except that it doesn't close the underlying stream. + * This can be useful if you want to keep on using the underlying stream. + */ + public void finish() throws IOException { + ensureNotFinished(); + flushBufferedData(); + compressedBuffer[MAGIC_LENGTH] = (byte) (COMPRESSION_METHOD_RAW | compressionLevel); + writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 1); + writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 5); + writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 9); + assert MAGIC_LENGTH + 13 == HEADER_LENGTH; + out.write(compressedBuffer, 0, HEADER_LENGTH); + finished = true; + out.flush(); + } + + private static void writeIntLE(int i, byte[] buf, int off) { + buf[off++] = (byte) i; + buf[off++] = (byte) (i >>> 8); + buf[off++] = (byte) (i >>> 16); + buf[off++] = (byte) (i >>> 24); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(out=" + out + ", blockSize=" + blockSize + + ", compressor=" + compressor + ", checksum=" + checksum + ")"; + } + +} \ No newline at end of file diff --git a/core/src/main/java/net/jpountz/lz4/LZ4InputStream.java b/core/src/main/java/net/jpountz/lz4/LZ4InputStream.java index b4e3e4dc..d6bb76c5 100644 --- a/core/src/main/java/net/jpountz/lz4/LZ4InputStream.java +++ b/core/src/main/java/net/jpountz/lz4/LZ4InputStream.java @@ -80,10 +80,6 @@ public class LZ4InputStream extends InputStream { return n - numBytesRemainingToSkip; } - public boolean hasBytesAvailableInDecompressedBuffer(int bytes) { - return decompressedBufferPosition + bytes <= decompressedBufferLength; - } - private boolean ensureBytesAvailableInDecompressedBuffer() throws IOException { while (decompressedBufferPosition >= decompressedBufferLength) { if (!fillBuffer()) { diff --git a/core/src/main/java/net/jpountz/lz4/LZ4StreamTest.java b/core/src/main/java/net/jpountz/lz4/LZ4StreamTest.java index 160f8549..0943b5f5 100644 --- a/core/src/main/java/net/jpountz/lz4/LZ4StreamTest.java +++ b/core/src/main/java/net/jpountz/lz4/LZ4StreamTest.java @@ -1,8 +1,8 @@ package net.jpountz.lz4; -import com.boydti.fawe.object.io.FastByteArrayOutputStream; -import com.boydti.fawe.object.io.FastByteArraysInputStream; import com.boydti.fawe.util.MainUtil; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Random; @@ -19,7 +19,7 @@ public class LZ4StreamTest { private Random rand; private byte randomContent[]; - private byte compressedOutput[][]; + private byte compressedOutput[]; @Before public void setUp() throws IOException { @@ -35,7 +35,7 @@ public class LZ4StreamTest { } private void compressContent() throws IOException { - FastByteArrayOutputStream compressedOutputStream = new FastByteArrayOutputStream(); + ByteArrayOutputStream compressedOutputStream = new ByteArrayOutputStream(); LZ4OutputStream os = new LZ4OutputStream(compressedOutputStream); int currentContentPosition = 0; @@ -69,13 +69,13 @@ public class LZ4StreamTest { os.close(); - compressedOutput = compressedOutputStream.toByteArrays(); + compressedOutput = compressedOutputStream.toByteArray(); } @Test public void randomizedTest() throws IOException { try { - InputStream is = new LZ4InputStream(new FastByteArraysInputStream(compressedOutput)); + InputStream is = new LZ4BlockInputStream(new ByteArrayInputStream(compressedOutput)); int currentContentPosition = 0;