Upstream/FastAsyncWorldedit
1
0
Fork 0
mirror of https://github.com/boy0001/FastAsyncWorldedit.git synced 2024-11-28 21:56:33 +01:00
Code Issues Projects Releases Wiki Activity

Fix higher compression values + message tweaks

Browse Source
This commit is contained in:
Jesse Boyd 2017-04-23 21:47:31 +10:00
parent f9174ffb56
commit a78a5e20ec
No known key found for this signature in database
GPG Key ID: 59F1DE6293AF6E1F
11 changed files with 554 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
core/src/main/java/com/boydti/fawe/config/BBC.java
View File

@ -26,25 +26,25 @@ public enum BBC {
SCHEMATIC_PASTING("&7The schematic is pasting. This cannot be undone.", "Info"),
LIGHTING_PROPOGATE_SELECTION("&7Lighting has been propogated in %s0 chunks. (Note: To remove light use //removelight)", "Info"),
UPDATED_LIGHTING_SELECTION("&7Lighting has been updated in %s0 chunks. (It may take a second for the packets to send)", "Info"),
SET_REGION("&7Selection set to your current WorldEdit region", "Info"),
SET_REGION("&7Selection set to your current allowed region", "Info"),
WORLDEDIT_COMMAND_LIMIT("&7Please wait until your current action completes", "Info"),
WORLDEDIT_DELAYED("&7Please wait while we process your WorldEdit action...", "Info"),
WORLDEDIT_DELAYED("&7Please wait while we process your FAWE action...", "Info"),
WORLDEDIT_RUN("&7Apologies for the delay. Now executing: %s", "Info"),
WORLDEDIT_COMPLETE("&7WorldEdit action completed.", "Info"),
WORLDEDIT_COMPLETE("&7Edit completed.", "Info"),
REQUIRE_SELECTION_IN_MASK("&7%s of your selection is not within your mask. You can only make edits within allowed regions.", "Info"),
WORLDEDIT_VOLUME("&7You cannot select a volume of %current%. The maximum volume you can modify is %max%.", "Info"),
WORLDEDIT_ITERATIONS("&7You cannot iterate %current% times. The maximum number of iterations allowed is %max%.", "Info"),
WORLDEDIT_UNSAFE("&7Access to that command has been blocked", "Info"),
WORLDEDIT_DANGEROUS_WORLDEDIT("&cProcessed unsafe WorldEdit at %s0 by %s1", "Info"),
WORLDEDIT_DANGEROUS_WORLDEDIT("&cProcessed unsafe edit at %s0 by %s1", "Info"),
WORLDEDIT_BYPASS("&7&oTo bypass your restrictions use &c/wea", "Info"),
WORLDEDIT_EXTEND("&cYour WorldEdit may have extended outside your allowed region.", "Error"),
WORLDEDIT_TOGGLE_TIPS_ON("&7Disabled WorldEdit tips.", "Info"),
WORLDEDIT_TOGGLE_TIPS_OFF("&7Enabled WorldEdit tips.", "Info"),
WORLDEDIT_EXTEND("&cYour edit may have extended outside your allowed region.", "Error"),
WORLDEDIT_TOGGLE_TIPS_ON("&7Disabled FAWE tips.", "Info"),
WORLDEDIT_TOGGLE_TIPS_OFF("&7Enabled FAWE tips.", "Info"),
WORLDEDIT_BYPASSED("&7Currently bypassing WorldEdit restriction.", "Info"),
WORLDEDIT_UNMASKED("&6Your WorldEdit is now unrestricted.", "Info"),
WORLDEDIT_BYPASSED("&7Currently bypassing FAWE restriction.", "Info"),
WORLDEDIT_UNMASKED("&6Your FAWE edits are now unrestricted.", "Info"),
WORLDEDIT_RESTRICTED("&6Your WorldEdit is now restricted.", "Info"),
WORLDEDIT_RESTRICTED("&6Your FAWE edits are now restricted.", "Info"),
WORLDEDIT_OOM_ADMIN("&cPossible options:\n&8 - &7//fast\n&8 - &7Do smaller edits\n&8 - &7Allocate more memory\n&8 - &7Disable `max-memory-percent`", "Info"),
COMPRESSED("History compressed. Saved ~ %s0b (%s1x smaller)", "Info"),
@ -231,7 +231,7 @@ public enum BBC {
NO_PERM("&cYou are lacking the permission node: %s0", "Error"),
SETTING_DISABLE("&cLacking setting: %s0","Error"),
SCHEMATIC_NOT_FOUND("&cSchematic not found: &7%s0", "Error"),
NO_REGION("&cYou have no current WorldEdit region", "Error"),
NO_REGION("&cYou have no current allowed region", "Error"),
NO_MASK("&cYou have no current mask set", "Error"),
NOT_PLAYER("&cYou must be a player to perform this action!", "Error"),
PLAYER_NOT_FOUND("&cPlayer not found:&7 %s0", "Error"),

32
core/src/main/java/com/boydti/fawe/util/MainUtil.java
View File

@ -65,11 +65,12 @@ import java.util.zip.GZIPInputStream;
import java.util.zip.Inflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import net.jpountz.lz4.LZ4BlockInputStream;
import net.jpountz.lz4.LZ4BlockOutputStream;
import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
import net.jpountz.lz4.LZ4FastDecompressor;
import net.jpountz.lz4.LZ4InputStream;
import net.jpountz.lz4.LZ4OutputStream;
import net.jpountz.lz4.LZ4Utils;
public class MainUtil {
@ -311,7 +312,7 @@ public class MainUtil {
}
public static FaweOutputStream getCompressedOS(OutputStream os, int amount, int buffer) throws IOException {
os.write((byte) -amount);
os.write((byte) 9 + amount);
os = new BufferedOutputStream(os, buffer);
if (amount == 0) {
return new FaweOutputStream(os);
@ -324,14 +325,14 @@ public class MainUtil {
LZ4Factory factory = LZ4Factory.fastestInstance();
int fastAmount = 1 + ((amount - 1) % 3);
for (int i = 0; i < fastAmount; i++) {
os = new LZ4OutputStream(os, buffer, factory.fastCompressor());
os = new LZ4BlockOutputStream(os, buffer, factory.fastCompressor());
}
int highAmount = amount > 3 ? 1 : 0;
for (int i = 0; i < highAmount; i++) {
if (amount == 9) {
os = new LZ4OutputStream(os, buffer, factory.highCompressor(17));
os = new LZ4BlockOutputStream(os, buffer, factory.highCompressor(17));
} else {
os = new LZ4OutputStream(os, buffer, factory.highCompressor());
os = new LZ4BlockOutputStream(os, buffer, factory.highCompressor());
}
}
return new FaweOutputStream(os);
@ -342,14 +343,21 @@ public class MainUtil {
}
public static FaweInputStream getCompressedIS(InputStream is, int buffer) throws IOException {
int amount = (byte) is.read();
int mode = (byte) is.read();
is = new BufferedInputStream(is, buffer);
if (amount == 0) {
if (mode == 0) {
return new FaweInputStream(is);
}
int amountAbs = Math.abs(amount);
boolean legacy;
if (mode > 9) {
legacy = false;
mode = -mode + 9;
} else {
legacy = true;
}
int amountAbs = Math.abs(mode);
if (amountAbs > 6) {
if (amount > 0) {
if (mode > 0) {
is = new BufferedInputStream(new GZIPInputStream(is, buffer));
} else {
is = new ZstdInputStream(is);
@ -357,7 +365,11 @@ public class MainUtil {
}
amountAbs = (1 + ((amountAbs - 1) % 3)) + (amountAbs > 3 ? 1 : 0);
for (int i = 0; i < amountAbs; i++) {
is = new LZ4InputStream(is);
if (legacy) {
is = new LZ4InputStream(is);
} else {
is = new LZ4BlockInputStream(is);
}
}
return new FaweInputStream(is);
}

6
core/src/main/java/com/sk89q/worldedit/command/ChunkCommands.java
View File

@ -94,7 +94,7 @@ public class ChunkCommands {
@Command(
aliases = { "delchunks" },
usage = "",
desc = "Delete chunks that your selection includes",
desc = "Deprecated, use anvil commands",
min = 0,
max = 0
)
@ -114,7 +114,7 @@ public class ChunkCommands {
out = new FileOutputStream("worldedit-delchunks.bat");
OutputStreamWriter writer = new OutputStreamWriter(out, "UTF-8");
writer.write("@ECHO off\r\n");
writer.write("ECHO This batch file was generated by WorldEdit.\r\n");
writer.write("ECHO This batch file was generated by FAWE.\r\n");
writer.write("ECHO It contains a list of chunks that were in the selected region\r\n");
writer.write("ECHO at the time that the /delchunks command was used. Run this file\r\n");
writer.write("ECHO in order to delete the chunk files listed in this file.\r\n");
@ -145,7 +145,7 @@ public class ChunkCommands {
out = new FileOutputStream("worldedit-delchunks.sh");
OutputStreamWriter writer = new OutputStreamWriter(out, "UTF-8");
writer.write("#!/bin/bash\n");
writer.write("echo This shell file was generated by WorldEdit.\n");
writer.write("echo This shell file was generated by FAWE.\n");
writer.write("echo It contains a list of chunks that were in the selected region\n");
writer.write("echo at the time that the /delchunks command was used. Run this file\n");
writer.write("echo in order to delete the chunk files listed in this file.\n");

4
core/src/main/java/com/sk89q/worldedit/command/GeneralCommands.java
View File

@ -43,7 +43,7 @@ public class GeneralCommands {
@Command(
aliases = { "/tips", "tips" },
desc = "Toggle WorldEdit tips"
desc = "Toggle FAWE tips"
)
public void tips(Player player, LocalSession session) throws WorldEditException {
FawePlayer<Object> fp = FawePlayer.wrap(player);
@ -57,7 +57,7 @@ public class GeneralCommands {
@Command(
aliases = { "/fast" },
usage = "[on|off]",
desc = "Toggles WorldEdit undo",
desc = "Toggles FAWE undo",
min = 0,
max = 1
)

4
core/src/main/java/com/sk89q/worldedit/command/SnapshotUtilCommands.java
View File

@ -95,10 +95,10 @@ public class SnapshotUtilCommands {
File dir = config.snapshotRepo.getDirectory();
try {
logger.info("WorldEdit found no snapshots: looked in: "
logger.info("FAWE found no snapshots: looked in: "
+ dir.getCanonicalPath());
} catch (IOException e) {
logger.info("WorldEdit found no snapshots: looked in "
logger.info("FAWE found no snapshots: looked in "
+ "(NON-RESOLVABLE PATH - does it exist?): "
+ dir.getPath());
}

32
core/src/main/java/com/sk89q/worldedit/command/WorldEditCommands.java
View File

@ -66,22 +66,9 @@ public class WorldEditCommands {
max = 0
)
public void version(Actor actor) throws WorldEditException {
actor.print(BBC.getPrefix() + "WorldEdit " + WorldEdit.getVersion());
PlatformManager pm = we.getPlatformManager();
actor.printDebug("------------------------------------");
actor.printDebug("Platforms:");
for (Platform platform : pm.getPlatforms()) {
actor.printDebug(String.format(" - %s (%s)", platform.getPlatformName(), platform.getPlatformVersion()));
}
actor.printDebug("Capabilities:");
for (Capability capability : Capability.values()) {
Platform platform = pm.queryCapability(capability);
actor.printDebug(String.format(" - %s: %s", capability.name(), platform != null ? platform.getPlatformName() : "NONE"));
}
actor.printDebug("------------------------------------");
FaweVersion fVer = Fawe.get().getVersion();
String fVerStr = fVer == null ? "unknown" : fVer.year + "." + fVer.month + "." + fVer.day + "-" + Integer.toHexString(fVer.hash) + "-" + fVer.build;
actor.print(BBC.getPrefix() + "FAWE " + fVerStr);
actor.print(BBC.getPrefix() + "FAWE " + fVerStr + " by Empire92");
if (fVer != null) {
actor.printDebug("------------------------------------");
FaweVersion version = Fawe.get().getVersion();
@ -99,8 +86,21 @@ public class WorldEditCommands {
actor.printDebug(" - UPDATES: Latest Version");
}
actor.printDebug("------------------------------------");
actor.printDebug("Wiki: " + "https://github.com/boy0001/FastAsyncWorldedit/wiki");
}
actor.print(BBC.getPrefix() + "WorldEdit " + WorldEdit.getVersion() + " by sk89q");
PlatformManager pm = we.getPlatformManager();
actor.printDebug("------------------------------------");
actor.printDebug("Platforms:");
for (Platform platform : pm.getPlatforms()) {
actor.printDebug(String.format(" - %s (%s)", platform.getPlatformName(), platform.getPlatformVersion()));
}
actor.printDebug("Capabilities:");
for (Capability capability : Capability.values()) {
Platform platform = pm.queryCapability(capability);
actor.printDebug(String.format(" - %s: %s", capability.name(), platform != null ? platform.getPlatformName() : "NONE"));
}
actor.printDebug("------------------------------------");
actor.printDebug("Wiki: " + "https://github.com/boy0001/FastAsyncWorldedit/wiki");
}
@Command(
@ -202,7 +202,7 @@ public class WorldEditCommands {
@Command(
aliases = { "help" },
usage = "[<command>]",
desc = "Displays help for WorldEdit commands",
desc = "Displays help for FAWE commands",
min = 0,
max = -1
)

2
core/src/main/java/com/sk89q/worldedit/extension/platform/CommandManager.java
View File

@ -236,7 +236,7 @@ public final class CommandManager {
.registerMethods(new ToolCommands(worldEdit))
.registerMethods(new UtilityCommands(worldEdit))
.group("worldedit", "we", "fawe")
.describeAs("WorldEdit commands")
.describeAs("FAWE commands")
.registerMethods(new WorldEditCommands(worldEdit)).parent().group("schematic", "schem", "/schematic", "/schem")
.describeAs("Schematic commands for saving/loading areas")
.registerMethods(new SchematicCommands(worldEdit)).parent().group("snapshot", "snap")

235
core/src/main/java/net/jpountz/lz4/LZ4BlockInputStream.java Normal file
View File

@ -0,0 +1,235 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.EOFException;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.Checksum;
import net.jpountz.util.SafeUtils;
import static net.jpountz.lz4.LZ4BlockOutputStream.*;
/**
* {@link InputStream} implementation to decode data written with
* {@link LZ4BlockOutputStream}. This class is not thread-safe and does not
* support {@link #mark(int)}/{@link #reset()}.
* @see LZ4BlockOutputStream
*/
public final class LZ4BlockInputStream extends FilterInputStream {
private final LZ4FastDecompressor decompressor;
private final Checksum checksum;
private byte[] buffer;
private byte[] compressedBuffer;
private int originalLen;
private int o;
private boolean finished;
/**
* Create a new {@link InputStream}.
*
* @param in the {@link InputStream} to poll
* @param decompressor the {@link LZ4FastDecompressor decompressor} instance to
* use
* @param checksum the {@link Checksum} instance to use, must be
* equivalent to the instance which has been used to
* write the stream
*/
public LZ4BlockInputStream(InputStream in, LZ4FastDecompressor decompressor, Checksum checksum) {
super(in);
this.decompressor = decompressor;
this.checksum = checksum;
this.buffer = new byte[0];
this.compressedBuffer = new byte[HEADER_LENGTH];
o = originalLen = 0;
finished = false;
}
public LZ4BlockInputStream(InputStream in, LZ4FastDecompressor decompressor) {
this(in, decompressor, null);
}
/**
* Create a new instance which uses the fastest {@link LZ4FastDecompressor} available.
* @see LZ4Factory#fastestInstance()
* @see #LZ4BlockInputStream(InputStream, LZ4FastDecompressor)
*/
public LZ4BlockInputStream(InputStream in) {
this(in, LZ4Factory.fastestInstance().fastDecompressor());
}
@Override
public int available() throws IOException {
return originalLen - o;
}
@Override
public int read() throws IOException {
if (finished) {
return -1;
}
if (o == originalLen) {
refill();
}
if (finished) {
return -1;
}
return buffer[o++] & 0xFF;
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
SafeUtils.checkRange(b, off, len);
if (finished) {
return -1;
}
if (o == originalLen) {
refill();
}
if (finished) {
return -1;
}
len = Math.min(len, originalLen - o);
System.arraycopy(buffer, o, b, off, len);
o += len;
return len;
}
@Override
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
@Override
public long skip(long n) throws IOException {
if (finished) {
return -1;
}
if (o == originalLen) {
refill();
}
if (finished) {
return -1;
}
final int skipped = (int) Math.min(n, originalLen - o);
o += skipped;
return skipped;
}
private void refill() throws IOException {
readFully(compressedBuffer, HEADER_LENGTH);
for (int i = 0; i < MAGIC_LENGTH; ++i) {
if (compressedBuffer[i] != MAGIC[i]) {
throw new IOException("Stream is corrupted");
}
}
final int token = compressedBuffer[MAGIC_LENGTH] & 0xFF;
final int compressionMethod = token & 0xF0;
final int compressionLevel = COMPRESSION_LEVEL_BASE + (token & 0x0F);
if (compressionMethod != COMPRESSION_METHOD_RAW && compressionMethod != COMPRESSION_METHOD_LZ4) {
throw new IOException("Stream is corrupted");
}
final int compressedLen = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 1);
originalLen = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 5);
final int check = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 9);
assert HEADER_LENGTH == MAGIC_LENGTH + 13;
if (originalLen > 1 << compressionLevel
|| originalLen < 0
|| compressedLen < 0
|| (originalLen == 0 && compressedLen != 0)
|| (originalLen != 0 && compressedLen == 0)
|| (compressionMethod == COMPRESSION_METHOD_RAW && originalLen != compressedLen)) {
throw new IOException("Stream is corrupted");
}
if (originalLen == 0 && compressedLen == 0) {
if (check != 0) {
throw new IOException("Stream is corrupted");
}
finished = true;
return;
}
if (buffer.length < originalLen) {
buffer = new byte[Math.max(originalLen, buffer.length * 3 / 2)];
}
switch (compressionMethod) {
case COMPRESSION_METHOD_RAW:
readFully(buffer, originalLen);
break;
case COMPRESSION_METHOD_LZ4:
if (compressedBuffer.length < originalLen) {
compressedBuffer = new byte[Math.max(compressedLen, compressedBuffer.length * 3 / 2)];
}
readFully(compressedBuffer, compressedLen);
try {
final int compressedLen2 = decompressor.decompress(compressedBuffer, 0, buffer, 0, originalLen);
if (compressedLen != compressedLen2) {
throw new IOException("Stream is corrupted");
}
} catch (LZ4Exception e) {
throw new IOException("Stream is corrupted", e);
}
break;
default:
throw new AssertionError();
}
if (checksum != null) {
checksum.reset();
checksum.update(buffer, 0, originalLen);
if ((int) checksum.getValue() != check) {
throw new IOException("Stream is corrupted");
}
}
o = 0;
}
private void readFully(byte[] b, int len) throws IOException {
int read = 0;
while (read < len) {
final int r = in.read(b, read, len - read);
if (r < 0) {
throw new EOFException("Stream ended prematurely");
}
read += r;
}
assert len == read;
}
@Override
public boolean markSupported() {
return false;
}
@SuppressWarnings("sync-override")
@Override
public void mark(int readlimit) {
// unsupported
}
@SuppressWarnings("sync-override")
@Override
public void reset() throws IOException {
throw new IOException("mark/reset not supported");
}
@Override
public String toString() {
return getClass().getSimpleName() + "(in=" + in
+ ", decompressor=" + decompressor + ", checksum=" + checksum + ")";
}
}

256
core/src/main/java/net/jpountz/lz4/LZ4BlockOutputStream.java Normal file
View File

@ -0,0 +1,256 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.zip.Checksum;
import net.jpountz.util.SafeUtils;
/**
* Streaming LZ4.
* <p>
* This class compresses data into fixed-size blocks of compressed data.
* @see LZ4BlockInputStream
*/
public final class LZ4BlockOutputStream extends FilterOutputStream {
static final byte[] MAGIC = new byte[] { 'L', 'Z', '4', 'B', 'l', 'o', 'c', 'k' };
static final int MAGIC_LENGTH = MAGIC.length;
static final int HEADER_LENGTH =
MAGIC_LENGTH // magic bytes
+ 1 // token
+ 4 // compressed length
+ 4 // decompressed length
+ 4; // checksum
static final int COMPRESSION_LEVEL_BASE = 10;
static final int MIN_BLOCK_SIZE = 64;
static final int MAX_BLOCK_SIZE = 1 << (COMPRESSION_LEVEL_BASE + 0x0F);
static final int COMPRESSION_METHOD_RAW = 0x10;
static final int COMPRESSION_METHOD_LZ4 = 0x20;
static final int DEFAULT_SEED = 0x9747b28c;
private static int compressionLevel(int blockSize) {
if (blockSize < MIN_BLOCK_SIZE) {
throw new IllegalArgumentException("blockSize must be >= " + MIN_BLOCK_SIZE + ", got " + blockSize);
} else if (blockSize > MAX_BLOCK_SIZE) {
throw new IllegalArgumentException("blockSize must be <= " + MAX_BLOCK_SIZE + ", got " + blockSize);
}
int compressionLevel = 32 - Integer.numberOfLeadingZeros(blockSize - 1); // ceil of log2
assert (1 << compressionLevel) >= blockSize;
assert blockSize * 2 > (1 << compressionLevel);
compressionLevel = Math.max(0, compressionLevel - COMPRESSION_LEVEL_BASE);
assert compressionLevel >= 0 && compressionLevel <= 0x0F;
return compressionLevel;
}
private final int blockSize;
private final int compressionLevel;
private final LZ4Compressor compressor;
private final Checksum checksum;
private final byte[] buffer;
private final byte[] compressedBuffer;
private final boolean syncFlush;
private boolean finished;
private int o;
/**
* Create a new {@link OutputStream} with configurable block size. Large
* blocks require more memory at compression and decompression time but
* should improve the compression ratio.
*
* @param out the {@link OutputStream} to feed
* @param blockSize the maximum number of bytes to try to compress at once,
* must be >= 64 and <= 32 M
* @param compressor the {@link LZ4Compressor} instance to use to compress
* data
* @param checksum the {@link Checksum} instance to use to check data for
* integrity.
* @param syncFlush true if pending data should also be flushed on {@link #flush()}
*/
public LZ4BlockOutputStream(OutputStream out, int blockSize, LZ4Compressor compressor, Checksum checksum, boolean syncFlush) {
super(out);
this.blockSize = blockSize;
this.compressor = compressor;
this.checksum = checksum;
this.compressionLevel = compressionLevel(blockSize);
this.buffer = new byte[blockSize];
final int compressedBlockSize = HEADER_LENGTH + compressor.maxCompressedLength(blockSize);
this.compressedBuffer = new byte[compressedBlockSize];
this.syncFlush = syncFlush;
o = 0;
finished = false;
System.arraycopy(MAGIC, 0, compressedBuffer, 0, MAGIC_LENGTH);
}
public LZ4BlockOutputStream(OutputStream out, int blockSize, LZ4Compressor compressor) {
this(out, blockSize, compressor, null, false);
}
/**
* Create a new instance which compresses with the standard LZ4 compression
* algorithm.
* @see #LZ4BlockOutputStream(OutputStream, int, LZ4Compressor)
* @see LZ4Factory#fastCompressor()
*/
public LZ4BlockOutputStream(OutputStream out, int blockSize) {
this(out, blockSize, LZ4Factory.fastestInstance().fastCompressor());
}
/**
* Create a new instance which compresses into blocks of 64 KB.
* @see #LZ4BlockOutputStream(OutputStream, int)
*/
public LZ4BlockOutputStream(OutputStream out) {
this(out, 1 << 16);
}
private void ensureNotFinished() {
if (finished) {
throw new IllegalStateException("This stream is already closed");
}
}
@Override
public void write(int b) throws IOException {
ensureNotFinished();
if (o == blockSize) {
flushBufferedData();
}
buffer[o++] = (byte) b;
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
SafeUtils.checkRange(b, off, len);
ensureNotFinished();
while (o + len > blockSize) {
final int l = blockSize - o;
System.arraycopy(b, off, buffer, o, blockSize - o);
o = blockSize;
flushBufferedData();
off += l;
len -= l;
}
System.arraycopy(b, off, buffer, o, len);
o += len;
}
@Override
public void write(byte[] b) throws IOException {
ensureNotFinished();
write(b, 0, b.length);
}
@Override
public void close() throws IOException {
if (!finished) {
finish();
}
if (out != null) {
out.close();
out = null;
}
}
private void flushBufferedData() throws IOException {
if (o == 0) {
return;
}
final int check;
if (checksum != null) {
checksum.reset();
checksum.update(buffer, 0, o);
check = (int) checksum.getValue();
} else {
check = 0;
}
int compressedLength = compressor.compress(buffer, 0, o, compressedBuffer, HEADER_LENGTH);
final int compressMethod;
if (compressedLength >= o) {
compressMethod = COMPRESSION_METHOD_RAW;
compressedLength = o;
System.arraycopy(buffer, 0, compressedBuffer, HEADER_LENGTH, o);
} else {
compressMethod = COMPRESSION_METHOD_LZ4;
}
compressedBuffer[MAGIC_LENGTH] = (byte) (compressMethod | compressionLevel);
writeIntLE(compressedLength, compressedBuffer, MAGIC_LENGTH + 1);
writeIntLE(o, compressedBuffer, MAGIC_LENGTH + 5);
writeIntLE(check, compressedBuffer, MAGIC_LENGTH + 9);
assert MAGIC_LENGTH + 13 == HEADER_LENGTH;
out.write(compressedBuffer, 0, HEADER_LENGTH + compressedLength);
o = 0;
}
/**
* Flush this compressed {@link OutputStream}.
*
* If the stream has been created with <code>syncFlush=true</code>, pending
* data will be compressed and appended to the underlying {@link OutputStream}
* before calling {@link OutputStream#flush()} on the underlying stream.
* Otherwise, this method just flushes the underlying stream, so pending
* data might not be available for reading until {@link #finish()} or
* {@link #close()} is called.
*/
@Override
public void flush() throws IOException {
if (out != null) {
if (syncFlush) {
flushBufferedData();
}
out.flush();
}
}
/**
* Same as {@link #close()} except that it doesn't close the underlying stream.
* This can be useful if you want to keep on using the underlying stream.
*/
public void finish() throws IOException {
ensureNotFinished();
flushBufferedData();
compressedBuffer[MAGIC_LENGTH] = (byte) (COMPRESSION_METHOD_RAW | compressionLevel);
writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 1);
writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 5);
writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 9);
assert MAGIC_LENGTH + 13 == HEADER_LENGTH;
out.write(compressedBuffer, 0, HEADER_LENGTH);
finished = true;
out.flush();
}
private static void writeIntLE(int i, byte[] buf, int off) {
buf[off++] = (byte) i;
buf[off++] = (byte) (i >>> 8);
buf[off++] = (byte) (i >>> 16);
buf[off++] = (byte) (i >>> 24);
}
@Override
public String toString() {
return getClass().getSimpleName() + "(out=" + out + ", blockSize=" + blockSize
+ ", compressor=" + compressor + ", checksum=" + checksum + ")";
}
}

4
core/src/main/java/net/jpountz/lz4/LZ4InputStream.java
View File

@ -80,10 +80,6 @@ public class LZ4InputStream extends InputStream {
return n - numBytesRemainingToSkip;
}
public boolean hasBytesAvailableInDecompressedBuffer(int bytes) {
return decompressedBufferPosition + bytes <= decompressedBufferLength;
}
private boolean ensureBytesAvailableInDecompressedBuffer() throws IOException {
while (decompressedBufferPosition >= decompressedBufferLength) {
if (!fillBuffer()) {

12
core/src/main/java/net/jpountz/lz4/LZ4StreamTest.java
View File

@ -1,8 +1,8 @@
package net.jpountz.lz4;
import com.boydti.fawe.object.io.FastByteArrayOutputStream;
import com.boydti.fawe.object.io.FastByteArraysInputStream;
import com.boydti.fawe.util.MainUtil;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Random;
@ -19,7 +19,7 @@ public class LZ4StreamTest {
private Random rand;
private byte randomContent[];
private byte compressedOutput[][];
private byte compressedOutput[];
@Before
public void setUp() throws IOException {
@ -35,7 +35,7 @@ public class LZ4StreamTest {
}
private void compressContent() throws IOException {
FastByteArrayOutputStream compressedOutputStream = new FastByteArrayOutputStream();
ByteArrayOutputStream compressedOutputStream = new ByteArrayOutputStream();
LZ4OutputStream os = new LZ4OutputStream(compressedOutputStream);
int currentContentPosition = 0;
@ -69,13 +69,13 @@ public class LZ4StreamTest {
os.close();
compressedOutput = compressedOutputStream.toByteArrays();
compressedOutput = compressedOutputStream.toByteArray();
}
@Test
public void randomizedTest() throws IOException {
try {
InputStream is = new LZ4InputStream(new FastByteArraysInputStream(compressedOutput));
InputStream is = new LZ4BlockInputStream(new ByteArrayInputStream(compressedOutput));
int currentContentPosition = 0;