Faster compression

Using java's inbuilt compression algorithms (gzip or deflator both)  was
a bit too slow for me.

Fawe is now using the LZ4 compression algorithm, which is super fast. So
fast that I decided to add a default 1 pass of compression to history.
Enabling further compression will do an additional high LZ4 compression.
This commit is contained in:
Jesse Boyd 2016-04-04 18:39:00 +10:00
parent fb15a425a7
commit 0dfbd44c72
72 changed files with 7376 additions and 121 deletions

View File

@ -41,6 +41,5 @@ subprojects {
maven {url "http://maven.sk89q.com/artifactory/repo/"}
maven {url "http://nexus.theyeticave.net/content/repositories/pub_releases"}
maven {url "http://repo.spongepowered.org/maven"}
}
}

View File

@ -18,6 +18,9 @@ commands:
stream:
description: (FAWE) Stream a schematic into the world
aliases: [/stream]
fawe:
description: (FAWE) Reload the plugin
aliases: [/fawe,/fawereload]
wrg:
description: (FAWE) Select your current WorldEdit Region.
aliases: [/wrg,wer,/wer,worldeditregion,/worldeditregion,/region]
@ -30,3 +33,5 @@ permissions:
default: false
fawe.fixlighting:
default: false
fawe.reload:
default: false

View File

@ -20,7 +20,7 @@ public class BukkitCommand implements CommandExecutor {
@Override
public boolean onCommand(final CommandSender sender, final Command cmd, final String label, final String[] args) {
final FawePlayer plr = Fawe.imp().wrap(sender);
if (!sender.hasPermission(this.cmd.getPerm()) || sender.isOp()) {
if (!sender.hasPermission(this.cmd.getPerm()) && !sender.isOp()) {
BBC.NO_PERM.send(plr, this.cmd.getPerm());
return true;
}

View File

@ -18,6 +18,9 @@ commands:
stream:
description: (FAWE) Stream a schematic into the world
aliases: [/stream]
fawe:
description: (FAWE) Reload the plugin
aliases: [/fawe,/fawereload]
wrg:
description: (FAWE) Select your current WorldEdit Region.
aliases: [/wrg,wer,/wer,worldeditregion,/worldeditregion,/region]
@ -30,3 +33,5 @@ permissions:
default: false
fawe.fixlighting:
default: false
fawe.reload:
default: false

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -1,6 +1,7 @@
package com.boydti.fawe;
import com.boydti.fawe.command.FixLighting;
import com.boydti.fawe.command.Reload;
import com.boydti.fawe.command.Stream;
import com.boydti.fawe.command.Wea;
import com.boydti.fawe.command.WorldEditRegion;
@ -43,6 +44,7 @@ import javax.management.InstanceAlreadyExistsException;
import javax.management.Notification;
import javax.management.NotificationEmitter;
import javax.management.NotificationListener;
import net.jpountz.util.Native;
/**[ WorldEdit action]
* |
@ -195,9 +197,10 @@ public class Fawe {
this.IMP.setupCommand("fixlighting", new FixLighting());
this.IMP.setupCommand("stream", new Stream());
this.IMP.setupCommand("wrg", new WorldEditRegion());
this.IMP.setupCommand("fawe", new Reload());
}
private void setupConfigs() {
public void setupConfigs() {
// Setting up config.yml
Settings.setup(new File(this.IMP.getDirectory(), "config.yml"));
// Setting up message.yml
@ -233,6 +236,11 @@ public class Fawe {
e.printStackTrace();
IMP.debug("Incompatible version of WorldEdit, please update the plugin or contact the Author!");
}
try {
Native.load();
} catch (Throwable e) {
e.printStackTrace();
}
}
private void setupMemoryListener() {

View File

@ -0,0 +1,19 @@
package com.boydti.fawe.command;
import com.boydti.fawe.Fawe;
import com.boydti.fawe.object.FaweCommand;
import com.boydti.fawe.object.FawePlayer;
public class Reload extends FaweCommand {
public Reload() {
super("fawe.reload");
}
@Override
public boolean execute(final FawePlayer player, final String... args) {
Fawe.get().setupConfigs();
player.sendMessage("&d[FAWE] Reloaded configuration");
return true;
}
}

View File

@ -25,7 +25,8 @@ public class Settings {
public static long MEM_FREE = 95;
public static boolean ENABLE_HARD_LIMIT = true;
public static boolean STORE_HISTORY_ON_DISK = false;
public static boolean COMPRESS_HISTORY = false;
public static int COMPRESSION_LEVEL = 0;
public static int BUFFER_SIZE = 59049;
public static boolean METRICS = true;
public static void setup(final File file) {
@ -51,7 +52,7 @@ public class Settings {
options.put("crash-mitigation", ENABLE_HARD_LIMIT);
options.put("fix-all-lighting", FIX_ALL_LIGHTING);
options.put("history.use-disk", STORE_HISTORY_ON_DISK);
options.put("history.compress", COMPRESS_HISTORY);
options.put("history.compress", false);
options.put("metrics", METRICS);
for (final Entry<String, Object> node : options.entrySet()) {
@ -70,7 +71,8 @@ public class Settings {
WE_BLACKLIST = config.getStringList("command-blacklist");
ENABLE_HARD_LIMIT = config.getBoolean("crash-mitigation");
METRICS = config.getBoolean("metrics");
COMPRESS_HISTORY = config.getBoolean("history.compress");
COMPRESSION_LEVEL = config.getInt("history.compression-level", config.getBoolean("history.compress") ? 1 : 0);
BUFFER_SIZE = config.getInt("history.buffer-size", 59049);
if (STORE_HISTORY_ON_DISK = config.getBoolean("history.use-disk")) {
LocalSession.MAX_HISTORY_SIZE = Integer.MAX_VALUE;
}

View File

@ -31,6 +31,10 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
import net.jpountz.lz4.LZ4InputStream;
import net.jpountz.lz4.LZ4OutputStream;
/**
* Store the change on disk
@ -232,7 +236,14 @@ public class DiskStorageHistory implements ChangeSet, FaweChangeSet {
bdFile.getParentFile().mkdirs();
bdFile.createNewFile();
FileOutputStream stream = new FileOutputStream(bdFile);
osBD = Settings.COMPRESS_HISTORY ? new GZIPOutputStream(stream, true) : stream;
LZ4Factory factory = LZ4Factory.fastestInstance();
LZ4Compressor compressor = factory.fastCompressor();
osBD = new LZ4OutputStream(stream, Settings.BUFFER_SIZE, factory.fastCompressor());
if (Settings.COMPRESSION_LEVEL > 0) {
// Deflater deflater = new Deflater(Math.min(9, Settings.COMPRESSION_LEVEL), true);
// osBD = new DeflaterOutputStream(osBD, deflater, true);
osBD = new LZ4OutputStream(osBD, Settings.BUFFER_SIZE, factory.highCompressor());
}
ox = x;
oz = z;
osBD.write((byte) (ox >> 24));
@ -307,7 +318,14 @@ public class DiskStorageHistory implements ChangeSet, FaweChangeSet {
final NBTInputStream nbtt = osNBTT != null ? new NBTInputStream(new GZIPInputStream(new FileInputStream(nbttFile))) : null;
FileInputStream fis = new FileInputStream(bdFile);
final InputStream gis = Settings.COMPRESS_HISTORY ? new GZIPInputStream(fis) : fis;
LZ4Factory factory = LZ4Factory.fastestInstance();
LZ4Compressor compressor = factory.fastCompressor();
final InputStream gis;
if (Settings.COMPRESSION_LEVEL > 0) {
gis = new LZ4InputStream(new LZ4InputStream(fis));
} else {
gis = new LZ4InputStream(fis);
}
gis.skip(8);
return new Iterator<Change>() {

View File

@ -1,19 +1,6 @@
package com.boydti.fawe.object.changeset;
import com.sk89q.worldedit.Vector;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import com.boydti.fawe.Fawe;
import com.boydti.fawe.FaweCache;
import com.boydti.fawe.config.Settings;
import com.boydti.fawe.util.MainUtil;
@ -23,10 +10,23 @@ import com.sk89q.jnbt.CompoundTag;
import com.sk89q.jnbt.IntTag;
import com.sk89q.jnbt.Tag;
import com.sk89q.worldedit.BlockVector;
import com.sk89q.worldedit.Vector;
import com.sk89q.worldedit.blocks.BaseBlock;
import com.sk89q.worldedit.history.change.BlockChange;
import com.sk89q.worldedit.history.change.Change;
import com.sk89q.worldedit.history.changeset.ChangeSet;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import net.jpountz.lz4.LZ4Factory;
import net.jpountz.lz4.LZ4InputStream;
import net.jpountz.lz4.LZ4OutputStream;
/**
* ChangeSet optimized for low memory usage
@ -40,23 +40,27 @@ public class MemoryOptimizedHistory implements ChangeSet, FaweChangeSet {
private ArrayDeque<CompoundTag> toTags;
private byte[] ids;
private Object lock;
private int decompressedLength;
private ByteArrayOutputStream idsStream;
private OutputStream idsStreamZip;
private ArrayDeque<Change> entities;
int ox;
int oz;
private final AtomicInteger size;
private int size;
public MemoryOptimizedHistory() {
size = new AtomicInteger();
}
@Override
public void add(int x, int y, int z, int combinedFrom, int combinedTo) {
size++;
try {
OutputStream stream = getBAOS(x, y, z);
//x
@ -124,7 +128,6 @@ public class MemoryOptimizedHistory implements ChangeSet, FaweChangeSet {
@Override
public void add(Change arg) {
size.incrementAndGet();
if ((arg instanceof BlockChange)) {
BlockChange change = (BlockChange) arg;
BlockVector loc = change.getPosition();
@ -143,8 +146,14 @@ public class MemoryOptimizedHistory implements ChangeSet, FaweChangeSet {
if (idsStreamZip != null) {
return idsStreamZip;
}
idsStream = new ByteArrayOutputStream(9216);
idsStreamZip = Settings.COMPRESS_HISTORY ? new GZIPOutputStream(idsStream, true) : idsStream;
LZ4Factory factory = LZ4Factory.fastestInstance();
idsStream = new ByteArrayOutputStream(Settings.BUFFER_SIZE);
idsStreamZip = new LZ4OutputStream(idsStream, Settings.BUFFER_SIZE, factory.fastCompressor());
if (Settings.COMPRESSION_LEVEL > 0) {
// Deflater deflater = new Deflater(Math.min(9, Settings.COMPRESSION_LEVEL), true);
// idsStreamZip = new DeflaterOutputStream(idsStreamZip, deflater, true);
idsStreamZip = new LZ4OutputStream(idsStreamZip, Settings.BUFFER_SIZE, factory.highCompressor());
}
ox = x;
oz = z;
return idsStreamZip;
@ -153,6 +162,13 @@ public class MemoryOptimizedHistory implements ChangeSet, FaweChangeSet {
@SuppressWarnings("resource")
public Iterator<Change> getIterator(final boolean dir) {
flush();
if (lock != null) {
try {
lock.wait();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
try {
Iterator<Change> idsIterator;
Iterator<Change> entsIterator = entities != null ? entities.iterator() : new ArrayList().iterator();
@ -160,9 +176,13 @@ public class MemoryOptimizedHistory implements ChangeSet, FaweChangeSet {
idsIterator = new ArrayList().iterator();
} else {
ByteArrayInputStream bais = new ByteArrayInputStream(ids);
final InputStream gis = Settings.COMPRESS_HISTORY ? new GZIPInputStream(bais) : bais;
final InputStream gis;
if (Settings.COMPRESSION_LEVEL > 0) {
gis = new LZ4InputStream(new LZ4InputStream(bais));
} else {
gis = new LZ4InputStream(bais);
}
idsIterator = new Iterator<Change>() {
private final Iterator<CompoundTag> lastFromIter = fromTags != null ? fromTags.iterator() : null;
private final Iterator<CompoundTag> lastToIter = toTags != null ? toTags.iterator() : null;
private CompoundTag lastFrom = read(lastFromIter);
@ -253,7 +273,7 @@ public class MemoryOptimizedHistory implements ChangeSet, FaweChangeSet {
@Override
public int size() {
return size.get();
return size;
}
@Override
@ -264,6 +284,14 @@ public class MemoryOptimizedHistory implements ChangeSet, FaweChangeSet {
idsStreamZip.flush();
idsStreamZip.close();
ids = idsStream.toByteArray();
// Estimate
int total = 0x18 * size;
int ratio = total / ids.length;
int saved = total - ids.length;
if (ratio > 3) {
// TODO remove this debug message
Fawe.debug("[FAWE] History compressed. Saved ~ " + saved + "b (" + ratio + "x smaller)");
}
idsStream = null;
idsStreamZip = null;
} catch (IOException e) {

View File

View File

@ -0,0 +1,248 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.lz4.LZ4BlockOutputStream.COMPRESSION_LEVEL_BASE;
import static net.jpountz.lz4.LZ4BlockOutputStream.COMPRESSION_METHOD_LZ4;
import static net.jpountz.lz4.LZ4BlockOutputStream.COMPRESSION_METHOD_RAW;
import static net.jpountz.lz4.LZ4BlockOutputStream.DEFAULT_SEED;
import static net.jpountz.lz4.LZ4BlockOutputStream.HEADER_LENGTH;
import static net.jpountz.lz4.LZ4BlockOutputStream.MAGIC;
import static net.jpountz.lz4.LZ4BlockOutputStream.MAGIC_LENGTH;
import java.io.EOFException;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.Checksum;
import net.jpountz.util.SafeUtils;
import net.jpountz.util.Utils;
import net.jpountz.xxhash.StreamingXXHash32;
import net.jpountz.xxhash.XXHash32;
import net.jpountz.xxhash.XXHashFactory;
/**
* {@link InputStream} implementation to decode data written with
* {@link LZ4BlockOutputStream}. This class is not thread-safe and does not
* support {@link #mark(int)}/{@link #reset()}.
* @see LZ4BlockOutputStream
*/
public final class LZ4BlockInputStream extends FilterInputStream {
private final LZ4FastDecompressor decompressor;
private final Checksum checksum;
private byte[] buffer;
private byte[] compressedBuffer;
private int originalLen;
private int o;
private boolean finished;
/**
* Create a new {@link InputStream}.
*
* @param in the {@link InputStream} to poll
* @param decompressor the {@link LZ4FastDecompressor decompressor} instance to
* use
* @param checksum the {@link Checksum} instance to use, must be
* equivalent to the instance which has been used to
* write the stream
*/
public LZ4BlockInputStream(InputStream in, LZ4FastDecompressor decompressor, Checksum checksum) {
super(in);
this.decompressor = decompressor;
this.checksum = checksum;
this.buffer = new byte[0];
this.compressedBuffer = new byte[HEADER_LENGTH];
o = originalLen = 0;
finished = false;
}
/**
* Create a new instance using {@link XXHash32} for checksuming.
* @see #LZ4BlockInputStream(InputStream, LZ4FastDecompressor, Checksum)
* @see StreamingXXHash32#asChecksum()
*/
public LZ4BlockInputStream(InputStream in, LZ4FastDecompressor decompressor) {
this(in, decompressor, XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum());
}
/**
* Create a new instance which uses the fastest {@link LZ4FastDecompressor} available.
* @see LZ4Factory#fastestInstance()
* @see #LZ4BlockInputStream(InputStream, LZ4FastDecompressor)
*/
public LZ4BlockInputStream(InputStream in) {
this(in, LZ4Factory.fastestInstance().fastDecompressor());
}
@Override
public int available() throws IOException {
return originalLen - o;
}
@Override
public int read() throws IOException {
if (finished) {
return -1;
}
if (o == originalLen) {
refill();
}
if (finished) {
return -1;
}
return buffer[o++] & 0xFF;
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
SafeUtils.checkRange(b, off, len);
if (finished) {
return -1;
}
if (o == originalLen) {
refill();
}
if (finished) {
return -1;
}
len = Math.min(len, originalLen - o);
System.arraycopy(buffer, o, b, off, len);
o += len;
return len;
}
@Override
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
@Override
public long skip(long n) throws IOException {
if (finished) {
return -1;
}
if (o == originalLen) {
refill();
}
if (finished) {
return -1;
}
final int skipped = (int) Math.min(n, originalLen - o);
o += skipped;
return skipped;
}
private void refill() throws IOException {
readFully(compressedBuffer, HEADER_LENGTH);
for (int i = 0; i < MAGIC_LENGTH; ++i) {
if (compressedBuffer[i] != MAGIC[i]) {
throw new IOException("Stream is corrupted");
}
}
final int token = compressedBuffer[MAGIC_LENGTH] & 0xFF;
final int compressionMethod = token & 0xF0;
final int compressionLevel = COMPRESSION_LEVEL_BASE + (token & 0x0F);
if (compressionMethod != COMPRESSION_METHOD_RAW && compressionMethod != COMPRESSION_METHOD_LZ4) {
throw new IOException("Stream is corrupted");
}
final int compressedLen = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 1);
originalLen = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 5);
final int check = SafeUtils.readIntLE(compressedBuffer, MAGIC_LENGTH + 9);
assert HEADER_LENGTH == MAGIC_LENGTH + 13;
if (originalLen > 1 << compressionLevel
|| originalLen < 0
|| compressedLen < 0
|| (originalLen == 0 && compressedLen != 0)
|| (originalLen != 0 && compressedLen == 0)
|| (compressionMethod == COMPRESSION_METHOD_RAW && originalLen != compressedLen)) {
throw new IOException("Stream is corrupted");
}
if (originalLen == 0 && compressedLen == 0) {
if (check != 0) {
throw new IOException("Stream is corrupted");
}
finished = true;
return;
}
if (buffer.length < originalLen) {
buffer = new byte[Math.max(originalLen, buffer.length * 3 / 2)];
}
switch (compressionMethod) {
case COMPRESSION_METHOD_RAW:
readFully(buffer, originalLen);
break;
case COMPRESSION_METHOD_LZ4:
if (compressedBuffer.length < originalLen) {
compressedBuffer = new byte[Math.max(compressedLen, compressedBuffer.length * 3 / 2)];
}
readFully(compressedBuffer, compressedLen);
try {
final int compressedLen2 = decompressor.decompress(compressedBuffer, 0, buffer, 0, originalLen);
if (compressedLen != compressedLen2) {
throw new IOException("Stream is corrupted");
}
} catch (LZ4Exception e) {
throw new IOException("Stream is corrupted", e);
}
break;
default:
throw new AssertionError();
}
checksum.reset();
checksum.update(buffer, 0, originalLen);
if ((int) checksum.getValue() != check) {
throw new IOException("Stream is corrupted");
}
o = 0;
}
private void readFully(byte[] b, int len) throws IOException {
int read = 0;
while (read < len) {
final int r = in.read(b, read, len - read);
if (r < 0) {
throw new EOFException("Stream ended prematurely");
}
read += r;
}
assert len == read;
}
@Override
public boolean markSupported() {
return false;
}
@SuppressWarnings("sync-override")
@Override
public void mark(int readlimit) {
// unsupported
}
@SuppressWarnings("sync-override")
@Override
public void reset() throws IOException {
throw new IOException("mark/reset not supported");
}
@Override
public String toString() {
return getClass().getSimpleName() + "(in=" + in
+ ", decompressor=" + decompressor + ", checksum=" + checksum + ")";
}
}

View File

@ -0,0 +1,259 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.zip.Checksum;
import net.jpountz.util.SafeUtils;
import net.jpountz.xxhash.StreamingXXHash32;
import net.jpountz.xxhash.XXHashFactory;
/**
* Streaming LZ4.
* <p>
* This class compresses data into fixed-size blocks of compressed data.
* @see LZ4BlockInputStream
*/
public final class LZ4BlockOutputStream extends FilterOutputStream {
static final byte[] MAGIC = new byte[] { 'L', 'Z', '4', 'B', 'l', 'o', 'c', 'k' };
static final int MAGIC_LENGTH = MAGIC.length;
static final int HEADER_LENGTH =
MAGIC_LENGTH // magic bytes
+ 1 // token
+ 4 // compressed length
+ 4 // decompressed length
+ 4; // checksum
static final int COMPRESSION_LEVEL_BASE = 10;
static final int MIN_BLOCK_SIZE = 64;
static final int MAX_BLOCK_SIZE = 1 << (COMPRESSION_LEVEL_BASE + 0x0F);
static final int COMPRESSION_METHOD_RAW = 0x10;
static final int COMPRESSION_METHOD_LZ4 = 0x20;
static final int DEFAULT_SEED = 0x9747b28c;
private static int compressionLevel(int blockSize) {
if (blockSize < MIN_BLOCK_SIZE) {
throw new IllegalArgumentException("blockSize must be >= " + MIN_BLOCK_SIZE + ", got " + blockSize);
} else if (blockSize > MAX_BLOCK_SIZE) {
throw new IllegalArgumentException("blockSize must be <= " + MAX_BLOCK_SIZE + ", got " + blockSize);
}
int compressionLevel = 32 - Integer.numberOfLeadingZeros(blockSize - 1); // ceil of log2
assert (1 << compressionLevel) >= blockSize;
assert blockSize * 2 > (1 << compressionLevel);
compressionLevel = Math.max(0, compressionLevel - COMPRESSION_LEVEL_BASE);
assert compressionLevel >= 0 && compressionLevel <= 0x0F;
return compressionLevel;
}
private final int blockSize;
private final int compressionLevel;
private final LZ4Compressor compressor;
private final Checksum checksum;
private final byte[] buffer;
private final byte[] compressedBuffer;
private final boolean syncFlush;
private boolean finished;
private int o;
/**
* Create a new {@link OutputStream} with configurable block size. Large
* blocks require more memory at compression and decompression time but
* should improve the compression ratio.
*
* @param out the {@link OutputStream} to feed
* @param blockSize the maximum number of bytes to try to compress at once,
* must be >= 64 and <= 32 M
* @param compressor the {@link LZ4Compressor} instance to use to compress
* data
* @param checksum the {@link Checksum} instance to use to check data for
* integrity.
* @param syncFlush true if pending data should also be flushed on {@link #flush()}
*/
public LZ4BlockOutputStream(OutputStream out, int blockSize, LZ4Compressor compressor, Checksum checksum, boolean syncFlush) {
super(out);
this.blockSize = blockSize;
this.compressor = compressor;
this.checksum = checksum;
this.compressionLevel = compressionLevel(blockSize);
this.buffer = new byte[blockSize];
final int compressedBlockSize = HEADER_LENGTH + compressor.maxCompressedLength(blockSize);
this.compressedBuffer = new byte[compressedBlockSize];
this.syncFlush = syncFlush;
o = 0;
finished = false;
System.arraycopy(MAGIC, 0, compressedBuffer, 0, MAGIC_LENGTH);
}
/**
* Create a new instance which checks stream integrity using
* {@link StreamingXXHash32} and doesn't sync flush.
* @see #LZ4BlockOutputStream(OutputStream, int, LZ4Compressor, Checksum, boolean)
* @see StreamingXXHash32#asChecksum()
*/
public LZ4BlockOutputStream(OutputStream out, int blockSize, LZ4Compressor compressor) {
this(out, blockSize, compressor, XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum(), false);
}
/**
* Create a new instance which compresses with the standard LZ4 compression
* algorithm.
* @see #LZ4BlockOutputStream(OutputStream, int, LZ4Compressor)
* @see LZ4Factory#fastCompressor()
*/
public LZ4BlockOutputStream(OutputStream out, int blockSize) {
this(out, blockSize, LZ4Factory.fastestInstance().fastCompressor());
}
/**
* Create a new instance which compresses into blocks of 64 KB.
* @see #LZ4BlockOutputStream(OutputStream, int)
*/
public LZ4BlockOutputStream(OutputStream out) {
this(out, 1 << 16);
}
private void ensureNotFinished() {
if (finished) {
throw new IllegalStateException("This stream is already closed");
}
}
@Override
public void write(int b) throws IOException {
ensureNotFinished();
if (o == blockSize) {
flushBufferedData();
}
buffer[o++] = (byte) b;
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
SafeUtils.checkRange(b, off, len);
ensureNotFinished();
while (o + len > blockSize) {
final int l = blockSize - o;
System.arraycopy(b, off, buffer, o, blockSize - o);
o = blockSize;
flushBufferedData();
off += l;
len -= l;
}
System.arraycopy(b, off, buffer, o, len);
o += len;
}
@Override
public void write(byte[] b) throws IOException {
ensureNotFinished();
write(b, 0, b.length);
}
@Override
public void close() throws IOException {
if (!finished) {
finish();
}
if (out != null) {
out.close();
out = null;
}
}
private void flushBufferedData() throws IOException {
if (o == 0) {
return;
}
checksum.reset();
checksum.update(buffer, 0, o);
final int check = (int) checksum.getValue();
int compressedLength = compressor.compress(buffer, 0, o, compressedBuffer, HEADER_LENGTH);
final int compressMethod;
if (compressedLength >= o) {
compressMethod = COMPRESSION_METHOD_RAW;
compressedLength = o;
System.arraycopy(buffer, 0, compressedBuffer, HEADER_LENGTH, o);
} else {
compressMethod = COMPRESSION_METHOD_LZ4;
}
compressedBuffer[MAGIC_LENGTH] = (byte) (compressMethod | compressionLevel);
writeIntLE(compressedLength, compressedBuffer, MAGIC_LENGTH + 1);
writeIntLE(o, compressedBuffer, MAGIC_LENGTH + 5);
writeIntLE(check, compressedBuffer, MAGIC_LENGTH + 9);
assert MAGIC_LENGTH + 13 == HEADER_LENGTH;
out.write(compressedBuffer, 0, HEADER_LENGTH + compressedLength);
o = 0;
}
/**
* Flush this compressed {@link OutputStream}.
*
* If the stream has been created with <code>syncFlush=true</code>, pending
* data will be compressed and appended to the underlying {@link OutputStream}
* before calling {@link OutputStream#flush()} on the underlying stream.
* Otherwise, this method just flushes the underlying stream, so pending
* data might not be available for reading until {@link #finish()} or
* {@link #close()} is called.
*/
@Override
public void flush() throws IOException {
if (out != null) {
if (syncFlush) {
flushBufferedData();
}
out.flush();
}
}
/**
* Same as {@link #close()} except that it doesn't close the underlying stream.
* This can be useful if you want to keep on using the underlying stream.
*/
public void finish() throws IOException {
ensureNotFinished();
flushBufferedData();
compressedBuffer[MAGIC_LENGTH] = (byte) (COMPRESSION_METHOD_RAW | compressionLevel);
writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 1);
writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 5);
writeIntLE(0, compressedBuffer, MAGIC_LENGTH + 9);
assert MAGIC_LENGTH + 13 == HEADER_LENGTH;
out.write(compressedBuffer, 0, HEADER_LENGTH);
finished = true;
out.flush();
}
private static void writeIntLE(int i, byte[] buf, int off) {
buf[off++] = (byte) i;
buf[off++] = (byte) (i >>> 8);
buf[off++] = (byte) (i >>> 16);
buf[off++] = (byte) (i >>> 24);
}
@Override
public String toString() {
return getClass().getSimpleName() + "(out=" + out + ", blockSize=" + blockSize
+ ", compressor=" + compressor + ", checksum=" + checksum + ")";
}
}

View File

@ -0,0 +1,237 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.lz4.LZ4Constants.COPY_LENGTH;
import static net.jpountz.lz4.LZ4Constants.LAST_LITERALS;
import static net.jpountz.lz4.LZ4Constants.ML_BITS;
import static net.jpountz.lz4.LZ4Constants.ML_MASK;
import static net.jpountz.lz4.LZ4Constants.RUN_MASK;
import static net.jpountz.util.ByteBufferUtils.readByte;
import static net.jpountz.util.ByteBufferUtils.readInt;
import static net.jpountz.util.ByteBufferUtils.readLong;
import static net.jpountz.util.ByteBufferUtils.writeByte;
import static net.jpountz.util.ByteBufferUtils.writeInt;
import static net.jpountz.util.ByteBufferUtils.writeLong;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
enum LZ4ByteBufferUtils {
;
static int hash(ByteBuffer buf, int i) {
return LZ4Utils.hash(readInt(buf, i));
}
static int hash64k(ByteBuffer buf, int i) {
return LZ4Utils.hash64k(readInt(buf, i));
}
static boolean readIntEquals(ByteBuffer buf, int i, int j) {
return buf.getInt(i) == buf.getInt(j);
}
static void safeIncrementalCopy(ByteBuffer dest, int matchOff, int dOff, int matchLen) {
for (int i = 0; i < matchLen; ++i) {
dest.put(dOff + i, dest.get(matchOff + i));
}
}
static void wildIncrementalCopy(ByteBuffer dest, int matchOff, int dOff, int matchCopyEnd) {
if (dOff - matchOff < 4) {
for (int i = 0; i < 4; ++i) {
writeByte(dest, dOff+i, readByte(dest, matchOff+i));
}
dOff += 4;
matchOff += 4;
int dec = 0;
assert dOff >= matchOff && dOff - matchOff < 8;
switch (dOff - matchOff) {
case 1:
matchOff -= 3;
break;
case 2:
matchOff -= 2;
break;
case 3:
matchOff -= 3;
dec = -1;
break;
case 5:
dec = 1;
break;
case 6:
dec = 2;
break;
case 7:
dec = 3;
break;
default:
break;
}
writeInt(dest, dOff, readInt(dest, matchOff));
dOff += 4;
matchOff -= dec;
} else if (dOff - matchOff < COPY_LENGTH) {
writeLong(dest, dOff, readLong(dest, matchOff));
dOff += dOff - matchOff;
}
while (dOff < matchCopyEnd) {
writeLong(dest, dOff, readLong(dest, matchOff));
dOff += 8;
matchOff += 8;
}
}
static int commonBytes(ByteBuffer src, int ref, int sOff, int srcLimit) {
int matchLen = 0;
while (sOff <= srcLimit - 8) {
if (readLong(src, sOff) == readLong(src, ref)) {
matchLen += 8;
ref += 8;
sOff += 8;
} else {
final int zeroBits;
if (src.order() == ByteOrder.BIG_ENDIAN) {
zeroBits = Long.numberOfLeadingZeros(readLong(src, sOff) ^ readLong(src, ref));
} else {
zeroBits = Long.numberOfTrailingZeros(readLong(src, sOff) ^ readLong(src, ref));
}
return matchLen + (zeroBits >>> 3);
}
}
while (sOff < srcLimit && readByte(src, ref++) == readByte(src, sOff++)) {
++matchLen;
}
return matchLen;
}
static int commonBytesBackward(ByteBuffer b, int o1, int o2, int l1, int l2) {
int count = 0;
while (o1 > l1 && o2 > l2 && b.get(--o1) == b.get(--o2)) {
++count;
}
return count;
}
static void safeArraycopy(ByteBuffer src, int sOff, ByteBuffer dest, int dOff, int len) {
for (int i = 0; i < len; ++i) {
dest.put(dOff + i, src.get(sOff + i));
}
}
static void wildArraycopy(ByteBuffer src, int sOff, ByteBuffer dest, int dOff, int len) {
assert src.order().equals(dest.order());
try {
for (int i = 0; i < len; i += 8) {
dest.putLong(dOff + i, src.getLong(sOff + i));
}
} catch (IndexOutOfBoundsException e) {
throw new LZ4Exception("Malformed input at offset " + sOff);
}
}
static int encodeSequence(ByteBuffer src, int anchor, int matchOff, int matchRef, int matchLen, ByteBuffer dest, int dOff, int destEnd) {
final int runLen = matchOff - anchor;
final int tokenOff = dOff++;
if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
int token;
if (runLen >= RUN_MASK) {
token = (byte) (RUN_MASK << ML_BITS);
dOff = writeLen(runLen - RUN_MASK, dest, dOff);
} else {
token = runLen << ML_BITS;
}
// copy literals
wildArraycopy(src, anchor, dest, dOff, runLen);
dOff += runLen;
// encode offset
final int matchDec = matchOff - matchRef;
dest.put(dOff++, (byte) matchDec);
dest.put(dOff++, (byte) (matchDec >>> 8));
// encode match len
matchLen -= 4;
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
if (matchLen >= ML_MASK) {
token |= ML_MASK;
dOff = writeLen(matchLen - RUN_MASK, dest, dOff);
} else {
token |= matchLen;
}
dest.put(tokenOff, (byte) token);
return dOff;
}
static int lastLiterals(ByteBuffer src, int sOff, int srcLen, ByteBuffer dest, int dOff, int destEnd) {
final int runLen = srcLen;
if (dOff + runLen + 1 + (runLen + 255 - RUN_MASK) / 255 > destEnd) {
throw new LZ4Exception();
}
if (runLen >= RUN_MASK) {
dest.put(dOff++, (byte) (RUN_MASK << ML_BITS));
dOff = writeLen(runLen - RUN_MASK, dest, dOff);
} else {
dest.put(dOff++, (byte) (runLen << ML_BITS));
}
// copy literals
safeArraycopy(src, sOff, dest, dOff, runLen);
dOff += runLen;
return dOff;
}
static int writeLen(int len, ByteBuffer dest, int dOff) {
while (len >= 0xFF) {
dest.put(dOff++, (byte) 0xFF);
len -= 0xFF;
}
dest.put(dOff++, (byte) len);
return dOff;
}
static class Match {
int start, ref, len;
void fix(int correction) {
start += correction;
ref += correction;
len -= correction;
}
int end() {
return start + len;
}
}
static void copyTo(Match m1, Match m2) {
m2.len = m1.len;
m2.start = m1.start;
m2.ref = m1.ref;
}
}

View File

@ -0,0 +1,126 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteBuffer;
import java.util.Arrays;
/**
* LZ4 compressor.
* <p>
* Instances of this class are thread-safe.
*/
public abstract class LZ4Compressor {
/** Return the maximum compressed length for an input of size <code>length</code>. */
@SuppressWarnings("static-method")
public final int maxCompressedLength(int length) {
return LZ4Utils.maxCompressedLength(length);
}
/**
* Compress <code>src[srcOff:srcOff+srcLen]</code> into
* <code>dest[destOff:destOff+destLen]</code> and return the compressed
* length.
*
* This method will throw a {@link LZ4Exception} if this compressor is unable
* to compress the input into less than <code>maxDestLen</code> bytes. To
* prevent this exception to be thrown, you should make sure that
* <code>maxDestLen >= maxCompressedLength(srcLen)</code>.
*
* @throws LZ4Exception if maxDestLen is too small
* @return the compressed size
*/
public abstract int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen);
/**
* Compress <code>src[srcOff:srcOff+srcLen]</code> into
* <code>dest[destOff:destOff+destLen]</code> and return the compressed
* length.
*
* This method will throw a {@link LZ4Exception} if this compressor is unable
* to compress the input into less than <code>maxDestLen</code> bytes. To
* prevent this exception to be thrown, you should make sure that
* <code>maxDestLen >= maxCompressedLength(srcLen)</code>.
*
* {@link ByteBuffer} positions remain unchanged.
*
* @throws LZ4Exception if maxDestLen is too small
* @return the compressed size
*/
public abstract int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen);
/**
* Convenience method, equivalent to calling
* {@link #compress(byte[], int, int, byte[], int, int) compress(src, srcOff, srcLen, dest, destOff, dest.length - destOff)}.
*/
public final int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff) {
return compress(src, srcOff, srcLen, dest, destOff, dest.length - destOff);
}
/**
* Convenience method, equivalent to calling
* {@link #compress(byte[], int, int, byte[], int) compress(src, 0, src.length, dest, 0)}.
*/
public final int compress(byte[] src, byte[] dest) {
return compress(src, 0, src.length, dest, 0);
}
/**
* Convenience method which returns <code>src[srcOff:srcOff+srcLen]</code>
* compressed.
* <p><b><span style="color:red">Warning</span></b>: this method has an
* important overhead due to the fact that it needs to allocate a buffer to
* compress into, and then needs to resize this buffer to the actual
* compressed length.</p>
* <p>Here is how this method is implemented:</p>
* <pre>
* final int maxCompressedLength = maxCompressedLength(srcLen);
* final byte[] compressed = new byte[maxCompressedLength];
* final int compressedLength = compress(src, srcOff, srcLen, compressed, 0);
* return Arrays.copyOf(compressed, compressedLength);
* </pre>
*/
public final byte[] compress(byte[] src, int srcOff, int srcLen) {
final int maxCompressedLength = maxCompressedLength(srcLen);
final byte[] compressed = new byte[maxCompressedLength];
final int compressedLength = compress(src, srcOff, srcLen, compressed, 0);
return Arrays.copyOf(compressed, compressedLength);
}
/**
* Convenience method, equivalent to calling
* {@link #compress(byte[], int, int) compress(src, 0, src.length)}.
*/
public final byte[] compress(byte[] src) {
return compress(src, 0, src.length);
}
/**
* Compress <code>src</code> into <code>dest</code>. Calling this method
* will update the positions of both {@link ByteBuffer}s.
*/
public final void compress(ByteBuffer src, ByteBuffer dest) {
final int cpLen = compress(src, src.position(), src.remaining(), dest, dest.position(), dest.remaining());
src.position(src.limit());
dest.position(dest.position() + cpLen);
}
@Override
public String toString() {
return getClass().getSimpleName();
}
}

View File

@ -0,0 +1,53 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
enum LZ4Constants {
;
static final int DEFAULT_COMPRESSION_LEVEL = 8+1;
static final int MAX_COMPRESSION_LEVEL = 16+1;
static final int MEMORY_USAGE = 14;
static final int NOT_COMPRESSIBLE_DETECTION_LEVEL = 6;
static final int MIN_MATCH = 4;
static final int HASH_LOG = MEMORY_USAGE - 2;
static final int HASH_TABLE_SIZE = 1 << HASH_LOG;
static final int SKIP_STRENGTH = Math.max(NOT_COMPRESSIBLE_DETECTION_LEVEL, 2);
static final int COPY_LENGTH = 8;
static final int LAST_LITERALS = 5;
static final int MF_LIMIT = COPY_LENGTH + MIN_MATCH;
static final int MIN_LENGTH = MF_LIMIT + 1;
static final int MAX_DISTANCE = 1 << 16;
static final int ML_BITS = 4;
static final int ML_MASK = (1 << ML_BITS) - 1;
static final int RUN_BITS = 8 - ML_BITS;
static final int RUN_MASK = (1 << RUN_BITS) - 1;
static final int LZ4_64K_LIMIT = (1 << 16) + (MF_LIMIT - 1);
static final int HASH_LOG_64K = HASH_LOG + 1;
static final int HASH_TABLE_SIZE_64K = 1 << HASH_LOG_64K;
static final int HASH_LOG_HC = 15;
static final int HASH_TABLE_SIZE_HC = 1 << HASH_LOG_HC;
static final int OPTIMAL_ML = ML_MASK - 1 + MIN_MATCH;
}

View File

@ -0,0 +1,25 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @deprecated Use {@link LZ4FastDecompressor} instead.
*/
@Deprecated
public interface LZ4Decompressor {
int decompress(byte[] src, int srcOff, byte[] dest, int destOff, int destLen);
}

View File

@ -0,0 +1,36 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* LZ4 compression or decompression error.
*/
public class LZ4Exception extends RuntimeException {
private static final long serialVersionUID = 1L;
public LZ4Exception(String msg, Throwable t) {
super(msg, t);
}
public LZ4Exception(String msg) {
super(msg);
}
public LZ4Exception() {
super();
}
}

View File

@ -0,0 +1,258 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import net.jpountz.util.Native;
import net.jpountz.util.Utils;
import static net.jpountz.lz4.LZ4Constants.DEFAULT_COMPRESSION_LEVEL;
import static net.jpountz.lz4.LZ4Constants.MAX_COMPRESSION_LEVEL;
/**
* Entry point for the LZ4 API.
* <p>
* This class has 3 instances<ul>
* <li>a {@link #nativeInstance() native} instance which is a JNI binding to
* <a href="http://code.google.com/p/lz4/">the original LZ4 C implementation</a>.
* <li>a {@link #safeInstance() safe Java} instance which is a pure Java port
* of the original C library,</li>
* <li>an {@link #unsafeInstance() unsafe Java} instance which is a Java port
* using the unofficial {@link sun.misc.Unsafe} API.
* </ul>
* <p>
* Only the {@link #safeInstance() safe instance} is guaranteed to work on your
* JVM, as a consequence it is advised to use the {@link #fastestInstance()} or
* {@link #fastestJavaInstance()} to pull a {@link LZ4Factory} instance.
* <p>
* All methods from this class are very costly, so you should get an instance
* once, and then reuse it whenever possible. This is typically done by storing
* a {@link LZ4Factory} instance in a static field.
*/
public final class LZ4Factory {
private static LZ4Factory instance(String impl) {
try {
return new LZ4Factory(impl);
} catch (Exception e) {
throw new AssertionError(e);
}
}
private static LZ4Factory NATIVE_INSTANCE,
JAVA_UNSAFE_INSTANCE,
JAVA_SAFE_INSTANCE;
/**
* Return a {@link LZ4Factory} instance that returns compressors and
* decompressors that are native bindings to the original C library.
* <p>
* Please note that this instance has some traps you should be aware of:<ol>
* <li>Upon loading this instance, files will be written to the temporary
* directory of the system. Although these files are supposed to be deleted
* when the JVM exits, they might remain on systems that don't support
* removal of files being used such as Windows.
* <li>The instance can only be loaded once per JVM. This can be a problem
* if your application uses multiple class loaders (such as most servlet
* containers): this instance will only be available to the children of the
* class loader which has loaded it. As a consequence, it is advised to
* either not use this instance in webapps or to put this library in the lib
* directory of your servlet container so that it is loaded by the system
* class loader.
* </ol>
*/
public static synchronized LZ4Factory nativeInstance() {
if (NATIVE_INSTANCE == null) {
NATIVE_INSTANCE = instance("JNI");
}
return NATIVE_INSTANCE;
}
/** Return a {@link LZ4Factory} instance that returns compressors and
* decompressors that are written with Java's official API. */
public static synchronized LZ4Factory safeInstance() {
if (JAVA_SAFE_INSTANCE == null) {
JAVA_SAFE_INSTANCE = instance("JavaSafe");
}
return JAVA_SAFE_INSTANCE;
}
/** Return a {@link LZ4Factory} instance that returns compressors and
* decompressors that may use {@link sun.misc.Unsafe} to speed up compression
* and decompression. */
public static synchronized LZ4Factory unsafeInstance() {
if (JAVA_UNSAFE_INSTANCE == null) {
JAVA_UNSAFE_INSTANCE = instance("JavaUnsafe");
}
return JAVA_UNSAFE_INSTANCE;
}
/**
* Return the fastest available {@link LZ4Factory} instance which does not
* rely on JNI bindings. It first tries to load the
* {@link #unsafeInstance() unsafe instance}, and then the
* {@link #safeInstance() safe Java instance} if the JVM doesn't have a
* working {@link sun.misc.Unsafe}.
*/
public static LZ4Factory fastestJavaInstance() {
if (Utils.isUnalignedAccessAllowed()) {
try {
return unsafeInstance();
} catch (Throwable t) {
return safeInstance();
}
} else {
return safeInstance();
}
}
/**
* Return the fastest available {@link LZ4Factory} instance. If the class
* loader is the system class loader and if the
* {@link #nativeInstance() native instance} loads successfully, then the
* {@link #nativeInstance() native instance} is returned, otherwise the
* {@link #fastestJavaInstance() fastest Java instance} is returned.
* <p>
* Please read {@link #nativeInstance() javadocs of nativeInstance()} before
* using this method.
*/
public static LZ4Factory fastestInstance() {
if (Native.isLoaded()
|| Native.class.getClassLoader() == ClassLoader.getSystemClassLoader()) {
try {
return nativeInstance();
} catch (Throwable t) {
return fastestJavaInstance();
}
} else {
return fastestJavaInstance();
}
}
@SuppressWarnings("unchecked")
private static <T> T classInstance(String cls) throws NoSuchFieldException, SecurityException, ClassNotFoundException, IllegalArgumentException, IllegalAccessException {
ClassLoader loader = LZ4Factory.class.getClassLoader();
loader = loader == null ? ClassLoader.getSystemClassLoader() : loader;
final Class<?> c = loader.loadClass(cls);
Field f = c.getField("INSTANCE");
return (T) f.get(null);
}
private final String impl;
private final LZ4Compressor fastCompressor;
private final LZ4Compressor highCompressor;
private final LZ4FastDecompressor fastDecompressor;
private final LZ4SafeDecompressor safeDecompressor;
private final LZ4Compressor[] highCompressors = new LZ4Compressor[MAX_COMPRESSION_LEVEL+1];
private LZ4Factory(String impl) throws ClassNotFoundException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, NoSuchMethodException, InstantiationException, InvocationTargetException {
this.impl = impl;
fastCompressor = classInstance("net.jpountz.lz4.LZ4" + impl + "Compressor");
highCompressor = classInstance("net.jpountz.lz4.LZ4HC" + impl + "Compressor");
fastDecompressor = classInstance("net.jpountz.lz4.LZ4" + impl + "FastDecompressor");
safeDecompressor = classInstance("net.jpountz.lz4.LZ4" + impl + "SafeDecompressor");
Constructor<? extends LZ4Compressor> highConstructor = highCompressor.getClass().getDeclaredConstructor(int.class);
highCompressors[DEFAULT_COMPRESSION_LEVEL] = highCompressor;
for(int level = 1; level <= MAX_COMPRESSION_LEVEL; level++) {
if(level == DEFAULT_COMPRESSION_LEVEL) continue;
highCompressors[level] = highConstructor.newInstance(level);
}
// quickly test that everything works as expected
final byte[] original = new byte[] {'a','b','c','d',' ',' ',' ',' ',' ',' ','a','b','c','d','e','f','g','h','i','j'};
for (LZ4Compressor compressor : Arrays.asList(fastCompressor, highCompressor)) {
final int maxCompressedLength = compressor.maxCompressedLength(original.length);
final byte[] compressed = new byte[maxCompressedLength];
final int compressedLength = compressor.compress(original, 0, original.length, compressed, 0, maxCompressedLength);
final byte[] restored = new byte[original.length];
fastDecompressor.decompress(compressed, 0, restored, 0, original.length);
if (!Arrays.equals(original, restored)) {
throw new AssertionError();
}
Arrays.fill(restored, (byte) 0);
final int decompressedLength = safeDecompressor.decompress(compressed, 0, compressedLength, restored, 0);
if (decompressedLength != original.length || !Arrays.equals(original, restored)) {
throw new AssertionError();
}
}
}
/** Return a blazing fast {@link LZ4Compressor}. */
public LZ4Compressor fastCompressor() {
return fastCompressor;
}
/** Return a {@link LZ4Compressor} which requires more memory than
* {@link #fastCompressor()} and is slower but compresses more efficiently. */
public LZ4Compressor highCompressor() {
return highCompressor;
}
/** Return a {@link LZ4Compressor} which requires more memory than
* {@link #fastCompressor()} and is slower but compresses more efficiently.
* The compression level can be customized.
* <p>For current implementations, the following is true about compression level:<ol>
* <li>It should be in range [1, 17]</li>
* <li>A compression level higher than 17 would be treated as 17.</li>
* <li>A compression level lower than 1 would be treated as 9.</li>
* </ol></p>
*/
public LZ4Compressor highCompressor(int compressionLevel) {
if(compressionLevel > MAX_COMPRESSION_LEVEL) {
compressionLevel = MAX_COMPRESSION_LEVEL;
} else if(compressionLevel < 1) {
compressionLevel = DEFAULT_COMPRESSION_LEVEL;
}
return highCompressors[compressionLevel];
}
/** Return a {@link LZ4FastDecompressor} instance. */
public LZ4FastDecompressor fastDecompressor() {
return fastDecompressor;
}
/** Return a {@link LZ4SafeDecompressor} instance. */
public LZ4SafeDecompressor safeDecompressor() {
return safeDecompressor;
}
/** Return a {@link LZ4UnknownSizeDecompressor} instance.
* @deprecated use {@link #safeDecompressor()} */
public LZ4UnknownSizeDecompressor unknownSizeDecompressor() {
return safeDecompressor();
}
/** Return a {@link LZ4Decompressor} instance.
* @deprecated use {@link #fastDecompressor()} */
public LZ4Decompressor decompressor() {
return fastDecompressor();
}
/** Prints the fastest instance. */
public static void main(String[] args) {
System.out.println("Fastest instance is " + fastestInstance());
System.out.println("Fastest Java instance is " + fastestJavaInstance());
}
@Override
public String toString() {
return getClass().getSimpleName() + ":" + impl;
}
}

View File

@ -0,0 +1,106 @@
package net.jpountz.lz4;
import java.nio.ByteBuffer;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* LZ4 decompressor that requires the size of the original input to be known.
* Use {@link LZ4SafeDecompressor} if you only know the size of the
* compressed stream.
* <p>
* Instances of this class are thread-safe.
*/
public abstract class LZ4FastDecompressor implements LZ4Decompressor {
/** Decompress <code>src[srcOff:]</code> into <code>dest[destOff:destOff+destLen]</code>
* and return the number of bytes read from <code>src</code>.
* <code>destLen</code> must be exactly the size of the decompressed data.
*
* @param destLen the <b>exact</b> size of the original input
* @return the number of bytes read to restore the original input
*/
public abstract int decompress(byte[] src, int srcOff, byte[] dest, int destOff, int destLen);
/** Decompress <code>src[srcOff:]</code> into <code>dest[destOff:destOff+destLen]</code>
* and return the number of bytes read from <code>src</code>.
* <code>destLen</code> must be exactly the size of the decompressed data.
* The positions and limits of the {@link ByteBuffer}s remain unchanged.
*
* @param destLen the <b>exact</b> size of the original input
* @return the number of bytes read to restore the original input
*/
public abstract int decompress(ByteBuffer src, int srcOff, ByteBuffer dest, int destOff, int destLen);
/**
* Convenience method, equivalent to calling
* {@link #decompress(byte[], int, byte[], int, int) decompress(src, 0, dest, 0, destLen)}.
*/
public final int decompress(byte[] src, byte[] dest, int destLen) {
return decompress(src, 0, dest, 0, destLen);
}
/**
* Convenience method, equivalent to calling
* {@link #decompress(byte[], byte[], int) decompress(src, dest, dest.length)}.
*/
public final int decompress(byte[] src, byte[] dest) {
return decompress(src, dest, dest.length);
}
/**
* Convenience method which returns <code>src[srcOff:?]</code>
* decompressed.
* <p><b><span style="color:red">Warning</span></b>: this method has an
* important overhead due to the fact that it needs to allocate a buffer to
* decompress into.</p>
* <p>Here is how this method is implemented:</p>
* <pre>
* final byte[] decompressed = new byte[destLen];
* decompress(src, srcOff, decompressed, 0, destLen);
* return decompressed;
* </pre>
*/
public final byte[] decompress(byte[] src, int srcOff, int destLen) {
final byte[] decompressed = new byte[destLen];
decompress(src, srcOff, decompressed, 0, destLen);
return decompressed;
}
/**
* Convenience method, equivalent to calling
* {@link #decompress(byte[], int, int) decompress(src, 0, destLen)}.
*/
public final byte[] decompress(byte[] src, int destLen) {
return decompress(src, 0, destLen);
}
/**
* Decompress <code>src</code> into <code>dest</code>. <code>dest</code>'s
* {@link ByteBuffer#remaining()} must be exactly the size of the decompressed
* data. This method moves the positions of the buffers.
*/
public final void decompress(ByteBuffer src, ByteBuffer dest) {
final int read = decompress(src, src.position(), dest, dest.position(), dest.remaining());
dest.position(dest.limit());
src.position(src.position() + read);
}
@Override
public String toString() {
return getClass().getSimpleName();
}
}

View File

@ -0,0 +1,88 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.lz4.LZ4Constants.DEFAULT_COMPRESSION_LEVEL;
import java.nio.ByteBuffer;
import net.jpountz.util.ByteBufferUtils;
import net.jpountz.util.SafeUtils;
/**
* High compression {@link LZ4Compressor}s implemented with JNI bindings to the
* original C implementation of LZ4.
*/
final class LZ4HCJNICompressor extends LZ4Compressor {
public static final LZ4HCJNICompressor INSTANCE = new LZ4HCJNICompressor();
private static LZ4Compressor SAFE_INSTANCE;
private final int compressionLevel;
LZ4HCJNICompressor() { this(DEFAULT_COMPRESSION_LEVEL); }
LZ4HCJNICompressor(int compressionLevel) {
this.compressionLevel = compressionLevel;
}
@Override
public int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
SafeUtils.checkRange(src, srcOff, srcLen);
SafeUtils.checkRange(dest, destOff, maxDestLen);
final int result = LZ4JNI.LZ4_compressHC(src, null, srcOff, srcLen, dest, null, destOff, maxDestLen, compressionLevel);
if (result <= 0) {
throw new LZ4Exception();
}
return result;
}
@Override
public int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
ByteBufferUtils.checkNotReadOnly(dest);
ByteBufferUtils.checkRange(src, srcOff, srcLen);
ByteBufferUtils.checkRange(dest, destOff, maxDestLen);
if ((src.hasArray() || src.isDirect()) && (dest.hasArray() || dest.isDirect())) {
byte[] srcArr = null, destArr = null;
ByteBuffer srcBuf = null, destBuf = null;
if (src.hasArray()) {
srcArr = src.array();
srcOff += src.arrayOffset();
} else {
assert src.isDirect();
srcBuf = src;
}
if (dest.hasArray()) {
destArr = dest.array();
destOff += dest.arrayOffset();
} else {
assert dest.isDirect();
destBuf = dest;
}
final int result = LZ4JNI.LZ4_compressHC(srcArr, srcBuf, srcOff, srcLen, destArr, destBuf, destOff, maxDestLen, compressionLevel);
if (result <= 0) {
throw new LZ4Exception();
}
return result;
} else {
LZ4Compressor safeInstance = SAFE_INSTANCE;
if (safeInstance == null) {
safeInstance = SAFE_INSTANCE = LZ4Factory.safeInstance().highCompressor(compressionLevel);
}
return safeInstance.compress(src, srcOff, srcLen, dest, destOff, maxDestLen);
}
}
}

View File

@ -0,0 +1,550 @@
// Auto-generated: DO NOT EDIT
package net.jpountz.lz4;
import static net.jpountz.lz4.LZ4Constants.*;
import static net.jpountz.lz4.LZ4Utils.*;
import java.nio.ByteBuffer;
import java.util.Arrays;
import net.jpountz.lz4.LZ4Utils.Match;
import net.jpountz.util.ByteBufferUtils;
import net.jpountz.util.SafeUtils;
/**
* High compression compressor.
*/
final class LZ4HCJavaSafeCompressor extends LZ4Compressor {
public static final LZ4Compressor INSTANCE = new LZ4HCJavaSafeCompressor();
private final int maxAttempts;
final int compressionLevel;
LZ4HCJavaSafeCompressor() { this(DEFAULT_COMPRESSION_LEVEL); }
LZ4HCJavaSafeCompressor(int compressionLevel) {
this.maxAttempts = 1<<(compressionLevel-1);
this.compressionLevel = compressionLevel;
}
private class HashTable {
static final int MASK = MAX_DISTANCE - 1;
int nextToUpdate;
private final int base;
private final int[] hashTable;
private final short[] chainTable;
HashTable(int base) {
this.base = base;
nextToUpdate = base;
hashTable = new int[HASH_TABLE_SIZE_HC];
Arrays.fill(hashTable, -1);
chainTable = new short[MAX_DISTANCE];
}
private int hashPointer(byte[] bytes, int off) {
final int v = SafeUtils.readInt(bytes, off);
return hashPointer(v);
}
private int hashPointer(ByteBuffer bytes, int off) {
final int v = ByteBufferUtils.readInt(bytes, off);
return hashPointer(v);
}
private int hashPointer(int v) {
final int h = hashHC(v);
return hashTable[h];
}
private int next(int off) {
return off - (chainTable[off & MASK] & 0xFFFF);
}
private void addHash(byte[] bytes, int off) {
final int v = SafeUtils.readInt(bytes, off);
addHash(v, off);
}
private void addHash(ByteBuffer bytes, int off) {
final int v = ByteBufferUtils.readInt(bytes, off);
addHash(v, off);
}
private void addHash(int v, int off) {
final int h = hashHC(v);
int delta = off - hashTable[h];
assert delta > 0 : delta;
if (delta >= MAX_DISTANCE) {
delta = MAX_DISTANCE - 1;
}
chainTable[off & MASK] = (short) delta;
hashTable[h] = off;
}
void insert(int off, byte[] bytes) {
for (; nextToUpdate < off; ++nextToUpdate) {
addHash(bytes, nextToUpdate);
}
}
void insert(int off, ByteBuffer bytes) {
for (; nextToUpdate < off; ++nextToUpdate) {
addHash(bytes, nextToUpdate);
}
}
boolean insertAndFindBestMatch(byte[] buf, int off, int matchLimit, Match match) {
match.start = off;
match.len = 0;
int delta = 0;
int repl = 0;
insert(off, buf);
int ref = hashPointer(buf, off);
if (ref >= off - 4 && ref <= off && ref >= base) { // potential repetition
if (LZ4SafeUtils.readIntEquals(buf, ref, off)) { // confirmed
delta = off - ref;
repl = match.len = MIN_MATCH + LZ4SafeUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
match.ref = ref;
}
ref = next(ref);
}
for (int i = 0; i < maxAttempts; ++i) {
if (ref < Math.max(base, off - MAX_DISTANCE + 1) || ref > off) {
break;
}
if (LZ4SafeUtils.readIntEquals(buf, ref, off)) {
final int matchLen = MIN_MATCH + LZ4SafeUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
if (matchLen > match.len) {
match.ref = ref;
match.len = matchLen;
}
}
ref = next(ref);
}
if (repl != 0) {
int ptr = off;
final int end = off + repl - (MIN_MATCH - 1);
while (ptr < end - delta) {
chainTable[ptr & MASK] = (short) delta; // pre load
++ptr;
}
do {
chainTable[ptr & MASK] = (short) delta;
hashTable[hashHC(SafeUtils.readInt(buf, ptr))] = ptr;
++ptr;
} while (ptr < end);
nextToUpdate = end;
}
return match.len != 0;
}
boolean insertAndFindWiderMatch(byte[] buf, int off, int startLimit, int matchLimit, int minLen, Match match) {
match.len = minLen;
insert(off, buf);
final int delta = off - startLimit;
int ref = hashPointer(buf, off);
for (int i = 0; i < maxAttempts; ++i) {
if (ref < Math.max(base, off - MAX_DISTANCE + 1) || ref > off) {
break;
}
if (LZ4SafeUtils.readIntEquals(buf, ref, off)) {
final int matchLenForward = MIN_MATCH +LZ4SafeUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
final int matchLenBackward = LZ4SafeUtils.commonBytesBackward(buf, ref, off, base, startLimit);
final int matchLen = matchLenBackward + matchLenForward;
if (matchLen > match.len) {
match.len = matchLen;
match.ref = ref - matchLenBackward;
match.start = off - matchLenBackward;
}
}
ref = next(ref);
}
return match.len > minLen;
}
boolean insertAndFindBestMatch(ByteBuffer buf, int off, int matchLimit, Match match) {
match.start = off;
match.len = 0;
int delta = 0;
int repl = 0;
insert(off, buf);
int ref = hashPointer(buf, off);
if (ref >= off - 4 && ref <= off && ref >= base) { // potential repetition
if (LZ4ByteBufferUtils.readIntEquals(buf, ref, off)) { // confirmed
delta = off - ref;
repl = match.len = MIN_MATCH + LZ4ByteBufferUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
match.ref = ref;
}
ref = next(ref);
}
for (int i = 0; i < maxAttempts; ++i) {
if (ref < Math.max(base, off - MAX_DISTANCE + 1) || ref > off) {
break;
}
if (LZ4ByteBufferUtils.readIntEquals(buf, ref, off)) {
final int matchLen = MIN_MATCH + LZ4ByteBufferUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
if (matchLen > match.len) {
match.ref = ref;
match.len = matchLen;
}
}
ref = next(ref);
}
if (repl != 0) {
int ptr = off;
final int end = off + repl - (MIN_MATCH - 1);
while (ptr < end - delta) {
chainTable[ptr & MASK] = (short) delta; // pre load
++ptr;
}
do {
chainTable[ptr & MASK] = (short) delta;
hashTable[hashHC(ByteBufferUtils.readInt(buf, ptr))] = ptr;
++ptr;
} while (ptr < end);
nextToUpdate = end;
}
return match.len != 0;
}
boolean insertAndFindWiderMatch(ByteBuffer buf, int off, int startLimit, int matchLimit, int minLen, Match match) {
match.len = minLen;
insert(off, buf);
final int delta = off - startLimit;
int ref = hashPointer(buf, off);
for (int i = 0; i < maxAttempts; ++i) {
if (ref < Math.max(base, off - MAX_DISTANCE + 1) || ref > off) {
break;
}
if (LZ4ByteBufferUtils.readIntEquals(buf, ref, off)) {
final int matchLenForward = MIN_MATCH +LZ4ByteBufferUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
final int matchLenBackward = LZ4ByteBufferUtils.commonBytesBackward(buf, ref, off, base, startLimit);
final int matchLen = matchLenBackward + matchLenForward;
if (matchLen > match.len) {
match.len = matchLen;
match.ref = ref - matchLenBackward;
match.start = off - matchLenBackward;
}
}
ref = next(ref);
}
return match.len > minLen;
}
}
@Override
public int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
SafeUtils.checkRange(src, srcOff, srcLen);
SafeUtils.checkRange(dest, destOff, maxDestLen);
final int srcEnd = srcOff + srcLen;
final int destEnd = destOff + maxDestLen;
final int mfLimit = srcEnd - MF_LIMIT;
final int matchLimit = srcEnd - LAST_LITERALS;
int sOff = srcOff;
int dOff = destOff;
int anchor = sOff++;
final HashTable ht = new HashTable(srcOff);
final Match match0 = new Match();
final Match match1 = new Match();
final Match match2 = new Match();
final Match match3 = new Match();
main:
while (sOff < mfLimit) {
if (!ht.insertAndFindBestMatch(src, sOff, matchLimit, match1)) {
++sOff;
continue;
}
// saved, in case we would skip too much
copyTo(match1, match0);
search2:
while (true) {
assert match1.start >= anchor;
if (match1.end() >= mfLimit
|| !ht.insertAndFindWiderMatch(src, match1.end() - 2, match1.start + 1, matchLimit, match1.len, match2)) {
// no better match
dOff = LZ4SafeUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
continue main;
}
if (match0.start < match1.start) {
if (match2.start < match1.start + match0.len) { // empirical
copyTo(match0, match1);
}
}
assert match2.start > match1.start;
if (match2.start - match1.start < 3) { // First Match too small : removed
copyTo(match2, match1);
continue search2;
}
search3:
while (true) {
if (match2.start - match1.start < OPTIMAL_ML) {
int newMatchLen = match1.len;
if (newMatchLen > OPTIMAL_ML) {
newMatchLen = OPTIMAL_ML;
}
if (match1.start + newMatchLen > match2.end() - MIN_MATCH) {
newMatchLen = match2.start - match1.start + match2.len - MIN_MATCH;
}
final int correction = newMatchLen - (match2.start - match1.start);
if (correction > 0) {
match2.fix(correction);
}
}
if (match2.start + match2.len >= mfLimit
|| !ht.insertAndFindWiderMatch(src, match2.end() - 3, match2.start, matchLimit, match2.len, match3)) {
// no better match -> 2 sequences to encode
if (match2.start < match1.end()) {
match1.len = match2.start - match1.start;
}
// encode seq 1
dOff = LZ4SafeUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
// encode seq 2
dOff = LZ4SafeUtils.encodeSequence(src, anchor, match2.start, match2.ref, match2.len, dest, dOff, destEnd);
anchor = sOff = match2.end();
continue main;
}
if (match3.start < match1.end() + 3) { // Not enough space for match 2 : remove it
if (match3.start >= match1.end()) { // // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1
if (match2.start < match1.end()) {
final int correction = match1.end() - match2.start;
match2.fix(correction);
if (match2.len < MIN_MATCH) {
copyTo(match3, match2);
}
}
dOff = LZ4SafeUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
copyTo(match3, match1);
copyTo(match2, match0);
continue search2;
}
copyTo(match3, match2);
continue search3;
}
// OK, now we have 3 ascending matches; let's write at least the first one
if (match2.start < match1.end()) {
if (match2.start - match1.start < ML_MASK) {
if (match1.len > OPTIMAL_ML) {
match1.len = OPTIMAL_ML;
}
if (match1.end() > match2.end() - MIN_MATCH) {
match1.len = match2.end() - match1.start - MIN_MATCH;
}
final int correction = match1.end() - match2.start;
match2.fix(correction);
} else {
match1.len = match2.start - match1.start;
}
}
dOff = LZ4SafeUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
copyTo(match2, match1);
copyTo(match3, match2);
continue search3;
}
}
}
dOff = LZ4SafeUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
@Override
public int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
if (src.hasArray() && dest.hasArray()) {
return compress(src.array(), srcOff + src.arrayOffset(), srcLen, dest.array(), destOff + dest.arrayOffset(), maxDestLen);
}
src = ByteBufferUtils.inNativeByteOrder(src);
dest = ByteBufferUtils.inNativeByteOrder(dest);
ByteBufferUtils.checkRange(src, srcOff, srcLen);
ByteBufferUtils.checkRange(dest, destOff, maxDestLen);
final int srcEnd = srcOff + srcLen;
final int destEnd = destOff + maxDestLen;
final int mfLimit = srcEnd - MF_LIMIT;
final int matchLimit = srcEnd - LAST_LITERALS;
int sOff = srcOff;
int dOff = destOff;
int anchor = sOff++;
final HashTable ht = new HashTable(srcOff);
final Match match0 = new Match();
final Match match1 = new Match();
final Match match2 = new Match();
final Match match3 = new Match();
main:
while (sOff < mfLimit) {
if (!ht.insertAndFindBestMatch(src, sOff, matchLimit, match1)) {
++sOff;
continue;
}
// saved, in case we would skip too much
copyTo(match1, match0);
search2:
while (true) {
assert match1.start >= anchor;
if (match1.end() >= mfLimit
|| !ht.insertAndFindWiderMatch(src, match1.end() - 2, match1.start + 1, matchLimit, match1.len, match2)) {
// no better match
dOff = LZ4ByteBufferUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
continue main;
}
if (match0.start < match1.start) {
if (match2.start < match1.start + match0.len) { // empirical
copyTo(match0, match1);
}
}
assert match2.start > match1.start;
if (match2.start - match1.start < 3) { // First Match too small : removed
copyTo(match2, match1);
continue search2;
}
search3:
while (true) {
if (match2.start - match1.start < OPTIMAL_ML) {
int newMatchLen = match1.len;
if (newMatchLen > OPTIMAL_ML) {
newMatchLen = OPTIMAL_ML;
}
if (match1.start + newMatchLen > match2.end() - MIN_MATCH) {
newMatchLen = match2.start - match1.start + match2.len - MIN_MATCH;
}
final int correction = newMatchLen - (match2.start - match1.start);
if (correction > 0) {
match2.fix(correction);
}
}
if (match2.start + match2.len >= mfLimit
|| !ht.insertAndFindWiderMatch(src, match2.end() - 3, match2.start, matchLimit, match2.len, match3)) {
// no better match -> 2 sequences to encode
if (match2.start < match1.end()) {
match1.len = match2.start - match1.start;
}
// encode seq 1
dOff = LZ4ByteBufferUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
// encode seq 2
dOff = LZ4ByteBufferUtils.encodeSequence(src, anchor, match2.start, match2.ref, match2.len, dest, dOff, destEnd);
anchor = sOff = match2.end();
continue main;
}
if (match3.start < match1.end() + 3) { // Not enough space for match 2 : remove it
if (match3.start >= match1.end()) { // // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1
if (match2.start < match1.end()) {
final int correction = match1.end() - match2.start;
match2.fix(correction);
if (match2.len < MIN_MATCH) {
copyTo(match3, match2);
}
}
dOff = LZ4ByteBufferUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
copyTo(match3, match1);
copyTo(match2, match0);
continue search2;
}
copyTo(match3, match2);
continue search3;
}
// OK, now we have 3 ascending matches; let's write at least the first one
if (match2.start < match1.end()) {
if (match2.start - match1.start < ML_MASK) {
if (match1.len > OPTIMAL_ML) {
match1.len = OPTIMAL_ML;
}
if (match1.end() > match2.end() - MIN_MATCH) {
match1.len = match2.end() - match1.start - MIN_MATCH;
}
final int correction = match1.end() - match2.start;
match2.fix(correction);
} else {
match1.len = match2.start - match1.start;
}
}
dOff = LZ4ByteBufferUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
copyTo(match2, match1);
copyTo(match3, match2);
continue search3;
}
}
}
dOff = LZ4ByteBufferUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
}

View File

@ -0,0 +1,550 @@
// Auto-generated: DO NOT EDIT
package net.jpountz.lz4;
import static net.jpountz.lz4.LZ4Constants.*;
import static net.jpountz.lz4.LZ4Utils.*;
import java.nio.ByteBuffer;
import java.util.Arrays;
import net.jpountz.lz4.LZ4Utils.Match;
import net.jpountz.util.ByteBufferUtils;
import net.jpountz.util.UnsafeUtils;
/**
* High compression compressor.
*/
final class LZ4HCJavaUnsafeCompressor extends LZ4Compressor {
public static final LZ4Compressor INSTANCE = new LZ4HCJavaUnsafeCompressor();
private final int maxAttempts;
final int compressionLevel;
LZ4HCJavaUnsafeCompressor() { this(DEFAULT_COMPRESSION_LEVEL); }
LZ4HCJavaUnsafeCompressor(int compressionLevel) {
this.maxAttempts = 1<<(compressionLevel-1);
this.compressionLevel = compressionLevel;
}
private class HashTable {
static final int MASK = MAX_DISTANCE - 1;
int nextToUpdate;
private final int base;
private final int[] hashTable;
private final short[] chainTable;
HashTable(int base) {
this.base = base;
nextToUpdate = base;
hashTable = new int[HASH_TABLE_SIZE_HC];
Arrays.fill(hashTable, -1);
chainTable = new short[MAX_DISTANCE];
}
private int hashPointer(byte[] bytes, int off) {
final int v = UnsafeUtils.readInt(bytes, off);
return hashPointer(v);
}
private int hashPointer(ByteBuffer bytes, int off) {
final int v = ByteBufferUtils.readInt(bytes, off);
return hashPointer(v);
}
private int hashPointer(int v) {
final int h = hashHC(v);
return hashTable[h];
}
private int next(int off) {
return off - (chainTable[off & MASK] & 0xFFFF);
}
private void addHash(byte[] bytes, int off) {
final int v = UnsafeUtils.readInt(bytes, off);
addHash(v, off);
}
private void addHash(ByteBuffer bytes, int off) {
final int v = ByteBufferUtils.readInt(bytes, off);
addHash(v, off);
}
private void addHash(int v, int off) {
final int h = hashHC(v);
int delta = off - hashTable[h];
assert delta > 0 : delta;
if (delta >= MAX_DISTANCE) {
delta = MAX_DISTANCE - 1;
}
chainTable[off & MASK] = (short) delta;
hashTable[h] = off;
}
void insert(int off, byte[] bytes) {
for (; nextToUpdate < off; ++nextToUpdate) {
addHash(bytes, nextToUpdate);
}
}
void insert(int off, ByteBuffer bytes) {
for (; nextToUpdate < off; ++nextToUpdate) {
addHash(bytes, nextToUpdate);
}
}
boolean insertAndFindBestMatch(byte[] buf, int off, int matchLimit, Match match) {
match.start = off;
match.len = 0;
int delta = 0;
int repl = 0;
insert(off, buf);
int ref = hashPointer(buf, off);
if (ref >= off - 4 && ref <= off && ref >= base) { // potential repetition
if (LZ4UnsafeUtils.readIntEquals(buf, ref, off)) { // confirmed
delta = off - ref;
repl = match.len = MIN_MATCH + LZ4UnsafeUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
match.ref = ref;
}
ref = next(ref);
}
for (int i = 0; i < maxAttempts; ++i) {
if (ref < Math.max(base, off - MAX_DISTANCE + 1) || ref > off) {
break;
}
if (LZ4UnsafeUtils.readIntEquals(buf, ref, off)) {
final int matchLen = MIN_MATCH + LZ4UnsafeUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
if (matchLen > match.len) {
match.ref = ref;
match.len = matchLen;
}
}
ref = next(ref);
}
if (repl != 0) {
int ptr = off;
final int end = off + repl - (MIN_MATCH - 1);
while (ptr < end - delta) {
chainTable[ptr & MASK] = (short) delta; // pre load
++ptr;
}
do {
chainTable[ptr & MASK] = (short) delta;
hashTable[hashHC(UnsafeUtils.readInt(buf, ptr))] = ptr;
++ptr;
} while (ptr < end);
nextToUpdate = end;
}
return match.len != 0;
}
boolean insertAndFindWiderMatch(byte[] buf, int off, int startLimit, int matchLimit, int minLen, Match match) {
match.len = minLen;
insert(off, buf);
final int delta = off - startLimit;
int ref = hashPointer(buf, off);
for (int i = 0; i < maxAttempts; ++i) {
if (ref < Math.max(base, off - MAX_DISTANCE + 1) || ref > off) {
break;
}
if (LZ4UnsafeUtils.readIntEquals(buf, ref, off)) {
final int matchLenForward = MIN_MATCH +LZ4UnsafeUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
final int matchLenBackward = LZ4UnsafeUtils.commonBytesBackward(buf, ref, off, base, startLimit);
final int matchLen = matchLenBackward + matchLenForward;
if (matchLen > match.len) {
match.len = matchLen;
match.ref = ref - matchLenBackward;
match.start = off - matchLenBackward;
}
}
ref = next(ref);
}
return match.len > minLen;
}
boolean insertAndFindBestMatch(ByteBuffer buf, int off, int matchLimit, Match match) {
match.start = off;
match.len = 0;
int delta = 0;
int repl = 0;
insert(off, buf);
int ref = hashPointer(buf, off);
if (ref >= off - 4 && ref <= off && ref >= base) { // potential repetition
if (LZ4ByteBufferUtils.readIntEquals(buf, ref, off)) { // confirmed
delta = off - ref;
repl = match.len = MIN_MATCH + LZ4ByteBufferUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
match.ref = ref;
}
ref = next(ref);
}
for (int i = 0; i < maxAttempts; ++i) {
if (ref < Math.max(base, off - MAX_DISTANCE + 1) || ref > off) {
break;
}
if (LZ4ByteBufferUtils.readIntEquals(buf, ref, off)) {
final int matchLen = MIN_MATCH + LZ4ByteBufferUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
if (matchLen > match.len) {
match.ref = ref;
match.len = matchLen;
}
}
ref = next(ref);
}
if (repl != 0) {
int ptr = off;
final int end = off + repl - (MIN_MATCH - 1);
while (ptr < end - delta) {
chainTable[ptr & MASK] = (short) delta; // pre load
++ptr;
}
do {
chainTable[ptr & MASK] = (short) delta;
hashTable[hashHC(ByteBufferUtils.readInt(buf, ptr))] = ptr;
++ptr;
} while (ptr < end);
nextToUpdate = end;
}
return match.len != 0;
}
boolean insertAndFindWiderMatch(ByteBuffer buf, int off, int startLimit, int matchLimit, int minLen, Match match) {
match.len = minLen;
insert(off, buf);
final int delta = off - startLimit;
int ref = hashPointer(buf, off);
for (int i = 0; i < maxAttempts; ++i) {
if (ref < Math.max(base, off - MAX_DISTANCE + 1) || ref > off) {
break;
}
if (LZ4ByteBufferUtils.readIntEquals(buf, ref, off)) {
final int matchLenForward = MIN_MATCH +LZ4ByteBufferUtils.commonBytes(buf, ref + MIN_MATCH, off + MIN_MATCH, matchLimit);
final int matchLenBackward = LZ4ByteBufferUtils.commonBytesBackward(buf, ref, off, base, startLimit);
final int matchLen = matchLenBackward + matchLenForward;
if (matchLen > match.len) {
match.len = matchLen;
match.ref = ref - matchLenBackward;
match.start = off - matchLenBackward;
}
}
ref = next(ref);
}
return match.len > minLen;
}
}
@Override
public int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
UnsafeUtils.checkRange(src, srcOff, srcLen);
UnsafeUtils.checkRange(dest, destOff, maxDestLen);
final int srcEnd = srcOff + srcLen;
final int destEnd = destOff + maxDestLen;
final int mfLimit = srcEnd - MF_LIMIT;
final int matchLimit = srcEnd - LAST_LITERALS;
int sOff = srcOff;
int dOff = destOff;
int anchor = sOff++;
final HashTable ht = new HashTable(srcOff);
final Match match0 = new Match();
final Match match1 = new Match();
final Match match2 = new Match();
final Match match3 = new Match();
main:
while (sOff < mfLimit) {
if (!ht.insertAndFindBestMatch(src, sOff, matchLimit, match1)) {
++sOff;
continue;
}
// saved, in case we would skip too much
copyTo(match1, match0);
search2:
while (true) {
assert match1.start >= anchor;
if (match1.end() >= mfLimit
|| !ht.insertAndFindWiderMatch(src, match1.end() - 2, match1.start + 1, matchLimit, match1.len, match2)) {
// no better match
dOff = LZ4UnsafeUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
continue main;
}
if (match0.start < match1.start) {
if (match2.start < match1.start + match0.len) { // empirical
copyTo(match0, match1);
}
}
assert match2.start > match1.start;
if (match2.start - match1.start < 3) { // First Match too small : removed
copyTo(match2, match1);
continue search2;
}
search3:
while (true) {
if (match2.start - match1.start < OPTIMAL_ML) {
int newMatchLen = match1.len;
if (newMatchLen > OPTIMAL_ML) {
newMatchLen = OPTIMAL_ML;
}
if (match1.start + newMatchLen > match2.end() - MIN_MATCH) {
newMatchLen = match2.start - match1.start + match2.len - MIN_MATCH;
}
final int correction = newMatchLen - (match2.start - match1.start);
if (correction > 0) {
match2.fix(correction);
}
}
if (match2.start + match2.len >= mfLimit
|| !ht.insertAndFindWiderMatch(src, match2.end() - 3, match2.start, matchLimit, match2.len, match3)) {
// no better match -> 2 sequences to encode
if (match2.start < match1.end()) {
match1.len = match2.start - match1.start;
}
// encode seq 1
dOff = LZ4UnsafeUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
// encode seq 2
dOff = LZ4UnsafeUtils.encodeSequence(src, anchor, match2.start, match2.ref, match2.len, dest, dOff, destEnd);
anchor = sOff = match2.end();
continue main;
}
if (match3.start < match1.end() + 3) { // Not enough space for match 2 : remove it
if (match3.start >= match1.end()) { // // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1
if (match2.start < match1.end()) {
final int correction = match1.end() - match2.start;
match2.fix(correction);
if (match2.len < MIN_MATCH) {
copyTo(match3, match2);
}
}
dOff = LZ4UnsafeUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
copyTo(match3, match1);
copyTo(match2, match0);
continue search2;
}
copyTo(match3, match2);
continue search3;
}
// OK, now we have 3 ascending matches; let's write at least the first one
if (match2.start < match1.end()) {
if (match2.start - match1.start < ML_MASK) {
if (match1.len > OPTIMAL_ML) {
match1.len = OPTIMAL_ML;
}
if (match1.end() > match2.end() - MIN_MATCH) {
match1.len = match2.end() - match1.start - MIN_MATCH;
}
final int correction = match1.end() - match2.start;
match2.fix(correction);
} else {
match1.len = match2.start - match1.start;
}
}
dOff = LZ4UnsafeUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
copyTo(match2, match1);
copyTo(match3, match2);
continue search3;
}
}
}
dOff = LZ4UnsafeUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
@Override
public int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
if (src.hasArray() && dest.hasArray()) {
return compress(src.array(), srcOff + src.arrayOffset(), srcLen, dest.array(), destOff + dest.arrayOffset(), maxDestLen);
}
src = ByteBufferUtils.inNativeByteOrder(src);
dest = ByteBufferUtils.inNativeByteOrder(dest);
ByteBufferUtils.checkRange(src, srcOff, srcLen);
ByteBufferUtils.checkRange(dest, destOff, maxDestLen);
final int srcEnd = srcOff + srcLen;
final int destEnd = destOff + maxDestLen;
final int mfLimit = srcEnd - MF_LIMIT;
final int matchLimit = srcEnd - LAST_LITERALS;
int sOff = srcOff;
int dOff = destOff;
int anchor = sOff++;
final HashTable ht = new HashTable(srcOff);
final Match match0 = new Match();
final Match match1 = new Match();
final Match match2 = new Match();
final Match match3 = new Match();
main:
while (sOff < mfLimit) {
if (!ht.insertAndFindBestMatch(src, sOff, matchLimit, match1)) {
++sOff;
continue;
}
// saved, in case we would skip too much
copyTo(match1, match0);
search2:
while (true) {
assert match1.start >= anchor;
if (match1.end() >= mfLimit
|| !ht.insertAndFindWiderMatch(src, match1.end() - 2, match1.start + 1, matchLimit, match1.len, match2)) {
// no better match
dOff = LZ4ByteBufferUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
continue main;
}
if (match0.start < match1.start) {
if (match2.start < match1.start + match0.len) { // empirical
copyTo(match0, match1);
}
}
assert match2.start > match1.start;
if (match2.start - match1.start < 3) { // First Match too small : removed
copyTo(match2, match1);
continue search2;
}
search3:
while (true) {
if (match2.start - match1.start < OPTIMAL_ML) {
int newMatchLen = match1.len;
if (newMatchLen > OPTIMAL_ML) {
newMatchLen = OPTIMAL_ML;
}
if (match1.start + newMatchLen > match2.end() - MIN_MATCH) {
newMatchLen = match2.start - match1.start + match2.len - MIN_MATCH;
}
final int correction = newMatchLen - (match2.start - match1.start);
if (correction > 0) {
match2.fix(correction);
}
}
if (match2.start + match2.len >= mfLimit
|| !ht.insertAndFindWiderMatch(src, match2.end() - 3, match2.start, matchLimit, match2.len, match3)) {
// no better match -> 2 sequences to encode
if (match2.start < match1.end()) {
match1.len = match2.start - match1.start;
}
// encode seq 1
dOff = LZ4ByteBufferUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
// encode seq 2
dOff = LZ4ByteBufferUtils.encodeSequence(src, anchor, match2.start, match2.ref, match2.len, dest, dOff, destEnd);
anchor = sOff = match2.end();
continue main;
}
if (match3.start < match1.end() + 3) { // Not enough space for match 2 : remove it
if (match3.start >= match1.end()) { // // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1
if (match2.start < match1.end()) {
final int correction = match1.end() - match2.start;
match2.fix(correction);
if (match2.len < MIN_MATCH) {
copyTo(match3, match2);
}
}
dOff = LZ4ByteBufferUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
copyTo(match3, match1);
copyTo(match2, match0);
continue search2;
}
copyTo(match3, match2);
continue search3;
}
// OK, now we have 3 ascending matches; let's write at least the first one
if (match2.start < match1.end()) {
if (match2.start - match1.start < ML_MASK) {
if (match1.len > OPTIMAL_ML) {
match1.len = OPTIMAL_ML;
}
if (match1.end() > match2.end() - MIN_MATCH) {
match1.len = match2.end() - match1.start - MIN_MATCH;
}
final int correction = match1.end() - match2.start;
match2.fix(correction);
} else {
match1.len = match2.start - match1.start;
}
}
dOff = LZ4ByteBufferUtils.encodeSequence(src, anchor, match1.start, match1.ref, match1.len, dest, dOff, destEnd);
anchor = sOff = match1.end();
copyTo(match2, match1);
copyTo(match3, match2);
continue search3;
}
}
}
dOff = LZ4ByteBufferUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
}

View File

@ -0,0 +1,133 @@
package net.jpountz.lz4;
import java.io.IOException;
import java.io.InputStream;
public class LZ4InputStream extends InputStream {
private static LZ4Factory factory = LZ4Factory.fastestInstance();
private final InputStream inputStream;
private final LZ4Decompressor decompressor;
private byte compressedBuffer[] = new byte[1048576];
private byte decompressedBuffer[] = new byte[1048576];
private int decompressedBufferPosition = 0;
private int decompressedBufferLength = 0;
public LZ4InputStream(InputStream stream) {
this.decompressor = factory.decompressor();
this.inputStream = stream;
}
@Override
public void close() throws IOException {
inputStream.close();
}
@Override
public int read() throws IOException {
if (ensureBytesAvailableInDecompressedBuffer())
return decompressedBuffer[decompressedBufferPosition++] & 0xFF;
return -1;
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (!ensureBytesAvailableInDecompressedBuffer())
return -1;
int numBytesRemainingToRead = len - off;
while (numBytesRemainingToRead > 0 && ensureBytesAvailableInDecompressedBuffer()) {
int numBytesToRead = numBytesRemainingToRead;
int numBytesRemainingInBlock = decompressedBufferLength - decompressedBufferPosition;
if (numBytesToRead > numBytesRemainingInBlock) {
numBytesToRead = numBytesRemainingInBlock;
}
System.arraycopy(decompressedBuffer, decompressedBufferPosition, b, off, numBytesToRead);
decompressedBufferPosition += numBytesToRead;
off += numBytesToRead;
numBytesRemainingToRead -= numBytesToRead;
}
return len - numBytesRemainingToRead;
}
@Override
public long skip(long n) throws IOException {
long numBytesRemainingToSkip = n;
while (numBytesRemainingToSkip > 0 && ensureBytesAvailableInDecompressedBuffer()) {
long numBytesToSkip = numBytesRemainingToSkip;
int numBytesRemainingInBlock = decompressedBufferLength - decompressedBufferPosition;
if (numBytesToSkip > numBytesRemainingInBlock) {
numBytesToSkip = numBytesRemainingInBlock;
}
numBytesRemainingToSkip -= numBytesToSkip;
decompressedBufferPosition += numBytesToSkip;
}
return n - numBytesRemainingToSkip;
}
private boolean ensureBytesAvailableInDecompressedBuffer() throws IOException {
while (decompressedBufferPosition >= decompressedBufferLength) {
if (!fillBuffer()) {
return false;
}
}
return true;
}
private boolean fillBuffer() throws IOException {
decompressedBufferLength = LZ4StreamHelper.readLength(inputStream);
int compressedBufferLength = LZ4StreamHelper.readLength(inputStream);
if (blockHeadersIndicateNoMoreData(compressedBufferLength, decompressedBufferLength)) {
return false;
}
ensureBufferCapacity(compressedBufferLength, decompressedBufferLength);
if (fillCompressedBuffer(compressedBufferLength)) {
decompressor.decompress(compressedBuffer, 0, decompressedBuffer, 0, decompressedBufferLength);
decompressedBufferPosition = 0;
return true;
}
return false;
}
private boolean blockHeadersIndicateNoMoreData(int compressedBufferLength, int decompressedBufferLength) {
return compressedBufferLength < 0 || decompressedBufferLength < 0;
}
private boolean fillCompressedBuffer(int compressedBufferLength) throws IOException {
int bytesRead = 0;
while (bytesRead < compressedBufferLength) {
int bytesReadInAttempt = inputStream.read(compressedBuffer, bytesRead, compressedBufferLength - bytesRead);
if (bytesReadInAttempt < 0)
return false;
bytesRead += bytesReadInAttempt;
}
return true;
}
private void ensureBufferCapacity(int compressedBufferLength, int decompressedBufferLength) {
if (compressedBufferLength > compressedBuffer.length) {
compressedBuffer = new byte[compressedBufferLength];
}
if (decompressedBufferLength > decompressedBuffer.length) {
decompressedBuffer = new byte[decompressedBufferLength];
}
}
}

View File

@ -0,0 +1,41 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteBuffer;
import net.jpountz.util.Native;
/**
* JNI bindings to the original C implementation of LZ4.
*/
enum LZ4JNI {
;
static {
Native.load();
init();
}
static native void init();
static native int LZ4_compress_limitedOutput(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, int srcLen, byte[] destArray, ByteBuffer destBuffer, int destOff, int maxDestLen);
static native int LZ4_compressHC(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, int srcLen, byte[] destArray, ByteBuffer destBuffer, int destOff, int maxDestLen, int compressionLevel);
static native int LZ4_decompress_fast(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, byte[] destArray, ByteBuffer destBuffer, int destOff, int destLen);
static native int LZ4_decompress_safe(byte[] srcArray, ByteBuffer srcBuffer, int srcOff, int srcLen, byte[] destArray, ByteBuffer destBuffer, int destOff, int maxDestLen);
static native int LZ4_compressBound(int len);
}

View File

@ -0,0 +1,80 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.util.ByteBufferUtils.checkNotReadOnly;
import static net.jpountz.util.ByteBufferUtils.checkRange;
import static net.jpountz.util.SafeUtils.checkRange;
import java.nio.ByteBuffer;
/**
* Fast {@link LZ4FastCompressor}s implemented with JNI bindings to the original C
* implementation of LZ4.
*/
final class LZ4JNICompressor extends LZ4Compressor {
public static final LZ4Compressor INSTANCE = new LZ4JNICompressor();
private static LZ4Compressor SAFE_INSTANCE;
@Override
public int compress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
checkRange(src, srcOff, srcLen);
checkRange(dest, destOff, maxDestLen);
final int result = LZ4JNI.LZ4_compress_limitedOutput(src, null, srcOff, srcLen, dest, null, destOff, maxDestLen);
if (result <= 0) {
throw new LZ4Exception("maxDestLen is too small");
}
return result;
}
@Override
public int compress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
checkNotReadOnly(dest);
checkRange(src, srcOff, srcLen);
checkRange(dest, destOff, maxDestLen);
if ((src.hasArray() || src.isDirect()) && (dest.hasArray() || dest.isDirect())) {
byte[] srcArr = null, destArr = null;
ByteBuffer srcBuf = null, destBuf = null;
if (src.hasArray()) {
srcArr = src.array();
srcOff += src.arrayOffset();
} else {
assert src.isDirect();
srcBuf = src;
}
if (dest.hasArray()) {
destArr = dest.array();
destOff += dest.arrayOffset();
} else {
assert dest.isDirect();
destBuf = dest;
}
final int result = LZ4JNI.LZ4_compress_limitedOutput(srcArr, srcBuf, srcOff, srcLen, destArr, destBuf, destOff, maxDestLen);
if (result <= 0) {
throw new LZ4Exception("maxDestLen is too small");
}
return result;
} else {
LZ4Compressor safeInstance = SAFE_INSTANCE;
if (safeInstance == null) {
safeInstance = SAFE_INSTANCE = LZ4Factory.safeInstance().fastCompressor();
}
return safeInstance.compress(src, srcOff, srcLen, dest, destOff, maxDestLen);
}
}
}

View File

@ -0,0 +1,82 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteBuffer;
import net.jpountz.util.ByteBufferUtils;
import net.jpountz.util.SafeUtils;
/**
* {@link LZ4FastDecompressor} implemented with JNI bindings to the original C
* implementation of LZ4.
*/
final class LZ4JNIFastDecompressor extends LZ4FastDecompressor {
public static final LZ4JNIFastDecompressor INSTANCE = new LZ4JNIFastDecompressor();
private static LZ4FastDecompressor SAFE_INSTANCE;
@Override
public final int decompress(byte[] src, int srcOff, byte[] dest, int destOff, int destLen) {
SafeUtils.checkRange(src, srcOff);
SafeUtils.checkRange(dest, destOff, destLen);
final int result = LZ4JNI.LZ4_decompress_fast(src, null, srcOff, dest, null, destOff, destLen);
if (result < 0) {
throw new LZ4Exception("Error decoding offset " + (srcOff - result) + " of input buffer");
}
return result;
}
@Override
public int decompress(ByteBuffer src, int srcOff, ByteBuffer dest, int destOff, int destLen) {
ByteBufferUtils.checkNotReadOnly(dest);
ByteBufferUtils.checkRange(src, srcOff);
ByteBufferUtils.checkRange(dest, destOff, destLen);
if ((src.hasArray() || src.isDirect()) && (dest.hasArray() || dest.isDirect())) {
byte[] srcArr = null, destArr = null;
ByteBuffer srcBuf = null, destBuf = null;
if (src.hasArray()) {
srcArr = src.array();
srcOff += src.arrayOffset();
} else {
assert src.isDirect();
srcBuf = src;
}
if (dest.hasArray()) {
destArr = dest.array();
destOff += dest.arrayOffset();
} else {
assert dest.isDirect();
destBuf = dest;
}
final int result = LZ4JNI.LZ4_decompress_fast(srcArr, srcBuf, srcOff, destArr, destBuf, destOff, destLen);
if (result < 0) {
throw new LZ4Exception("Error decoding offset " + (srcOff - result) + " of input buffer");
}
return result;
} else {
LZ4FastDecompressor safeInstance = SAFE_INSTANCE;
if (safeInstance == null) {
safeInstance = SAFE_INSTANCE = LZ4Factory.safeInstance().fastDecompressor();
}
return safeInstance.decompress(src, srcOff, dest, destOff, destLen);
}
}
}

View File

@ -0,0 +1,81 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteBuffer;
import net.jpountz.util.ByteBufferUtils;
import net.jpountz.util.SafeUtils;
/**
* {@link LZ4SafeDecompressor} implemented with JNI bindings to the original C
* implementation of LZ4.
*/
final class LZ4JNISafeDecompressor extends LZ4SafeDecompressor {
public static final LZ4JNISafeDecompressor INSTANCE = new LZ4JNISafeDecompressor();
private static LZ4SafeDecompressor SAFE_INSTANCE;
@Override
public final int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen) {
SafeUtils.checkRange(src, srcOff, srcLen);
SafeUtils.checkRange(dest, destOff, maxDestLen);
final int result = LZ4JNI.LZ4_decompress_safe(src, null, srcOff, srcLen, dest, null, destOff, maxDestLen);
if (result < 0) {
throw new LZ4Exception("Error decoding offset " + (srcOff - result) + " of input buffer");
}
return result;
}
@Override
public int decompress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen) {
ByteBufferUtils.checkNotReadOnly(dest);
ByteBufferUtils.checkRange(src, srcOff, srcLen);
ByteBufferUtils.checkRange(dest, destOff, maxDestLen);
if ((src.hasArray() || src.isDirect()) && (dest.hasArray() || dest.isDirect())) {
byte[] srcArr = null, destArr = null;
ByteBuffer srcBuf = null, destBuf = null;
if (src.hasArray()) {
srcArr = src.array();
srcOff += src.arrayOffset();
} else {
assert src.isDirect();
srcBuf = src;
}
if (dest.hasArray()) {
destArr = dest.array();
destOff += dest.arrayOffset();
} else {
assert dest.isDirect();
destBuf = dest;
}
final int result = LZ4JNI.LZ4_decompress_safe(srcArr, srcBuf, srcOff, srcLen, destArr, destBuf, destOff, maxDestLen);
if (result < 0) {
throw new LZ4Exception("Error decoding offset " + (srcOff - result) + " of input buffer");
}
return result;
} else {
LZ4SafeDecompressor safeInstance = SAFE_INSTANCE;
if (safeInstance == null) {
safeInstance = SAFE_INSTANCE = LZ4Factory.safeInstance().safeDecompressor();
}
return safeInstance.decompress(src, srcOff, srcLen, dest, destOff, maxDestLen);
}
}
}

View File

@ -0,0 +1,511 @@
// Auto-generated: DO NOT EDIT
package net.jpountz.lz4;
import static net.jpountz.lz4.LZ4Constants.*;
import static net.jpountz.lz4.LZ4Utils.*;
import java.nio.ByteBuffer;
import java.util.Arrays;
import net.jpountz.util.ByteBufferUtils;
import net.jpountz.util.SafeUtils;
/**
* Compressor.
*/
final class LZ4JavaSafeCompressor extends LZ4Compressor {
public static final LZ4Compressor INSTANCE = new LZ4JavaSafeCompressor();
static int compress64k(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int destEnd) {
final int srcEnd = srcOff + srcLen;
final int srcLimit = srcEnd - LAST_LITERALS;
final int mflimit = srcEnd - MF_LIMIT;
int sOff = srcOff, dOff = destOff;
int anchor = sOff;
if (srcLen >= MIN_LENGTH) {
final short[] hashTable = new short[HASH_TABLE_SIZE_64K];
++sOff;
main:
while (true) {
// find a match
int forwardOff = sOff;
int ref;
int step = 1;
int searchMatchNb = 1 << SKIP_STRENGTH;
do {
sOff = forwardOff;
forwardOff += step;
step = searchMatchNb++ >>> SKIP_STRENGTH;
if (forwardOff > mflimit) {
break main;
}
final int h = hash64k(SafeUtils.readInt(src, sOff));
ref = srcOff + SafeUtils.readShort(hashTable, h);
SafeUtils.writeShort(hashTable, h, sOff - srcOff);
} while (!LZ4SafeUtils.readIntEquals(src, ref, sOff));
// catch up
final int excess = LZ4SafeUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor);
sOff -= excess;
ref -= excess;
// sequence == refsequence
final int runLen = sOff - anchor;
// encode literal length
int tokenOff = dOff++;
if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
if (runLen >= RUN_MASK) {
SafeUtils.writeByte(dest, tokenOff, RUN_MASK << ML_BITS);
dOff = LZ4SafeUtils.writeLen(runLen - RUN_MASK, dest, dOff);
} else {
SafeUtils.writeByte(dest, tokenOff, runLen << ML_BITS);
}
// copy literals
LZ4SafeUtils.wildArraycopy(src, anchor, dest, dOff, runLen);
dOff += runLen;
while (true) {
// encode offset
SafeUtils.writeShortLE(dest, dOff, (short) (sOff - ref));
dOff += 2;
// count nb matches
sOff += MIN_MATCH;
ref += MIN_MATCH;
final int matchLen = LZ4SafeUtils.commonBytes(src, ref, sOff, srcLimit);
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
sOff += matchLen;
// encode match len
if (matchLen >= ML_MASK) {
SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | ML_MASK);
dOff = LZ4SafeUtils.writeLen(matchLen - ML_MASK, dest, dOff);
} else {
SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | matchLen);
}
// test end of chunk
if (sOff > mflimit) {
anchor = sOff;
break main;
}
// fill table
SafeUtils.writeShort(hashTable, hash64k(SafeUtils.readInt(src, sOff - 2)), sOff - 2 - srcOff);
// test next position
final int h = hash64k(SafeUtils.readInt(src, sOff));
ref = srcOff + SafeUtils.readShort(hashTable, h);
SafeUtils.writeShort(hashTable, h, sOff - srcOff);
if (!LZ4SafeUtils.readIntEquals(src, sOff, ref)) {
break;
}
tokenOff = dOff++;
SafeUtils.writeByte(dest, tokenOff, 0);
}
// prepare next loop
anchor = sOff++;
}
}
dOff = LZ4SafeUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
@Override
public int compress(byte[] src, final int srcOff, int srcLen, byte[] dest, final int destOff, int maxDestLen) {
SafeUtils.checkRange(src, srcOff, srcLen);
SafeUtils.checkRange(dest, destOff, maxDestLen);
final int destEnd = destOff + maxDestLen;
if (srcLen < LZ4_64K_LIMIT) {
return compress64k(src, srcOff, srcLen, dest, destOff, destEnd);
}
final int srcEnd = srcOff + srcLen;
final int srcLimit = srcEnd - LAST_LITERALS;
final int mflimit = srcEnd - MF_LIMIT;
int sOff = srcOff, dOff = destOff;
int anchor = sOff++;
final int[] hashTable = new int[HASH_TABLE_SIZE];
Arrays.fill(hashTable, anchor);
main:
while (true) {
// find a match
int forwardOff = sOff;
int ref;
int step = 1;
int searchMatchNb = 1 << SKIP_STRENGTH;
int back;
do {
sOff = forwardOff;
forwardOff += step;
step = searchMatchNb++ >>> SKIP_STRENGTH;
if (forwardOff > mflimit) {
break main;
}
final int h = hash(SafeUtils.readInt(src, sOff));
ref = SafeUtils.readInt(hashTable, h);
back = sOff - ref;
SafeUtils.writeInt(hashTable, h, sOff);
} while (back >= MAX_DISTANCE || !LZ4SafeUtils.readIntEquals(src, ref, sOff));
final int excess = LZ4SafeUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor);
sOff -= excess;
ref -= excess;
// sequence == refsequence
final int runLen = sOff - anchor;
// encode literal length
int tokenOff = dOff++;
if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
if (runLen >= RUN_MASK) {
SafeUtils.writeByte(dest, tokenOff, RUN_MASK << ML_BITS);
dOff = LZ4SafeUtils.writeLen(runLen - RUN_MASK, dest, dOff);
} else {
SafeUtils.writeByte(dest, tokenOff, runLen << ML_BITS);
}
// copy literals
LZ4SafeUtils.wildArraycopy(src, anchor, dest, dOff, runLen);
dOff += runLen;
while (true) {
// encode offset
SafeUtils.writeShortLE(dest, dOff, back);
dOff += 2;
// count nb matches
sOff += MIN_MATCH;
final int matchLen = LZ4SafeUtils.commonBytes(src, ref + MIN_MATCH, sOff, srcLimit);
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
sOff += matchLen;
// encode match len
if (matchLen >= ML_MASK) {
SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | ML_MASK);
dOff = LZ4SafeUtils.writeLen(matchLen - ML_MASK, dest, dOff);
} else {
SafeUtils.writeByte(dest, tokenOff, SafeUtils.readByte(dest, tokenOff) | matchLen);
}
// test end of chunk
if (sOff > mflimit) {
anchor = sOff;
break main;
}
// fill table
SafeUtils.writeInt(hashTable, hash(SafeUtils.readInt(src, sOff - 2)), sOff - 2);
// test next position
final int h = hash(SafeUtils.readInt(src, sOff));
ref = SafeUtils.readInt(hashTable, h);
SafeUtils.writeInt(hashTable, h, sOff);
back = sOff - ref;
if (back >= MAX_DISTANCE || !LZ4SafeUtils.readIntEquals(src, ref, sOff)) {
break;
}
tokenOff = dOff++;
SafeUtils.writeByte(dest, tokenOff, 0);
}
// prepare next loop
anchor = sOff++;
}
dOff = LZ4SafeUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
static int compress64k(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int destEnd) {
final int srcEnd = srcOff + srcLen;
final int srcLimit = srcEnd - LAST_LITERALS;
final int mflimit = srcEnd - MF_LIMIT;
int sOff = srcOff, dOff = destOff;
int anchor = sOff;
if (srcLen >= MIN_LENGTH) {
final short[] hashTable = new short[HASH_TABLE_SIZE_64K];
++sOff;
main:
while (true) {
// find a match
int forwardOff = sOff;
int ref;
int step = 1;
int searchMatchNb = 1 << SKIP_STRENGTH;
do {
sOff = forwardOff;
forwardOff += step;
step = searchMatchNb++ >>> SKIP_STRENGTH;
if (forwardOff > mflimit) {
break main;
}
final int h = hash64k(ByteBufferUtils.readInt(src, sOff));
ref = srcOff + SafeUtils.readShort(hashTable, h);
SafeUtils.writeShort(hashTable, h, sOff - srcOff);
} while (!LZ4ByteBufferUtils.readIntEquals(src, ref, sOff));
// catch up
final int excess = LZ4ByteBufferUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor);
sOff -= excess;
ref -= excess;
// sequence == refsequence
final int runLen = sOff - anchor;
// encode literal length
int tokenOff = dOff++;
if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
if (runLen >= RUN_MASK) {
ByteBufferUtils.writeByte(dest, tokenOff, RUN_MASK << ML_BITS);
dOff = LZ4ByteBufferUtils.writeLen(runLen - RUN_MASK, dest, dOff);
} else {
ByteBufferUtils.writeByte(dest, tokenOff, runLen << ML_BITS);
}
// copy literals
LZ4ByteBufferUtils.wildArraycopy(src, anchor, dest, dOff, runLen);
dOff += runLen;
while (true) {
// encode offset
ByteBufferUtils.writeShortLE(dest, dOff, (short) (sOff - ref));
dOff += 2;
// count nb matches
sOff += MIN_MATCH;
ref += MIN_MATCH;
final int matchLen = LZ4ByteBufferUtils.commonBytes(src, ref, sOff, srcLimit);
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
sOff += matchLen;
// encode match len
if (matchLen >= ML_MASK) {
ByteBufferUtils.writeByte(dest, tokenOff, ByteBufferUtils.readByte(dest, tokenOff) | ML_MASK);
dOff = LZ4ByteBufferUtils.writeLen(matchLen - ML_MASK, dest, dOff);
} else {
ByteBufferUtils.writeByte(dest, tokenOff, ByteBufferUtils.readByte(dest, tokenOff) | matchLen);
}
// test end of chunk
if (sOff > mflimit) {
anchor = sOff;
break main;
}
// fill table
SafeUtils.writeShort(hashTable, hash64k(ByteBufferUtils.readInt(src, sOff - 2)), sOff - 2 - srcOff);
// test next position
final int h = hash64k(ByteBufferUtils.readInt(src, sOff));
ref = srcOff + SafeUtils.readShort(hashTable, h);
SafeUtils.writeShort(hashTable, h, sOff - srcOff);
if (!LZ4ByteBufferUtils.readIntEquals(src, sOff, ref)) {
break;
}
tokenOff = dOff++;
ByteBufferUtils.writeByte(dest, tokenOff, 0);
}
// prepare next loop
anchor = sOff++;
}
}
dOff = LZ4ByteBufferUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
@Override
public int compress(ByteBuffer src, final int srcOff, int srcLen, ByteBuffer dest, final int destOff, int maxDestLen) {
if (src.hasArray() && dest.hasArray()) {
return compress(src.array(), srcOff + src.arrayOffset(), srcLen, dest.array(), destOff + dest.arrayOffset(), maxDestLen);
}
src = ByteBufferUtils.inNativeByteOrder(src);
dest = ByteBufferUtils.inNativeByteOrder(dest);
ByteBufferUtils.checkRange(src, srcOff, srcLen);
ByteBufferUtils.checkRange(dest, destOff, maxDestLen);
final int destEnd = destOff + maxDestLen;
if (srcLen < LZ4_64K_LIMIT) {
return compress64k(src, srcOff, srcLen, dest, destOff, destEnd);
}
final int srcEnd = srcOff + srcLen;
final int srcLimit = srcEnd - LAST_LITERALS;
final int mflimit = srcEnd - MF_LIMIT;
int sOff = srcOff, dOff = destOff;
int anchor = sOff++;
final int[] hashTable = new int[HASH_TABLE_SIZE];
Arrays.fill(hashTable, anchor);
main:
while (true) {
// find a match
int forwardOff = sOff;
int ref;
int step = 1;
int searchMatchNb = 1 << SKIP_STRENGTH;
int back;
do {
sOff = forwardOff;
forwardOff += step;
step = searchMatchNb++ >>> SKIP_STRENGTH;
if (forwardOff > mflimit) {
break main;
}
final int h = hash(ByteBufferUtils.readInt(src, sOff));
ref = SafeUtils.readInt(hashTable, h);
back = sOff - ref;
SafeUtils.writeInt(hashTable, h, sOff);
} while (back >= MAX_DISTANCE || !LZ4ByteBufferUtils.readIntEquals(src, ref, sOff));
final int excess = LZ4ByteBufferUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor);
sOff -= excess;
ref -= excess;
// sequence == refsequence
final int runLen = sOff - anchor;
// encode literal length
int tokenOff = dOff++;
if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
if (runLen >= RUN_MASK) {
ByteBufferUtils.writeByte(dest, tokenOff, RUN_MASK << ML_BITS);
dOff = LZ4ByteBufferUtils.writeLen(runLen - RUN_MASK, dest, dOff);
} else {
ByteBufferUtils.writeByte(dest, tokenOff, runLen << ML_BITS);
}
// copy literals
LZ4ByteBufferUtils.wildArraycopy(src, anchor, dest, dOff, runLen);
dOff += runLen;
while (true) {
// encode offset
ByteBufferUtils.writeShortLE(dest, dOff, back);
dOff += 2;
// count nb matches
sOff += MIN_MATCH;
final int matchLen = LZ4ByteBufferUtils.commonBytes(src, ref + MIN_MATCH, sOff, srcLimit);
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
sOff += matchLen;
// encode match len
if (matchLen >= ML_MASK) {
ByteBufferUtils.writeByte(dest, tokenOff, ByteBufferUtils.readByte(dest, tokenOff) | ML_MASK);
dOff = LZ4ByteBufferUtils.writeLen(matchLen - ML_MASK, dest, dOff);
} else {
ByteBufferUtils.writeByte(dest, tokenOff, ByteBufferUtils.readByte(dest, tokenOff) | matchLen);
}
// test end of chunk
if (sOff > mflimit) {
anchor = sOff;
break main;
}
// fill table
SafeUtils.writeInt(hashTable, hash(ByteBufferUtils.readInt(src, sOff - 2)), sOff - 2);
// test next position
final int h = hash(ByteBufferUtils.readInt(src, sOff));
ref = SafeUtils.readInt(hashTable, h);
SafeUtils.writeInt(hashTable, h, sOff);
back = sOff - ref;
if (back >= MAX_DISTANCE || !LZ4ByteBufferUtils.readIntEquals(src, ref, sOff)) {
break;
}
tokenOff = dOff++;
ByteBufferUtils.writeByte(dest, tokenOff, 0);
}
// prepare next loop
anchor = sOff++;
}
dOff = LZ4ByteBufferUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
}

View File

@ -0,0 +1,205 @@
// Auto-generated: DO NOT EDIT
package net.jpountz.lz4;
import static net.jpountz.lz4.LZ4Constants.*;
import java.nio.ByteBuffer;
import net.jpountz.util.ByteBufferUtils;
import net.jpountz.util.SafeUtils;
/**
* Decompressor.
*/
final class LZ4JavaSafeFastDecompressor extends LZ4FastDecompressor {
public static final LZ4FastDecompressor INSTANCE = new LZ4JavaSafeFastDecompressor();
@Override
public int decompress(byte[] src, final int srcOff, byte[] dest, final int destOff, int destLen) {
SafeUtils.checkRange(src, srcOff);
SafeUtils.checkRange(dest, destOff, destLen);
if (destLen == 0) {
if (SafeUtils.readByte(src, srcOff) != 0) {
throw new LZ4Exception("Malformed input at " + srcOff);
}
return 1;
}
final int destEnd = destOff + destLen;
int sOff = srcOff;
int dOff = destOff;
while (true) {
final int token = SafeUtils.readByte(src, sOff) & 0xFF;
++sOff;
// literals
int literalLen = token >>> ML_BITS;
if (literalLen == RUN_MASK) {
byte len = (byte) 0xFF;
while ((len = SafeUtils.readByte(src, sOff++)) == (byte) 0xFF) {
literalLen += 0xFF;
}
literalLen += len & 0xFF;
}
final int literalCopyEnd = dOff + literalLen;
if (literalCopyEnd > destEnd - COPY_LENGTH) {
if (literalCopyEnd != destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
} else {
LZ4SafeUtils.safeArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
break; // EOF
}
}
LZ4SafeUtils.wildArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
// matchs
final int matchDec = SafeUtils.readShortLE(src, sOff);
sOff += 2;
int matchOff = dOff - matchDec;
if (matchOff < destOff) {
throw new LZ4Exception("Malformed input at " + sOff);
}
int matchLen = token & ML_MASK;
if (matchLen == ML_MASK) {
byte len = (byte) 0xFF;
while ((len = SafeUtils.readByte(src, sOff++)) == (byte) 0xFF) {
matchLen += 0xFF;
}
matchLen += len & 0xFF;
}
matchLen += MIN_MATCH;
final int matchCopyEnd = dOff + matchLen;
if (matchCopyEnd > destEnd - COPY_LENGTH) {
if (matchCopyEnd > destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
}
LZ4SafeUtils.safeIncrementalCopy(dest, matchOff, dOff, matchLen);
} else {
LZ4SafeUtils.wildIncrementalCopy(dest, matchOff, dOff, matchCopyEnd);
}
dOff = matchCopyEnd;
}
return sOff - srcOff;
}
@Override
public int decompress(ByteBuffer src, final int srcOff, ByteBuffer dest, final int destOff, int destLen) {
if (src.hasArray() && dest.hasArray()) {
return decompress(src.array(), srcOff + src.arrayOffset(), dest.array(), destOff + dest.arrayOffset(), destLen);
}
src = ByteBufferUtils.inNativeByteOrder(src);
dest = ByteBufferUtils.inNativeByteOrder(dest);
ByteBufferUtils.checkRange(src, srcOff);
ByteBufferUtils.checkRange(dest, destOff, destLen);
if (destLen == 0) {
if (ByteBufferUtils.readByte(src, srcOff) != 0) {
throw new LZ4Exception("Malformed input at " + srcOff);
}
return 1;
}
final int destEnd = destOff + destLen;
int sOff = srcOff;
int dOff = destOff;
while (true) {
final int token = ByteBufferUtils.readByte(src, sOff) & 0xFF;
++sOff;
// literals
int literalLen = token >>> ML_BITS;
if (literalLen == RUN_MASK) {
byte len = (byte) 0xFF;
while ((len = ByteBufferUtils.readByte(src, sOff++)) == (byte) 0xFF) {
literalLen += 0xFF;
}
literalLen += len & 0xFF;
}
final int literalCopyEnd = dOff + literalLen;
if (literalCopyEnd > destEnd - COPY_LENGTH) {
if (literalCopyEnd != destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
} else {
LZ4ByteBufferUtils.safeArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
break; // EOF
}
}
LZ4ByteBufferUtils.wildArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
// matchs
final int matchDec = ByteBufferUtils.readShortLE(src, sOff);
sOff += 2;
int matchOff = dOff - matchDec;
if (matchOff < destOff) {
throw new LZ4Exception("Malformed input at " + sOff);
}
int matchLen = token & ML_MASK;
if (matchLen == ML_MASK) {
byte len = (byte) 0xFF;
while ((len = ByteBufferUtils.readByte(src, sOff++)) == (byte) 0xFF) {
matchLen += 0xFF;
}
matchLen += len & 0xFF;
}
matchLen += MIN_MATCH;
final int matchCopyEnd = dOff + matchLen;
if (matchCopyEnd > destEnd - COPY_LENGTH) {
if (matchCopyEnd > destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
}
LZ4ByteBufferUtils.safeIncrementalCopy(dest, matchOff, dOff, matchLen);
} else {
LZ4ByteBufferUtils.wildIncrementalCopy(dest, matchOff, dOff, matchCopyEnd);
}
dOff = matchCopyEnd;
}
return sOff - srcOff;
}
}

View File

@ -0,0 +1,213 @@
// Auto-generated: DO NOT EDIT
package net.jpountz.lz4;
import static net.jpountz.lz4.LZ4Constants.*;
import java.nio.ByteBuffer;
import net.jpountz.util.ByteBufferUtils;
import net.jpountz.util.SafeUtils;
/**
* Decompressor.
*/
final class LZ4JavaSafeSafeDecompressor extends LZ4SafeDecompressor {
public static final LZ4SafeDecompressor INSTANCE = new LZ4JavaSafeSafeDecompressor();
@Override
public int decompress(byte[] src, final int srcOff, final int srcLen , byte[] dest, final int destOff, int destLen) {
SafeUtils.checkRange(src, srcOff, srcLen);
SafeUtils.checkRange(dest, destOff, destLen);
if (destLen == 0) {
if (srcLen != 1 || SafeUtils.readByte(src, srcOff) != 0) {
throw new LZ4Exception("Output buffer too small");
}
return 0;
}
final int srcEnd = srcOff + srcLen;
final int destEnd = destOff + destLen;
int sOff = srcOff;
int dOff = destOff;
while (true) {
final int token = SafeUtils.readByte(src, sOff) & 0xFF;
++sOff;
// literals
int literalLen = token >>> ML_BITS;
if (literalLen == RUN_MASK) {
byte len = (byte) 0xFF;
while (sOff < srcEnd &&(len = SafeUtils.readByte(src, sOff++)) == (byte) 0xFF) {
literalLen += 0xFF;
}
literalLen += len & 0xFF;
}
final int literalCopyEnd = dOff + literalLen;
if (literalCopyEnd > destEnd - COPY_LENGTH || sOff + literalLen > srcEnd - COPY_LENGTH) {
if (literalCopyEnd > destEnd) {
throw new LZ4Exception();
} else if (sOff + literalLen != srcEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
} else {
LZ4SafeUtils.safeArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
break; // EOF
}
}
LZ4SafeUtils.wildArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
// matchs
final int matchDec = SafeUtils.readShortLE(src, sOff);
sOff += 2;
int matchOff = dOff - matchDec;
if (matchOff < destOff) {
throw new LZ4Exception("Malformed input at " + sOff);
}
int matchLen = token & ML_MASK;
if (matchLen == ML_MASK) {
byte len = (byte) 0xFF;
while (sOff < srcEnd &&(len = SafeUtils.readByte(src, sOff++)) == (byte) 0xFF) {
matchLen += 0xFF;
}
matchLen += len & 0xFF;
}
matchLen += MIN_MATCH;
final int matchCopyEnd = dOff + matchLen;
if (matchCopyEnd > destEnd - COPY_LENGTH) {
if (matchCopyEnd > destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
}
LZ4SafeUtils.safeIncrementalCopy(dest, matchOff, dOff, matchLen);
} else {
LZ4SafeUtils.wildIncrementalCopy(dest, matchOff, dOff, matchCopyEnd);
}
dOff = matchCopyEnd;
}
return dOff - destOff;
}
@Override
public int decompress(ByteBuffer src, final int srcOff, final int srcLen , ByteBuffer dest, final int destOff, int destLen) {
if (src.hasArray() && dest.hasArray()) {
return decompress(src.array(), srcOff + src.arrayOffset(), srcLen, dest.array(), destOff + dest.arrayOffset(), destLen);
}
src = ByteBufferUtils.inNativeByteOrder(src);
dest = ByteBufferUtils.inNativeByteOrder(dest);
ByteBufferUtils.checkRange(src, srcOff, srcLen);
ByteBufferUtils.checkRange(dest, destOff, destLen);
if (destLen == 0) {
if (srcLen != 1 || ByteBufferUtils.readByte(src, srcOff) != 0) {
throw new LZ4Exception("Output buffer too small");
}
return 0;
}
final int srcEnd = srcOff + srcLen;
final int destEnd = destOff + destLen;
int sOff = srcOff;
int dOff = destOff;
while (true) {
final int token = ByteBufferUtils.readByte(src, sOff) & 0xFF;
++sOff;
// literals
int literalLen = token >>> ML_BITS;
if (literalLen == RUN_MASK) {
byte len = (byte) 0xFF;
while (sOff < srcEnd &&(len = ByteBufferUtils.readByte(src, sOff++)) == (byte) 0xFF) {
literalLen += 0xFF;
}
literalLen += len & 0xFF;
}
final int literalCopyEnd = dOff + literalLen;
if (literalCopyEnd > destEnd - COPY_LENGTH || sOff + literalLen > srcEnd - COPY_LENGTH) {
if (literalCopyEnd > destEnd) {
throw new LZ4Exception();
} else if (sOff + literalLen != srcEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
} else {
LZ4ByteBufferUtils.safeArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
break; // EOF
}
}
LZ4ByteBufferUtils.wildArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
// matchs
final int matchDec = ByteBufferUtils.readShortLE(src, sOff);
sOff += 2;
int matchOff = dOff - matchDec;
if (matchOff < destOff) {
throw new LZ4Exception("Malformed input at " + sOff);
}
int matchLen = token & ML_MASK;
if (matchLen == ML_MASK) {
byte len = (byte) 0xFF;
while (sOff < srcEnd &&(len = ByteBufferUtils.readByte(src, sOff++)) == (byte) 0xFF) {
matchLen += 0xFF;
}
matchLen += len & 0xFF;
}
matchLen += MIN_MATCH;
final int matchCopyEnd = dOff + matchLen;
if (matchCopyEnd > destEnd - COPY_LENGTH) {
if (matchCopyEnd > destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
}
LZ4ByteBufferUtils.safeIncrementalCopy(dest, matchOff, dOff, matchLen);
} else {
LZ4ByteBufferUtils.wildIncrementalCopy(dest, matchOff, dOff, matchCopyEnd);
}
dOff = matchCopyEnd;
}
return dOff - destOff;
}
}

View File

@ -0,0 +1,511 @@
// Auto-generated: DO NOT EDIT
package net.jpountz.lz4;
import static net.jpountz.lz4.LZ4Constants.*;
import static net.jpountz.lz4.LZ4Utils.*;
import java.nio.ByteBuffer;
import java.util.Arrays;
import net.jpountz.util.ByteBufferUtils;
import net.jpountz.util.UnsafeUtils;
/**
* Compressor.
*/
final class LZ4JavaUnsafeCompressor extends LZ4Compressor {
public static final LZ4Compressor INSTANCE = new LZ4JavaUnsafeCompressor();
static int compress64k(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int destEnd) {
final int srcEnd = srcOff + srcLen;
final int srcLimit = srcEnd - LAST_LITERALS;
final int mflimit = srcEnd - MF_LIMIT;
int sOff = srcOff, dOff = destOff;
int anchor = sOff;
if (srcLen >= MIN_LENGTH) {
final short[] hashTable = new short[HASH_TABLE_SIZE_64K];
++sOff;
main:
while (true) {
// find a match
int forwardOff = sOff;
int ref;
int step = 1;
int searchMatchNb = 1 << SKIP_STRENGTH;
do {
sOff = forwardOff;
forwardOff += step;
step = searchMatchNb++ >>> SKIP_STRENGTH;
if (forwardOff > mflimit) {
break main;
}
final int h = hash64k(UnsafeUtils.readInt(src, sOff));
ref = srcOff + UnsafeUtils.readShort(hashTable, h);
UnsafeUtils.writeShort(hashTable, h, sOff - srcOff);
} while (!LZ4UnsafeUtils.readIntEquals(src, ref, sOff));
// catch up
final int excess = LZ4UnsafeUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor);
sOff -= excess;
ref -= excess;
// sequence == refsequence
final int runLen = sOff - anchor;
// encode literal length
int tokenOff = dOff++;
if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
if (runLen >= RUN_MASK) {
UnsafeUtils.writeByte(dest, tokenOff, RUN_MASK << ML_BITS);
dOff = LZ4UnsafeUtils.writeLen(runLen - RUN_MASK, dest, dOff);
} else {
UnsafeUtils.writeByte(dest, tokenOff, runLen << ML_BITS);
}
// copy literals
LZ4UnsafeUtils.wildArraycopy(src, anchor, dest, dOff, runLen);
dOff += runLen;
while (true) {
// encode offset
UnsafeUtils.writeShortLE(dest, dOff, (short) (sOff - ref));
dOff += 2;
// count nb matches
sOff += MIN_MATCH;
ref += MIN_MATCH;
final int matchLen = LZ4UnsafeUtils.commonBytes(src, ref, sOff, srcLimit);
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
sOff += matchLen;
// encode match len
if (matchLen >= ML_MASK) {
UnsafeUtils.writeByte(dest, tokenOff, UnsafeUtils.readByte(dest, tokenOff) | ML_MASK);
dOff = LZ4UnsafeUtils.writeLen(matchLen - ML_MASK, dest, dOff);
} else {
UnsafeUtils.writeByte(dest, tokenOff, UnsafeUtils.readByte(dest, tokenOff) | matchLen);
}
// test end of chunk
if (sOff > mflimit) {
anchor = sOff;
break main;
}
// fill table
UnsafeUtils.writeShort(hashTable, hash64k(UnsafeUtils.readInt(src, sOff - 2)), sOff - 2 - srcOff);
// test next position
final int h = hash64k(UnsafeUtils.readInt(src, sOff));
ref = srcOff + UnsafeUtils.readShort(hashTable, h);
UnsafeUtils.writeShort(hashTable, h, sOff - srcOff);
if (!LZ4UnsafeUtils.readIntEquals(src, sOff, ref)) {
break;
}
tokenOff = dOff++;
UnsafeUtils.writeByte(dest, tokenOff, 0);
}
// prepare next loop
anchor = sOff++;
}
}
dOff = LZ4UnsafeUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
@Override
public int compress(byte[] src, final int srcOff, int srcLen, byte[] dest, final int destOff, int maxDestLen) {
UnsafeUtils.checkRange(src, srcOff, srcLen);
UnsafeUtils.checkRange(dest, destOff, maxDestLen);
final int destEnd = destOff + maxDestLen;
if (srcLen < LZ4_64K_LIMIT) {
return compress64k(src, srcOff, srcLen, dest, destOff, destEnd);
}
final int srcEnd = srcOff + srcLen;
final int srcLimit = srcEnd - LAST_LITERALS;
final int mflimit = srcEnd - MF_LIMIT;
int sOff = srcOff, dOff = destOff;
int anchor = sOff++;
final int[] hashTable = new int[HASH_TABLE_SIZE];
Arrays.fill(hashTable, anchor);
main:
while (true) {
// find a match
int forwardOff = sOff;
int ref;
int step = 1;
int searchMatchNb = 1 << SKIP_STRENGTH;
int back;
do {
sOff = forwardOff;
forwardOff += step;
step = searchMatchNb++ >>> SKIP_STRENGTH;
if (forwardOff > mflimit) {
break main;
}
final int h = hash(UnsafeUtils.readInt(src, sOff));
ref = UnsafeUtils.readInt(hashTable, h);
back = sOff - ref;
UnsafeUtils.writeInt(hashTable, h, sOff);
} while (back >= MAX_DISTANCE || !LZ4UnsafeUtils.readIntEquals(src, ref, sOff));
final int excess = LZ4UnsafeUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor);
sOff -= excess;
ref -= excess;
// sequence == refsequence
final int runLen = sOff - anchor;
// encode literal length
int tokenOff = dOff++;
if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
if (runLen >= RUN_MASK) {
UnsafeUtils.writeByte(dest, tokenOff, RUN_MASK << ML_BITS);
dOff = LZ4UnsafeUtils.writeLen(runLen - RUN_MASK, dest, dOff);
} else {
UnsafeUtils.writeByte(dest, tokenOff, runLen << ML_BITS);
}
// copy literals
LZ4UnsafeUtils.wildArraycopy(src, anchor, dest, dOff, runLen);
dOff += runLen;
while (true) {
// encode offset
UnsafeUtils.writeShortLE(dest, dOff, back);
dOff += 2;
// count nb matches
sOff += MIN_MATCH;
final int matchLen = LZ4UnsafeUtils.commonBytes(src, ref + MIN_MATCH, sOff, srcLimit);
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
sOff += matchLen;
// encode match len
if (matchLen >= ML_MASK) {
UnsafeUtils.writeByte(dest, tokenOff, UnsafeUtils.readByte(dest, tokenOff) | ML_MASK);
dOff = LZ4UnsafeUtils.writeLen(matchLen - ML_MASK, dest, dOff);
} else {
UnsafeUtils.writeByte(dest, tokenOff, UnsafeUtils.readByte(dest, tokenOff) | matchLen);
}
// test end of chunk
if (sOff > mflimit) {
anchor = sOff;
break main;
}
// fill table
UnsafeUtils.writeInt(hashTable, hash(UnsafeUtils.readInt(src, sOff - 2)), sOff - 2);
// test next position
final int h = hash(UnsafeUtils.readInt(src, sOff));
ref = UnsafeUtils.readInt(hashTable, h);
UnsafeUtils.writeInt(hashTable, h, sOff);
back = sOff - ref;
if (back >= MAX_DISTANCE || !LZ4UnsafeUtils.readIntEquals(src, ref, sOff)) {
break;
}
tokenOff = dOff++;
UnsafeUtils.writeByte(dest, tokenOff, 0);
}
// prepare next loop
anchor = sOff++;
}
dOff = LZ4UnsafeUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
static int compress64k(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int destEnd) {
final int srcEnd = srcOff + srcLen;
final int srcLimit = srcEnd - LAST_LITERALS;
final int mflimit = srcEnd - MF_LIMIT;
int sOff = srcOff, dOff = destOff;
int anchor = sOff;
if (srcLen >= MIN_LENGTH) {
final short[] hashTable = new short[HASH_TABLE_SIZE_64K];
++sOff;
main:
while (true) {
// find a match
int forwardOff = sOff;
int ref;
int step = 1;
int searchMatchNb = 1 << SKIP_STRENGTH;
do {
sOff = forwardOff;
forwardOff += step;
step = searchMatchNb++ >>> SKIP_STRENGTH;
if (forwardOff > mflimit) {
break main;
}
final int h = hash64k(ByteBufferUtils.readInt(src, sOff));
ref = srcOff + UnsafeUtils.readShort(hashTable, h);
UnsafeUtils.writeShort(hashTable, h, sOff - srcOff);
} while (!LZ4ByteBufferUtils.readIntEquals(src, ref, sOff));
// catch up
final int excess = LZ4ByteBufferUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor);
sOff -= excess;
ref -= excess;
// sequence == refsequence
final int runLen = sOff - anchor;
// encode literal length
int tokenOff = dOff++;
if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
if (runLen >= RUN_MASK) {
ByteBufferUtils.writeByte(dest, tokenOff, RUN_MASK << ML_BITS);
dOff = LZ4ByteBufferUtils.writeLen(runLen - RUN_MASK, dest, dOff);
} else {
ByteBufferUtils.writeByte(dest, tokenOff, runLen << ML_BITS);
}
// copy literals
LZ4ByteBufferUtils.wildArraycopy(src, anchor, dest, dOff, runLen);
dOff += runLen;
while (true) {
// encode offset
ByteBufferUtils.writeShortLE(dest, dOff, (short) (sOff - ref));
dOff += 2;
// count nb matches
sOff += MIN_MATCH;
ref += MIN_MATCH;
final int matchLen = LZ4ByteBufferUtils.commonBytes(src, ref, sOff, srcLimit);
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
sOff += matchLen;
// encode match len
if (matchLen >= ML_MASK) {
ByteBufferUtils.writeByte(dest, tokenOff, ByteBufferUtils.readByte(dest, tokenOff) | ML_MASK);
dOff = LZ4ByteBufferUtils.writeLen(matchLen - ML_MASK, dest, dOff);
} else {
ByteBufferUtils.writeByte(dest, tokenOff, ByteBufferUtils.readByte(dest, tokenOff) | matchLen);
}
// test end of chunk
if (sOff > mflimit) {
anchor = sOff;
break main;
}
// fill table
UnsafeUtils.writeShort(hashTable, hash64k(ByteBufferUtils.readInt(src, sOff - 2)), sOff - 2 - srcOff);
// test next position
final int h = hash64k(ByteBufferUtils.readInt(src, sOff));
ref = srcOff + UnsafeUtils.readShort(hashTable, h);
UnsafeUtils.writeShort(hashTable, h, sOff - srcOff);
if (!LZ4ByteBufferUtils.readIntEquals(src, sOff, ref)) {
break;
}
tokenOff = dOff++;
ByteBufferUtils.writeByte(dest, tokenOff, 0);
}
// prepare next loop
anchor = sOff++;
}
}
dOff = LZ4ByteBufferUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
@Override
public int compress(ByteBuffer src, final int srcOff, int srcLen, ByteBuffer dest, final int destOff, int maxDestLen) {
if (src.hasArray() && dest.hasArray()) {
return compress(src.array(), srcOff + src.arrayOffset(), srcLen, dest.array(), destOff + dest.arrayOffset(), maxDestLen);
}
src = ByteBufferUtils.inNativeByteOrder(src);
dest = ByteBufferUtils.inNativeByteOrder(dest);
ByteBufferUtils.checkRange(src, srcOff, srcLen);
ByteBufferUtils.checkRange(dest, destOff, maxDestLen);
final int destEnd = destOff + maxDestLen;
if (srcLen < LZ4_64K_LIMIT) {
return compress64k(src, srcOff, srcLen, dest, destOff, destEnd);
}
final int srcEnd = srcOff + srcLen;
final int srcLimit = srcEnd - LAST_LITERALS;
final int mflimit = srcEnd - MF_LIMIT;
int sOff = srcOff, dOff = destOff;
int anchor = sOff++;
final int[] hashTable = new int[HASH_TABLE_SIZE];
Arrays.fill(hashTable, anchor);
main:
while (true) {
// find a match
int forwardOff = sOff;
int ref;
int step = 1;
int searchMatchNb = 1 << SKIP_STRENGTH;
int back;
do {
sOff = forwardOff;
forwardOff += step;
step = searchMatchNb++ >>> SKIP_STRENGTH;
if (forwardOff > mflimit) {
break main;
}
final int h = hash(ByteBufferUtils.readInt(src, sOff));
ref = UnsafeUtils.readInt(hashTable, h);
back = sOff - ref;
UnsafeUtils.writeInt(hashTable, h, sOff);
} while (back >= MAX_DISTANCE || !LZ4ByteBufferUtils.readIntEquals(src, ref, sOff));
final int excess = LZ4ByteBufferUtils.commonBytesBackward(src, ref, sOff, srcOff, anchor);
sOff -= excess;
ref -= excess;
// sequence == refsequence
final int runLen = sOff - anchor;
// encode literal length
int tokenOff = dOff++;
if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
if (runLen >= RUN_MASK) {
ByteBufferUtils.writeByte(dest, tokenOff, RUN_MASK << ML_BITS);
dOff = LZ4ByteBufferUtils.writeLen(runLen - RUN_MASK, dest, dOff);
} else {
ByteBufferUtils.writeByte(dest, tokenOff, runLen << ML_BITS);
}
// copy literals
LZ4ByteBufferUtils.wildArraycopy(src, anchor, dest, dOff, runLen);
dOff += runLen;
while (true) {
// encode offset
ByteBufferUtils.writeShortLE(dest, dOff, back);
dOff += 2;
// count nb matches
sOff += MIN_MATCH;
final int matchLen = LZ4ByteBufferUtils.commonBytes(src, ref + MIN_MATCH, sOff, srcLimit);
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
sOff += matchLen;
// encode match len
if (matchLen >= ML_MASK) {
ByteBufferUtils.writeByte(dest, tokenOff, ByteBufferUtils.readByte(dest, tokenOff) | ML_MASK);
dOff = LZ4ByteBufferUtils.writeLen(matchLen - ML_MASK, dest, dOff);
} else {
ByteBufferUtils.writeByte(dest, tokenOff, ByteBufferUtils.readByte(dest, tokenOff) | matchLen);
}
// test end of chunk
if (sOff > mflimit) {
anchor = sOff;
break main;
}
// fill table
UnsafeUtils.writeInt(hashTable, hash(ByteBufferUtils.readInt(src, sOff - 2)), sOff - 2);
// test next position
final int h = hash(ByteBufferUtils.readInt(src, sOff));
ref = UnsafeUtils.readInt(hashTable, h);
UnsafeUtils.writeInt(hashTable, h, sOff);
back = sOff - ref;
if (back >= MAX_DISTANCE || !LZ4ByteBufferUtils.readIntEquals(src, ref, sOff)) {
break;
}
tokenOff = dOff++;
ByteBufferUtils.writeByte(dest, tokenOff, 0);
}
// prepare next loop
anchor = sOff++;
}
dOff = LZ4ByteBufferUtils.lastLiterals(src, anchor, srcEnd - anchor, dest, dOff, destEnd);
return dOff - destOff;
}
}

View File

@ -0,0 +1,205 @@
// Auto-generated: DO NOT EDIT
package net.jpountz.lz4;
import static net.jpountz.lz4.LZ4Constants.*;
import java.nio.ByteBuffer;
import net.jpountz.util.ByteBufferUtils;
import net.jpountz.util.UnsafeUtils;
/**
* Decompressor.
*/
final class LZ4JavaUnsafeFastDecompressor extends LZ4FastDecompressor {
public static final LZ4FastDecompressor INSTANCE = new LZ4JavaUnsafeFastDecompressor();
@Override
public int decompress(byte[] src, final int srcOff, byte[] dest, final int destOff, int destLen) {
UnsafeUtils.checkRange(src, srcOff);
UnsafeUtils.checkRange(dest, destOff, destLen);
if (destLen == 0) {
if (UnsafeUtils.readByte(src, srcOff) != 0) {
throw new LZ4Exception("Malformed input at " + srcOff);
}
return 1;
}
final int destEnd = destOff + destLen;
int sOff = srcOff;
int dOff = destOff;
while (true) {
final int token = UnsafeUtils.readByte(src, sOff) & 0xFF;
++sOff;
// literals
int literalLen = token >>> ML_BITS;
if (literalLen == RUN_MASK) {
byte len = (byte) 0xFF;
while ((len = UnsafeUtils.readByte(src, sOff++)) == (byte) 0xFF) {
literalLen += 0xFF;
}
literalLen += len & 0xFF;
}
final int literalCopyEnd = dOff + literalLen;
if (literalCopyEnd > destEnd - COPY_LENGTH) {
if (literalCopyEnd != destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
} else {
LZ4UnsafeUtils.safeArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
break; // EOF
}
}
LZ4UnsafeUtils.wildArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
// matchs
final int matchDec = UnsafeUtils.readShortLE(src, sOff);
sOff += 2;
int matchOff = dOff - matchDec;
if (matchOff < destOff) {
throw new LZ4Exception("Malformed input at " + sOff);
}
int matchLen = token & ML_MASK;
if (matchLen == ML_MASK) {
byte len = (byte) 0xFF;
while ((len = UnsafeUtils.readByte(src, sOff++)) == (byte) 0xFF) {
matchLen += 0xFF;
}
matchLen += len & 0xFF;
}
matchLen += MIN_MATCH;
final int matchCopyEnd = dOff + matchLen;
if (matchCopyEnd > destEnd - COPY_LENGTH) {
if (matchCopyEnd > destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
}
LZ4UnsafeUtils.safeIncrementalCopy(dest, matchOff, dOff, matchLen);
} else {
LZ4UnsafeUtils.wildIncrementalCopy(dest, matchOff, dOff, matchCopyEnd);
}
dOff = matchCopyEnd;
}
return sOff - srcOff;
}
@Override
public int decompress(ByteBuffer src, final int srcOff, ByteBuffer dest, final int destOff, int destLen) {
if (src.hasArray() && dest.hasArray()) {
return decompress(src.array(), srcOff + src.arrayOffset(), dest.array(), destOff + dest.arrayOffset(), destLen);
}
src = ByteBufferUtils.inNativeByteOrder(src);
dest = ByteBufferUtils.inNativeByteOrder(dest);
ByteBufferUtils.checkRange(src, srcOff);
ByteBufferUtils.checkRange(dest, destOff, destLen);
if (destLen == 0) {
if (ByteBufferUtils.readByte(src, srcOff) != 0) {
throw new LZ4Exception("Malformed input at " + srcOff);
}
return 1;
}
final int destEnd = destOff + destLen;
int sOff = srcOff;
int dOff = destOff;
while (true) {
final int token = ByteBufferUtils.readByte(src, sOff) & 0xFF;
++sOff;
// literals
int literalLen = token >>> ML_BITS;
if (literalLen == RUN_MASK) {
byte len = (byte) 0xFF;
while ((len = ByteBufferUtils.readByte(src, sOff++)) == (byte) 0xFF) {
literalLen += 0xFF;
}
literalLen += len & 0xFF;
}
final int literalCopyEnd = dOff + literalLen;
if (literalCopyEnd > destEnd - COPY_LENGTH) {
if (literalCopyEnd != destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
} else {
LZ4ByteBufferUtils.safeArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
break; // EOF
}
}
LZ4ByteBufferUtils.wildArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
// matchs
final int matchDec = ByteBufferUtils.readShortLE(src, sOff);
sOff += 2;
int matchOff = dOff - matchDec;
if (matchOff < destOff) {
throw new LZ4Exception("Malformed input at " + sOff);
}
int matchLen = token & ML_MASK;
if (matchLen == ML_MASK) {
byte len = (byte) 0xFF;
while ((len = ByteBufferUtils.readByte(src, sOff++)) == (byte) 0xFF) {
matchLen += 0xFF;
}
matchLen += len & 0xFF;
}
matchLen += MIN_MATCH;
final int matchCopyEnd = dOff + matchLen;
if (matchCopyEnd > destEnd - COPY_LENGTH) {
if (matchCopyEnd > destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
}
LZ4ByteBufferUtils.safeIncrementalCopy(dest, matchOff, dOff, matchLen);
} else {
LZ4ByteBufferUtils.wildIncrementalCopy(dest, matchOff, dOff, matchCopyEnd);
}
dOff = matchCopyEnd;
}
return sOff - srcOff;
}
}

View File

@ -0,0 +1,213 @@
// Auto-generated: DO NOT EDIT
package net.jpountz.lz4;
import static net.jpountz.lz4.LZ4Constants.*;
import java.nio.ByteBuffer;
import net.jpountz.util.ByteBufferUtils;
import net.jpountz.util.UnsafeUtils;
/**
* Decompressor.
*/
final class LZ4JavaUnsafeSafeDecompressor extends LZ4SafeDecompressor {
public static final LZ4SafeDecompressor INSTANCE = new LZ4JavaUnsafeSafeDecompressor();
@Override
public int decompress(byte[] src, final int srcOff, final int srcLen , byte[] dest, final int destOff, int destLen) {
UnsafeUtils.checkRange(src, srcOff, srcLen);
UnsafeUtils.checkRange(dest, destOff, destLen);
if (destLen == 0) {
if (srcLen != 1 || UnsafeUtils.readByte(src, srcOff) != 0) {
throw new LZ4Exception("Output buffer too small");
}
return 0;
}
final int srcEnd = srcOff + srcLen;
final int destEnd = destOff + destLen;
int sOff = srcOff;
int dOff = destOff;
while (true) {
final int token = UnsafeUtils.readByte(src, sOff) & 0xFF;
++sOff;
// literals
int literalLen = token >>> ML_BITS;
if (literalLen == RUN_MASK) {
byte len = (byte) 0xFF;
while (sOff < srcEnd &&(len = UnsafeUtils.readByte(src, sOff++)) == (byte) 0xFF) {
literalLen += 0xFF;
}
literalLen += len & 0xFF;
}
final int literalCopyEnd = dOff + literalLen;
if (literalCopyEnd > destEnd - COPY_LENGTH || sOff + literalLen > srcEnd - COPY_LENGTH) {
if (literalCopyEnd > destEnd) {
throw new LZ4Exception();
} else if (sOff + literalLen != srcEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
} else {
LZ4UnsafeUtils.safeArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
break; // EOF
}
}
LZ4UnsafeUtils.wildArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
// matchs
final int matchDec = UnsafeUtils.readShortLE(src, sOff);
sOff += 2;
int matchOff = dOff - matchDec;
if (matchOff < destOff) {
throw new LZ4Exception("Malformed input at " + sOff);
}
int matchLen = token & ML_MASK;
if (matchLen == ML_MASK) {
byte len = (byte) 0xFF;
while (sOff < srcEnd &&(len = UnsafeUtils.readByte(src, sOff++)) == (byte) 0xFF) {
matchLen += 0xFF;
}
matchLen += len & 0xFF;
}
matchLen += MIN_MATCH;
final int matchCopyEnd = dOff + matchLen;
if (matchCopyEnd > destEnd - COPY_LENGTH) {
if (matchCopyEnd > destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
}
LZ4UnsafeUtils.safeIncrementalCopy(dest, matchOff, dOff, matchLen);
} else {
LZ4UnsafeUtils.wildIncrementalCopy(dest, matchOff, dOff, matchCopyEnd);
}
dOff = matchCopyEnd;
}
return dOff - destOff;
}
@Override
public int decompress(ByteBuffer src, final int srcOff, final int srcLen , ByteBuffer dest, final int destOff, int destLen) {
if (src.hasArray() && dest.hasArray()) {
return decompress(src.array(), srcOff + src.arrayOffset(), srcLen, dest.array(), destOff + dest.arrayOffset(), destLen);
}
src = ByteBufferUtils.inNativeByteOrder(src);
dest = ByteBufferUtils.inNativeByteOrder(dest);
ByteBufferUtils.checkRange(src, srcOff, srcLen);
ByteBufferUtils.checkRange(dest, destOff, destLen);
if (destLen == 0) {
if (srcLen != 1 || ByteBufferUtils.readByte(src, srcOff) != 0) {
throw new LZ4Exception("Output buffer too small");
}
return 0;
}
final int srcEnd = srcOff + srcLen;
final int destEnd = destOff + destLen;
int sOff = srcOff;
int dOff = destOff;
while (true) {
final int token = ByteBufferUtils.readByte(src, sOff) & 0xFF;
++sOff;
// literals
int literalLen = token >>> ML_BITS;
if (literalLen == RUN_MASK) {
byte len = (byte) 0xFF;
while (sOff < srcEnd &&(len = ByteBufferUtils.readByte(src, sOff++)) == (byte) 0xFF) {
literalLen += 0xFF;
}
literalLen += len & 0xFF;
}
final int literalCopyEnd = dOff + literalLen;
if (literalCopyEnd > destEnd - COPY_LENGTH || sOff + literalLen > srcEnd - COPY_LENGTH) {
if (literalCopyEnd > destEnd) {
throw new LZ4Exception();
} else if (sOff + literalLen != srcEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
} else {
LZ4ByteBufferUtils.safeArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
break; // EOF
}
}
LZ4ByteBufferUtils.wildArraycopy(src, sOff, dest, dOff, literalLen);
sOff += literalLen;
dOff = literalCopyEnd;
// matchs
final int matchDec = ByteBufferUtils.readShortLE(src, sOff);
sOff += 2;
int matchOff = dOff - matchDec;
if (matchOff < destOff) {
throw new LZ4Exception("Malformed input at " + sOff);
}
int matchLen = token & ML_MASK;
if (matchLen == ML_MASK) {
byte len = (byte) 0xFF;
while (sOff < srcEnd &&(len = ByteBufferUtils.readByte(src, sOff++)) == (byte) 0xFF) {
matchLen += 0xFF;
}
matchLen += len & 0xFF;
}
matchLen += MIN_MATCH;
final int matchCopyEnd = dOff + matchLen;
if (matchCopyEnd > destEnd - COPY_LENGTH) {
if (matchCopyEnd > destEnd) {
throw new LZ4Exception("Malformed input at " + sOff);
}
LZ4ByteBufferUtils.safeIncrementalCopy(dest, matchOff, dOff, matchLen);
} else {
LZ4ByteBufferUtils.wildIncrementalCopy(dest, matchOff, dOff, matchCopyEnd);
}
dOff = matchCopyEnd;
}
return dOff - destOff;
}
}

View File

@ -0,0 +1,73 @@
package net.jpountz.lz4;
import java.io.OutputStream;
import java.io.IOException;
public class LZ4OutputStream extends OutputStream {
private static final LZ4Factory lz4Factory = LZ4Factory.fastestInstance();
private final LZ4Compressor compressor;
private static final int ONE_MEGABYTE = 1048576;
private final byte[] compressionInputBuffer;
private final byte[] compressionOutputBuffer;
private final OutputStream underlyingOutputStream;
private int bytesRemainingInCompressionInputBuffer = 0;
private int currentCompressionInputBufferPosition = 0;
public LZ4OutputStream(OutputStream os) throws IOException {
this(os, ONE_MEGABYTE, lz4Factory.fastCompressor());
}
public LZ4OutputStream(OutputStream underlyingOutputStream, int blocksize, LZ4Compressor compressor) throws IOException {
compressionInputBuffer = new byte[blocksize];
this.compressor = compressor;
this.underlyingOutputStream = underlyingOutputStream;
this.bytesRemainingInCompressionInputBuffer = blocksize;
this.currentCompressionInputBufferPosition = 0;
this.compressionOutputBuffer = new byte[compressor.maxCompressedLength(blocksize)];
}
public void write(byte[] b, int off, int len) throws IOException {
if (len <= bytesRemainingInCompressionInputBuffer) {
System.arraycopy(b, off, compressionInputBuffer, currentCompressionInputBufferPosition, len);
currentCompressionInputBufferPosition += len;
bytesRemainingInCompressionInputBuffer -= len;
} else {
// len > bytesRemainingInCompressionInputBuffer
while (len > 0) {
int bytesToCopy = Math.min(bytesRemainingInCompressionInputBuffer, len);
System.arraycopy(b, off, compressionInputBuffer, currentCompressionInputBufferPosition, bytesToCopy);
currentCompressionInputBufferPosition += bytesToCopy;
bytesRemainingInCompressionInputBuffer -= bytesToCopy;
flush();
len -= bytesToCopy;
off += bytesToCopy;
}
}
}
public void write(int i) throws IOException {
byte b = (byte)i;
if (0 == bytesRemainingInCompressionInputBuffer) {
flush();
}
compressionInputBuffer[currentCompressionInputBufferPosition] = b;
bytesRemainingInCompressionInputBuffer--;
currentCompressionInputBufferPosition++;
}
public void flush() throws IOException {
if(currentCompressionInputBufferPosition > 0) {
LZ4StreamHelper.writeLength(currentCompressionInputBufferPosition, this.underlyingOutputStream);
int bytesCompressed = compressor.compress(compressionInputBuffer, 0, currentCompressionInputBufferPosition, compressionOutputBuffer, 0, compressionOutputBuffer.length);
LZ4StreamHelper.writeLength(bytesCompressed, this.underlyingOutputStream);
underlyingOutputStream.write(compressionOutputBuffer, 0, bytesCompressed);
bytesRemainingInCompressionInputBuffer = compressionInputBuffer.length;
currentCompressionInputBufferPosition = 0;
}
}
public void close() throws IOException {
flush();
underlyingOutputStream.close();
}
}

View File

@ -0,0 +1,117 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteBuffer;
import java.util.Arrays;
/**
* LZ4 decompressor that requires the size of the compressed data to be known.
* <p>
* Implementations of this class are usually a little slower than those of
* {@link LZ4FastDecompressor} but do not require the size of the original data to
* be known.
*/
public abstract class LZ4SafeDecompressor implements LZ4UnknownSizeDecompressor {
/**
* Decompress <code>src[srcOff:srcLen]</code> into
* <code>dest[destOff:destOff+maxDestLen]</code> and returns the number of
* decompressed bytes written into <code>dest</code>.
*
* @param srcLen the exact size of the compressed stream
* @return the original input size
* @throws LZ4Exception if maxDestLen is too small
*/
public abstract int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen);
/**
* Uncompress <code>src[srcOff:srcLen]</code> into
* <code>dest[destOff:destOff+maxDestLen]</code> and returns the number of
* decompressed bytes written into <code>dest</code>.
*
* @param srcLen the exact size of the compressed stream
* @return the original input size
* @throws LZ4Exception if maxDestLen is too small
*/
public abstract int decompress(ByteBuffer src, int srcOff, int srcLen, ByteBuffer dest, int destOff, int maxDestLen);
/**
* Convenience method, equivalent to calling
* {@link #decompress(byte[], int, int, byte[], int, int) decompress(src, srcOff, srcLen, dest, destOff, dest.length - destOff)}.
*/
public final int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff) {
return decompress(src, srcOff, srcLen, dest, destOff, dest.length - destOff);
}
/**
* Convenience method, equivalent to calling
* {@link #decompress(byte[], int, int, byte[], int) decompress(src, 0, src.length, dest, 0)}
*/
public final int decompress(byte[] src, byte[] dest) {
return decompress(src, 0, src.length, dest, 0);
}
/**
* Convenience method which returns <code>src[srcOff:srcOff+srcLen]</code>
* decompressed.
* <p><b><span style="color:red">Warning</span></b>: this method has an
* important overhead due to the fact that it needs to allocate a buffer to
* decompress into, and then needs to resize this buffer to the actual
* decompressed length.</p>
* <p>Here is how this method is implemented:</p>
* <pre>
* byte[] decompressed = new byte[maxDestLen];
* final int decompressedLength = decompress(src, srcOff, srcLen, decompressed, 0, maxDestLen);
* if (decompressedLength != decompressed.length) {
* decompressed = Arrays.copyOf(decompressed, decompressedLength);
* }
* return decompressed;
* </pre>
*/
public final byte[] decompress(byte[] src, int srcOff, int srcLen, int maxDestLen) {
byte[] decompressed = new byte[maxDestLen];
final int decompressedLength = decompress(src, srcOff, srcLen, decompressed, 0, maxDestLen);
if (decompressedLength != decompressed.length) {
decompressed = Arrays.copyOf(decompressed, decompressedLength);
}
return decompressed;
}
/**
* Convenience method, equivalent to calling
* {@link #decompress(byte[], int, int, int) decompress(src, 0, src.length, maxDestLen)}.
*/
public final byte[] decompress(byte[] src, int maxDestLen) {
return decompress(src, 0, src.length, maxDestLen);
}
/**
* Decompress <code>src</code> into <code>dest</code>. <code>src</code>'s
* {@link ByteBuffer#remaining()} must be exactly the size of the compressed
* data. This method moves the positions of the buffers.
*/
public final void decompress(ByteBuffer src, ByteBuffer dest) {
final int decompressed = decompress(src, src.position(), src.remaining(), dest, dest.position(), dest.remaining());
src.position(src.limit());
dest.position(dest.position() + decompressed);
}
@Override
public String toString() {
return getClass().getSimpleName();
}
}

View File

@ -0,0 +1,179 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.lz4.LZ4Constants.LAST_LITERALS;
import static net.jpountz.lz4.LZ4Constants.ML_BITS;
import static net.jpountz.lz4.LZ4Constants.ML_MASK;
import static net.jpountz.lz4.LZ4Constants.RUN_MASK;
import net.jpountz.util.SafeUtils;
enum LZ4SafeUtils {
;
static int hash(byte[] buf, int i) {
return LZ4Utils.hash(SafeUtils.readInt(buf, i));
}
static int hash64k(byte[] buf, int i) {
return LZ4Utils.hash64k(SafeUtils.readInt(buf, i));
}
static boolean readIntEquals(byte[] buf, int i, int j) {
return buf[i] == buf[j] && buf[i+1] == buf[j+1] && buf[i+2] == buf[j+2] && buf[i+3] == buf[j+3];
}
static void safeIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchLen) {
for (int i = 0; i < matchLen; ++i) {
dest[dOff + i] = dest[matchOff + i];
}
}
static void wildIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchCopyEnd) {
do {
copy8Bytes(dest, matchOff, dest, dOff);
matchOff += 8;
dOff += 8;
} while (dOff < matchCopyEnd);
}
static void copy8Bytes(byte[] src, int sOff, byte[] dest, int dOff) {
for (int i = 0; i < 8; ++i) {
dest[dOff + i] = src[sOff + i];
}
}
static int commonBytes(byte[] b, int o1, int o2, int limit) {
int count = 0;
while (o2 < limit && b[o1++] == b[o2++]) {
++count;
}
return count;
}
static int commonBytesBackward(byte[] b, int o1, int o2, int l1, int l2) {
int count = 0;
while (o1 > l1 && o2 > l2 && b[--o1] == b[--o2]) {
++count;
}
return count;
}
static void safeArraycopy(byte[] src, int sOff, byte[] dest, int dOff, int len) {
System.arraycopy(src, sOff, dest, dOff, len);
}
static void wildArraycopy(byte[] src, int sOff, byte[] dest, int dOff, int len) {
try {
for (int i = 0; i < len; i += 8) {
copy8Bytes(src, sOff + i, dest, dOff + i);
}
} catch (ArrayIndexOutOfBoundsException e) {
throw new LZ4Exception("Malformed input at offset " + sOff);
}
}
static int encodeSequence(byte[] src, int anchor, int matchOff, int matchRef, int matchLen, byte[] dest, int dOff, int destEnd) {
final int runLen = matchOff - anchor;
final int tokenOff = dOff++;
if (dOff + runLen + (2 + 1 + LAST_LITERALS) + (runLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
int token;
if (runLen >= RUN_MASK) {
token = (byte) (RUN_MASK << ML_BITS);
dOff = writeLen(runLen - RUN_MASK, dest, dOff);
} else {
token = runLen << ML_BITS;
}
// copy literals
wildArraycopy(src, anchor, dest, dOff, runLen);
dOff += runLen;
// encode offset
final int matchDec = matchOff - matchRef;
dest[dOff++] = (byte) matchDec;
dest[dOff++] = (byte) (matchDec >>> 8);
// encode match len
matchLen -= 4;
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
if (matchLen >= ML_MASK) {
token |= ML_MASK;
dOff = writeLen(matchLen - RUN_MASK, dest, dOff);
} else {
token |= matchLen;
}
dest[tokenOff] = (byte) token;
return dOff;
}
static int lastLiterals(byte[] src, int sOff, int srcLen, byte[] dest, int dOff, int destEnd) {
final int runLen = srcLen;
if (dOff + runLen + 1 + (runLen + 255 - RUN_MASK) / 255 > destEnd) {
throw new LZ4Exception();
}
if (runLen >= RUN_MASK) {
dest[dOff++] = (byte) (RUN_MASK << ML_BITS);
dOff = writeLen(runLen - RUN_MASK, dest, dOff);
} else {
dest[dOff++] = (byte) (runLen << ML_BITS);
}
// copy literals
System.arraycopy(src, sOff, dest, dOff, runLen);
dOff += runLen;
return dOff;
}
static int writeLen(int len, byte[] dest, int dOff) {
while (len >= 0xFF) {
dest[dOff++] = (byte) 0xFF;
len -= 0xFF;
}
dest[dOff++] = (byte) len;
return dOff;
}
static class Match {
int start, ref, len;
void fix(int correction) {
start += correction;
ref += correction;
len -= correction;
}
int end() {
return start + len;
}
}
static void copyTo(Match m1, Match m2) {
m2.len = m1.len;
m2.start = m1.start;
m2.ref = m1.ref;
}
}

View File

@ -0,0 +1,36 @@
package net.jpountz.lz4;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.IOException;
public class LZ4StreamHelper {
static void writeLength(int length, OutputStream os) throws IOException {
int b1 = ((length & 0xff000000) >> 24);
int b2 = ((length & 0x00ff0000) >> 16);
int b3 = ((length & 0x0000ff00) >> 8);
int b4 = (length & 0xff0000ff);
os.write(b1);
os.write(b2);
os.write(b3);
os.write(b4);
}
// network order, big endian, most significant byte first
// package scope
static int readLength(InputStream is) throws IOException {
int b1 = is.read();
int b2 = is.read();
int b3 = is.read();
int b4 = is.read();
int length;
if((-1 == b1) || (-1 == b2) || (-1 == b3) || (-1 == b4)) {
length = -1;
}
else {
length = ((b1 << 24) | (b2 << 16) | (b3 << 8) | b4);
}
return length;
}
}

View File

@ -0,0 +1,150 @@
package net.jpountz.lz4;
import static junit.framework.Assert.assertEquals;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Random;
import junit.framework.Assert;
import org.junit.Before;
import org.junit.Test;
public class LZ4StreamTest {
private long seed;
private Random rand;
private byte randomContent[];
private byte compressedOutput[];
@Before
public void setUp() throws IOException {
seed = System.currentTimeMillis();
rand = new Random(seed);
int randomContentLength = rand.nextInt(10000000) + 10000000;
randomContent = new byte[randomContentLength];
rand.nextBytes(randomContent);
compressContent();
}
private void compressContent() throws IOException {
ByteArrayOutputStream compressedOutputStream = new ByteArrayOutputStream();
LZ4OutputStream os = new LZ4OutputStream(compressedOutputStream);
int currentContentPosition = 0;
while(currentContentPosition < randomContent.length) {
int testBlockSize = rand.nextInt(500000);
if(testBlockSize > randomContent.length - currentContentPosition)
testBlockSize = randomContent.length - currentContentPosition;
boolean writeByteByByte = true; //rand.nextBoolean();
if(writeByteByByte) {
for(int i=0;i<testBlockSize;i++) {
os.write(randomContent[currentContentPosition++]);
}
} else {
boolean writeDirectlyFromContent = rand.nextBoolean();
if(writeDirectlyFromContent) {
os.write(randomContent, currentContentPosition, testBlockSize);
} else {
byte b[] = new byte[testBlockSize];
System.arraycopy(randomContent, currentContentPosition, b, 0, testBlockSize);
os.write(b);
}
currentContentPosition += testBlockSize;
}
}
os.close();
compressedOutput = compressedOutputStream.toByteArray();
}
@Test
public void randomizedTest() throws IOException {
try {
InputStream is = new LZ4InputStream(new ByteArrayInputStream(compressedOutput));
int currentContentPosition = 0;
while(currentContentPosition < randomContent.length) {
int testBlockSize = rand.nextInt(500000);
boolean shouldTestBytes = rand.nextBoolean();
if(shouldTestBytes) {
boolean shouldReadOneByOne = rand.nextBoolean();
if(shouldReadOneByOne) {
currentContentPosition += assertContentByteByByte(is, currentContentPosition, testBlockSize);
} else {
currentContentPosition += assertContentInSingleBlock(is, currentContentPosition, testBlockSize);
}
} else {
currentContentPosition += skipContent(is, testBlockSize);
}
}
assertEquals(-1, is.read(new byte[100]));
assertEquals(-1, is.read());
} catch(Throwable t) {
t.printStackTrace();
Assert.fail("Exception was thrown. Seed value was " + seed);
}
}
private int assertContentByteByByte(InputStream is, int currentContentPosition, int testBlockSize) throws IOException {
int readContentLength = 0;
while(readContentLength < testBlockSize) {
int readContent = is.read();
if(readContent == -1)
break;
assertEquals(randomContent[currentContentPosition + readContentLength], (byte)readContent);
readContentLength++;
}
return readContentLength;
}
private int assertContentInSingleBlock(InputStream is, int currentContentPosition, int testBlockSize) throws IOException {
int readContentLength;
byte readContent[] = new byte[testBlockSize];
readContentLength = is.read(readContent);
assertEqualContent(readContent, currentContentPosition, readContentLength);
return readContentLength;
}
private int skipContent(InputStream is, int testBlockSize) throws IOException {
return (int) is.skip(testBlockSize);
}
private void assertEqualContent(byte readContent[], int uncompressedContentOffset, int readContentLength) {
if(readContentLength < 0 && uncompressedContentOffset < randomContent.length)
Assert.fail("Decompressed content was incomplete. Index " + uncompressedContentOffset + ". Seed was " + seed);
for(int i=0;i<readContentLength;i++) {
String message = "Bytes differed! Seed value was " + seed;
Assert.assertEquals(message, randomContent[uncompressedContentOffset + i], readContent[i]);
}
}
}

View File

@ -0,0 +1,27 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @deprecated Use {@link LZ4SafeDecompressor} instead.
*/
@Deprecated
public interface LZ4UnknownSizeDecompressor {
int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff, int maxDestLen);
int decompress(byte[] src, int srcOff, int srcLen, byte[] dest, int destOff);
}

View File

@ -0,0 +1,73 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.lz4.LZ4Constants.HASH_LOG;
import static net.jpountz.lz4.LZ4Constants.HASH_LOG_64K;
import static net.jpountz.lz4.LZ4Constants.HASH_LOG_HC;
import static net.jpountz.lz4.LZ4Constants.LAST_LITERALS;
import static net.jpountz.lz4.LZ4Constants.MIN_MATCH;
import static net.jpountz.lz4.LZ4Constants.ML_BITS;
import static net.jpountz.lz4.LZ4Constants.ML_MASK;
import static net.jpountz.lz4.LZ4Constants.RUN_MASK;
import net.jpountz.util.SafeUtils;
enum LZ4Utils {
;
private static final int MAX_INPUT_SIZE = 0x7E000000;
static int maxCompressedLength(int length) {
if (length < 0) {
throw new IllegalArgumentException("length must be >= 0, got " + length);
} else if (length >= MAX_INPUT_SIZE) {
throw new IllegalArgumentException("length must be < " + MAX_INPUT_SIZE);
}
return length + length / 255 + 16;
}
static int hash(int i) {
return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG);
}
static int hash64k(int i) {
return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG_64K);
}
static int hashHC(int i) {
return (i * -1640531535) >>> ((MIN_MATCH * 8) - HASH_LOG_HC);
}
static class Match {
int start, ref, len;
void fix(int correction) {
start += correction;
ref += correction;
len -= correction;
}
int end() {
return start + len;
}
}
static void copyTo(Match m1, Match m2) {
m2.len = m1.len;
m2.start = m1.start;
m2.ref = m1.ref;
}
}

View File

@ -0,0 +1,55 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>LZ4 compression. The entry point of the API is the
{@link net.jpountz.lz4.LZ4Factory} class, which gives access to
{@link net.jpountz.lz4.LZ4Compressor compressors} and
{@link net.jpountz.lz4.LZ4SafeDecompressor decompressors}.</p>
<p>Sample usage:</p>
<pre class="prettyprint">
LZ4Factory factory = LZ4Factory.fastestInstance();
byte[] data = "12345345234572".getBytes("UTF-8");
final int decompressedLength = data.length;
// compress data
LZ4Compressor compressor = factory.fastCompressor();
int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
byte[] compressed = new byte[maxCompressedLength];
int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
// decompress data
// - method 1: when the decompressed length is known
LZ4FastDecompressor decompressor = factory.fastDecompressor();
byte[] restored = new byte[decompressedLength];
int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
// compressedLength == compressedLength2
// - method 2: when the compressed length is known (a little slower)
// the destination buffer needs to be over-sized
LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0);
// decompressedLength == decompressedLength2
</pre>
</body>
</html>

View File

@ -0,0 +1,92 @@
package net.jpountz.util;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.ReadOnlyBufferException;
public enum ByteBufferUtils {
;
public static void checkRange(ByteBuffer buf, int off, int len) {
SafeUtils.checkLength(len);
if (len > 0) {
checkRange(buf, off);
checkRange(buf, off + len - 1);
}
}
public static void checkRange(ByteBuffer buf, int off) {
if (off < 0 || off >= buf.capacity()) {
throw new ArrayIndexOutOfBoundsException(off);
}
}
public static ByteBuffer inLittleEndianOrder(ByteBuffer buf) {
if (buf.order().equals(ByteOrder.LITTLE_ENDIAN)) {
return buf;
} else {
return buf.duplicate().order(ByteOrder.LITTLE_ENDIAN);
}
}
public static ByteBuffer inNativeByteOrder(ByteBuffer buf) {
if (buf.order().equals(Utils.NATIVE_BYTE_ORDER)) {
return buf;
} else {
return buf.duplicate().order(Utils.NATIVE_BYTE_ORDER);
}
}
public static byte readByte(ByteBuffer buf, int i) {
return buf.get(i);
}
public static void writeInt(ByteBuffer buf, int i, int v) {
assert buf.order() == Utils.NATIVE_BYTE_ORDER;
buf.putInt(i, v);
}
public static int readInt(ByteBuffer buf, int i) {
assert buf.order() == Utils.NATIVE_BYTE_ORDER;
return buf.getInt(i);
}
public static int readIntLE(ByteBuffer buf, int i) {
assert buf.order() == ByteOrder.LITTLE_ENDIAN;
return buf.getInt(i);
}
public static void writeLong(ByteBuffer buf, int i, long v) {
assert buf.order() == Utils.NATIVE_BYTE_ORDER;
buf.putLong(i, v);
}
public static long readLong(ByteBuffer buf, int i) {
assert buf.order() == Utils.NATIVE_BYTE_ORDER;
return buf.getLong(i);
}
public static long readLongLE(ByteBuffer buf, int i) {
assert buf.order() == ByteOrder.LITTLE_ENDIAN;
return buf.getLong(i);
}
public static void writeByte(ByteBuffer dest, int off, int i) {
dest.put(off, (byte) i);
}
public static void writeShortLE(ByteBuffer dest, int off, int i) {
dest.put(off, (byte) i);
dest.put(off + 1, (byte) (i >>> 8));
}
public static void checkNotReadOnly(ByteBuffer buffer) {
if (buffer.isReadOnly()) {
throw new ReadOnlyBufferException();
}
}
public static int readShortLE(ByteBuffer buf, int i) {
return (buf.get(i) & 0xFF) | ((buf.get(i+1) & 0xFF) << 8);
}
}

View File

@ -0,0 +1,206 @@
package net.jpountz.lz4;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.lz4.LZ4Constants.COPY_LENGTH;
import static net.jpountz.lz4.LZ4Constants.LAST_LITERALS;
import static net.jpountz.lz4.LZ4Constants.ML_BITS;
import static net.jpountz.lz4.LZ4Constants.ML_MASK;
import static net.jpountz.lz4.LZ4Constants.RUN_MASK;
import static net.jpountz.util.UnsafeUtils.readByte;
import static net.jpountz.util.UnsafeUtils.readInt;
import static net.jpountz.util.UnsafeUtils.readLong;
import static net.jpountz.util.UnsafeUtils.readShort;
import static net.jpountz.util.UnsafeUtils.writeByte;
import static net.jpountz.util.UnsafeUtils.writeInt;
import static net.jpountz.util.UnsafeUtils.writeLong;
import static net.jpountz.util.UnsafeUtils.writeShort;
import static net.jpountz.util.Utils.NATIVE_BYTE_ORDER;
import java.nio.ByteOrder;
enum LZ4UnsafeUtils {
;
static void safeArraycopy(byte[] src, int srcOff, byte[] dest, int destOff, int len) {
final int fastLen = len & 0xFFFFFFF8;
wildArraycopy(src, srcOff, dest, destOff, fastLen);
for (int i = 0, slowLen = len & 0x7; i < slowLen; i += 1) {
writeByte(dest, destOff + fastLen + i, readByte(src, srcOff + fastLen + i));
}
}
static void wildArraycopy(byte[] src, int srcOff, byte[] dest, int destOff, int len) {
for (int i = 0; i < len; i += 8) {
writeLong(dest, destOff + i, readLong(src, srcOff + i));
}
}
static void wildIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchCopyEnd) {
if (dOff - matchOff < 4) {
for (int i = 0; i < 4; ++i) {
writeByte(dest, dOff+i, readByte(dest, matchOff+i));
}
dOff += 4;
matchOff += 4;
int dec = 0;
assert dOff >= matchOff && dOff - matchOff < 8;
switch (dOff - matchOff) {
case 1:
matchOff -= 3;
break;
case 2:
matchOff -= 2;
break;
case 3:
matchOff -= 3;
dec = -1;
break;
case 5:
dec = 1;
break;
case 6:
dec = 2;
break;
case 7:
dec = 3;
break;
default:
break;
}
writeInt(dest, dOff, readInt(dest, matchOff));
dOff += 4;
matchOff -= dec;
} else if (dOff - matchOff < COPY_LENGTH) {
writeLong(dest, dOff, readLong(dest, matchOff));
dOff += dOff - matchOff;
}
while (dOff < matchCopyEnd) {
writeLong(dest, dOff, readLong(dest, matchOff));
dOff += 8;
matchOff += 8;
}
}
static void safeIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchLen) {
for (int i = 0; i < matchLen; ++i) {
dest[dOff + i] = dest[matchOff + i];
writeByte(dest, dOff + i, readByte(dest, matchOff + i));
}
}
static int readShortLittleEndian(byte[] src, int srcOff) {
short s = readShort(src, srcOff);
if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
s = Short.reverseBytes(s);
}
return s & 0xFFFF;
}
static void writeShortLittleEndian(byte[] dest, int destOff, int value) {
short s = (short) value;
if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
s = Short.reverseBytes(s);
}
writeShort(dest, destOff, s);
}
static boolean readIntEquals(byte[] src, int ref, int sOff) {
return readInt(src, ref) == readInt(src, sOff);
}
static int commonBytes(byte[] src, int ref, int sOff, int srcLimit) {
int matchLen = 0;
while (sOff <= srcLimit - 8) {
if (readLong(src, sOff) == readLong(src, ref)) {
matchLen += 8;
ref += 8;
sOff += 8;
} else {
final int zeroBits;
if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
zeroBits = Long.numberOfLeadingZeros(readLong(src, sOff) ^ readLong(src, ref));
} else {
zeroBits = Long.numberOfTrailingZeros(readLong(src, sOff) ^ readLong(src, ref));
}
return matchLen + (zeroBits >>> 3);
}
}
while (sOff < srcLimit && readByte(src, ref++) == readByte(src, sOff++)) {
++matchLen;
}
return matchLen;
}
static int writeLen(int len, byte[] dest, int dOff) {
while (len >= 0xFF) {
writeByte(dest, dOff++, 0xFF);
len -= 0xFF;
}
writeByte(dest, dOff++, len);
return dOff;
}
static int encodeSequence(byte[] src, int anchor, int matchOff, int matchRef, int matchLen, byte[] dest, int dOff, int destEnd) {
final int runLen = matchOff - anchor;
final int tokenOff = dOff++;
int token;
if (runLen >= RUN_MASK) {
token = (byte) (RUN_MASK << ML_BITS);
dOff = writeLen(runLen - RUN_MASK, dest, dOff);
} else {
token = runLen << ML_BITS;
}
// copy literals
wildArraycopy(src, anchor, dest, dOff, runLen);
dOff += runLen;
// encode offset
final int matchDec = matchOff - matchRef;
dest[dOff++] = (byte) matchDec;
dest[dOff++] = (byte) (matchDec >>> 8);
// encode match len
matchLen -= 4;
if (dOff + (1 + LAST_LITERALS) + (matchLen >>> 8) > destEnd) {
throw new LZ4Exception("maxDestLen is too small");
}
if (matchLen >= ML_MASK) {
token |= ML_MASK;
dOff = writeLen(matchLen - RUN_MASK, dest, dOff);
} else {
token |= matchLen;
}
dest[tokenOff] = (byte) token;
return dOff;
}
static int commonBytesBackward(byte[] b, int o1, int o2, int l1, int l2) {
int count = 0;
while (o1 > l1 && o2 > l2 && readByte(b, --o1) == readByte(b, --o2)) {
++count;
}
return count;
}
static int lastLiterals(byte[] src, int sOff, int srcLen, byte[] dest, int dOff, int destEnd) {
return LZ4SafeUtils.lastLiterals(src, sOff, srcLen, dest, dOff, destEnd);
}
}

View File

@ -0,0 +1,125 @@
package net.jpountz.util;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import com.boydti.fawe.Fawe;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
/** FOR INTERNAL USE ONLY */
public enum Native {
;
private enum OS {
// Even on Windows, the default compiler from cpptasks (gcc) uses .so as a shared lib extension
WINDOWS("win32", "so"), LINUX("linux", "so"), MAC("darwin", "dylib"), SOLARIS("solaris", "so");
public final String name, libExtension;
OS(String name, String libExtension) {
this.name = name;
this.libExtension = libExtension;
}
}
private static String arch() {
return System.getProperty("os.arch");
}
private static OS os() {
String osName = System.getProperty("os.name");
if (osName.contains("Linux")) {
return OS.LINUX;
} else if (osName.contains("Mac")) {
return OS.MAC;
} else if (osName.contains("Windows")) {
return OS.WINDOWS;
} else if (osName.contains("Solaris") || osName.contains("SunOS")) {
return OS.SOLARIS;
} else {
throw new UnsupportedOperationException("Unsupported operating system: "
+ osName);
}
}
private static String resourceName() {
OS os = os();
return "/" + os.name + "/" + arch() + "/liblz4-java." + os.libExtension;
}
private static boolean loaded = false;
public static synchronized boolean isLoaded() {
return loaded;
}
public static synchronized void load() {
if (loaded) {
return;
}
String resourceName = resourceName();
System.out.println("NAME: " + resourceName);
System.out.println("WORK: " + Fawe.class.getResourceAsStream("/LICENSE"));
System.out.println("WORK: " + Fawe.class.getResourceAsStream("/win32/amd64/liblz4-java.so"));
InputStream is = Fawe.class.getResourceAsStream(resourceName);
if (is == null) {
throw new UnsupportedOperationException("Unsupported OS/arch, cannot find " + resourceName + ". Please try building from source.");
}
File tempLib;
try {
tempLib = File.createTempFile("liblz4-java", "." + os().libExtension);
// copy to tempLib
FileOutputStream out = new FileOutputStream(tempLib);
try {
byte[] buf = new byte[4096];
while (true) {
int read = is.read(buf);
if (read == -1) {
break;
}
out.write(buf, 0, read);
}
try {
out.close();
out = null;
} catch (IOException e) {
// ignore
}
System.load(tempLib.getAbsolutePath());
loaded = true;
} finally {
try {
if (out != null) {
out.close();
}
} catch (IOException e) {
// ignore
}
if (tempLib != null && tempLib.exists()) {
if (!loaded) {
tempLib.delete();
} else {
// try to delete on exit, does it work on Windows?
tempLib.deleteOnExit();
}
}
}
} catch (IOException e) {
throw new ExceptionInInitializerError("Cannot unpack liblz4-java");
}
}
}

View File

@ -0,0 +1,95 @@
package net.jpountz.util;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteOrder;
public enum SafeUtils {
;
public static void checkRange(byte[] buf, int off) {
if (off < 0 || off >= buf.length) {
throw new ArrayIndexOutOfBoundsException(off);
}
}
public static void checkRange(byte[] buf, int off, int len) {
checkLength(len);
if (len > 0) {
checkRange(buf, off);
checkRange(buf, off + len - 1);
}
}
public static void checkLength(int len) {
if (len < 0) {
throw new IllegalArgumentException("lengths must be >= 0");
}
}
public static byte readByte(byte[] buf, int i) {
return buf[i];
}
public static int readIntBE(byte[] buf, int i) {
return ((buf[i] & 0xFF) << 24) | ((buf[i+1] & 0xFF) << 16) | ((buf[i+2] & 0xFF) << 8) | (buf[i+3] & 0xFF);
}
public static int readIntLE(byte[] buf, int i) {
return (buf[i] & 0xFF) | ((buf[i+1] & 0xFF) << 8) | ((buf[i+2] & 0xFF) << 16) | ((buf[i+3] & 0xFF) << 24);
}
public static int readInt(byte[] buf, int i) {
if (Utils.NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
return readIntBE(buf, i);
} else {
return readIntLE(buf, i);
}
}
public static long readLongLE(byte[] buf, int i) {
return (buf[i] & 0xFFL) | ((buf[i+1] & 0xFFL) << 8) | ((buf[i+2] & 0xFFL) << 16) | ((buf[i+3] & 0xFFL) << 24)
| ((buf[i+4] & 0xFFL) << 32) | ((buf[i+5] & 0xFFL) << 40) | ((buf[i+6] & 0xFFL) << 48) | ((buf[i+7] & 0xFFL) << 56);
}
public static void writeShortLE(byte[] buf, int off, int v) {
buf[off++] = (byte) v;
buf[off++] = (byte) (v >>> 8);
}
public static void writeInt(int[] buf, int off, int v) {
buf[off] = v;
}
public static int readInt(int[] buf, int off) {
return buf[off];
}
public static void writeByte(byte[] dest, int off, int i) {
dest[off] = (byte) i;
}
public static void writeShort(short[] buf, int off, int v) {
buf[off] = (short) v;
}
public static int readShortLE(byte[] buf, int i) {
return (buf[i] & 0xFF) | ((buf[i+1] & 0xFF) << 8);
}
public static int readShort(short[] buf, int off) {
return buf[off] & 0xFFFF;
}
}

View File

@ -0,0 +1,147 @@
package net.jpountz.util;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.util.Utils.NATIVE_BYTE_ORDER;
import java.lang.reflect.Field;
import java.nio.ByteOrder;
import sun.misc.Unsafe;
public enum UnsafeUtils {
;
private static final Unsafe UNSAFE;
private static final long BYTE_ARRAY_OFFSET;
private static final int BYTE_ARRAY_SCALE;
private static final long INT_ARRAY_OFFSET;
private static final int INT_ARRAY_SCALE;
private static final long SHORT_ARRAY_OFFSET;
private static final int SHORT_ARRAY_SCALE;
static {
try {
Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe");
theUnsafe.setAccessible(true);
UNSAFE = (Unsafe) theUnsafe.get(null);
BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class);
BYTE_ARRAY_SCALE = UNSAFE.arrayIndexScale(byte[].class);
INT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(int[].class);
INT_ARRAY_SCALE = UNSAFE.arrayIndexScale(int[].class);
SHORT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(short[].class);
SHORT_ARRAY_SCALE = UNSAFE.arrayIndexScale(short[].class);
} catch (IllegalAccessException e) {
throw new ExceptionInInitializerError("Cannot access Unsafe");
} catch (NoSuchFieldException e) {
throw new ExceptionInInitializerError("Cannot access Unsafe");
} catch (SecurityException e) {
throw new ExceptionInInitializerError("Cannot access Unsafe");
}
}
public static void checkRange(byte[] buf, int off) {
SafeUtils.checkRange(buf, off);
}
public static void checkRange(byte[] buf, int off, int len) {
SafeUtils.checkRange(buf, off, len);
}
public static void checkLength(int len) {
SafeUtils.checkLength(len);
}
public static byte readByte(byte[] src, int srcOff) {
return UNSAFE.getByte(src, BYTE_ARRAY_OFFSET + BYTE_ARRAY_SCALE * srcOff);
}
public static void writeByte(byte[] src, int srcOff, byte value) {
UNSAFE.putByte(src, BYTE_ARRAY_OFFSET + BYTE_ARRAY_SCALE * srcOff, (byte) value);
}
public static void writeByte(byte[] src, int srcOff, int value) {
writeByte(src, srcOff, (byte) value);
}
public static long readLong(byte[] src, int srcOff) {
return UNSAFE.getLong(src, BYTE_ARRAY_OFFSET + srcOff);
}
public static long readLongLE(byte[] src, int srcOff) {
long i = readLong(src, srcOff);
if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
i = Long.reverseBytes(i);
}
return i;
}
public static void writeLong(byte[] dest, int destOff, long value) {
UNSAFE.putLong(dest, BYTE_ARRAY_OFFSET + destOff, value);
}
public static int readInt(byte[] src, int srcOff) {
return UNSAFE.getInt(src, BYTE_ARRAY_OFFSET + srcOff);
}
public static int readIntLE(byte[] src, int srcOff) {
int i = readInt(src, srcOff);
if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
i = Integer.reverseBytes(i);
}
return i;
}
public static void writeInt(byte[] dest, int destOff, int value) {
UNSAFE.putInt(dest, BYTE_ARRAY_OFFSET + destOff, value);
}
public static short readShort(byte[] src, int srcOff) {
return UNSAFE.getShort(src, BYTE_ARRAY_OFFSET + srcOff);
}
public static int readShortLE(byte[] src, int srcOff) {
short s = readShort(src, srcOff);
if (NATIVE_BYTE_ORDER == ByteOrder.BIG_ENDIAN) {
s = Short.reverseBytes(s);
}
return s & 0xFFFF;
}
public static void writeShort(byte[] dest, int destOff, short value) {
UNSAFE.putShort(dest, BYTE_ARRAY_OFFSET + destOff, value);
}
public static void writeShortLE(byte[] buf, int off, int v) {
writeByte(buf, off, (byte) v);
writeByte(buf, off + 1, (byte) (v >>> 8));
}
public static int readInt(int[] src, int srcOff) {
return UNSAFE.getInt(src, INT_ARRAY_OFFSET + INT_ARRAY_SCALE * srcOff);
}
public static void writeInt(int[] dest, int destOff, int value) {
UNSAFE.putInt(dest, INT_ARRAY_OFFSET + INT_ARRAY_SCALE * destOff, value);
}
public static int readShort(short[] src, int srcOff) {
return UNSAFE.getShort(src, SHORT_ARRAY_OFFSET + SHORT_ARRAY_SCALE * srcOff) & 0xFFFF;
}
public static void writeShort(short[] dest, int destOff, int value) {
UNSAFE.putShort(dest, SHORT_ARRAY_OFFSET + SHORT_ARRAY_SCALE * destOff, (short) value);
}
}

View File

@ -0,0 +1,35 @@
package net.jpountz.util;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteOrder;
public enum Utils {
;
public static final ByteOrder NATIVE_BYTE_ORDER = ByteOrder.nativeOrder();
private static final boolean unalignedAccessAllowed;
static {
String arch = System.getProperty("os.arch");
unalignedAccessAllowed = arch.equals("i386") || arch.equals("x86")
|| arch.equals("amd64") || arch.equals("x86_64");
}
public static boolean isUnalignedAccessAllowed() {
return unalignedAccessAllowed;
}
}

View File

@ -0,0 +1,22 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>Utility classes.</p>
</body>
</html>

View File

@ -0,0 +1,42 @@
package net.jpountz.xxhash;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.xxhash.XXHashConstants.PRIME1;
import static net.jpountz.xxhash.XXHashConstants.PRIME2;
abstract class AbstractStreamingXXHash32Java extends StreamingXXHash32 {
int v1, v2, v3, v4, memSize;
long totalLen;
final byte[] memory;
AbstractStreamingXXHash32Java(int seed) {
super(seed);
memory = new byte[16];
reset();
}
@Override
public void reset() {
v1 = seed + PRIME1 + PRIME2;
v2 = seed + PRIME2;
v3 = seed + 0;
v4 = seed - PRIME1;
totalLen = 0;
memSize = 0;
}
}

View File

@ -0,0 +1,43 @@
package net.jpountz.xxhash;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.xxhash.XXHashConstants.PRIME64_1;
import static net.jpountz.xxhash.XXHashConstants.PRIME64_2;
abstract class AbstractStreamingXXHash64Java extends StreamingXXHash64 {
int memSize;
long v1, v2, v3, v4;
long totalLen;
final byte[] memory;
AbstractStreamingXXHash64Java(long seed) {
super(seed);
memory = new byte[32];
reset();
}
@Override
public void reset() {
v1 = seed + PRIME64_1 + PRIME64_2;
v2 = seed + PRIME64_2;
v3 = seed + 0;
v4 = seed - PRIME64_1;
totalLen = 0;
memSize = 0;
}
}

View File

@ -0,0 +1,111 @@
package net.jpountz.xxhash;
import java.util.zip.Checksum;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Streaming interface for {@link XXHash32}.
* <p>
* This API is compatible with the {@link XXHash32 block API} and the following
* code samples are equivalent:
* <pre class="prettyprint">
* int hash(XXHashFactory xxhashFactory, byte[] buf, int off, int len, int seed) {
* return xxhashFactory.hash32().hash(buf, off, len, seed);
* }
* </pre>
* <pre class="prettyprint">
* int hash(XXHashFactory xxhashFactory, byte[] buf, int off, int len, int seed) {
* StreamingXXHash32 sh32 = xxhashFactory.newStreamingHash32(seed);
* sh32.update(buf, off, len);
* return sh32.getValue();
* }
* </pre>
* <p>
* Instances of this class are <b>not</b> thread-safe.
*/
public abstract class StreamingXXHash32 {
interface Factory {
StreamingXXHash32 newStreamingHash(int seed);
}
final int seed;
StreamingXXHash32(int seed) {
this.seed = seed;
}
/**
* Get the value of the checksum.
*/
public abstract int getValue();
/**
* Update the value of the hash with buf[off:off+len].
*/
public abstract void update(byte[] buf, int off, int len);
/**
* Reset this instance to the state it had right after instantiation. The
* seed remains unchanged.
*/
public abstract void reset();
@Override
public String toString() {
return getClass().getSimpleName() + "(seed=" + seed + ")";
}
/**
* Return a {@link Checksum} view of this instance. Modifications to the view
* will modify this instance too and vice-versa.
*/
public final Checksum asChecksum() {
return new Checksum() {
@Override
public long getValue() {
return StreamingXXHash32.this.getValue() & 0xFFFFFFFL;
}
@Override
public void reset() {
StreamingXXHash32.this.reset();
}
@Override
public void update(int b) {
StreamingXXHash32.this.update(new byte[] {(byte) b}, 0, 1);
}
@Override
public void update(byte[] b, int off, int len) {
StreamingXXHash32.this.update(b, off, len);
}
@Override
public String toString() {
return StreamingXXHash32.this.toString();
}
};
}
}

View File

@ -0,0 +1,71 @@
package net.jpountz.xxhash;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
final class StreamingXXHash32JNI extends StreamingXXHash32 {
static class Factory implements StreamingXXHash32.Factory {
public static final StreamingXXHash32.Factory INSTANCE = new Factory();
@Override
public StreamingXXHash32 newStreamingHash(int seed) {
return new StreamingXXHash32JNI(seed);
}
}
private long state;
StreamingXXHash32JNI(int seed) {
super(seed);
state = XXHashJNI.XXH32_init(seed);
}
private void checkState() {
if (state == 0) {
throw new AssertionError("Already finalized");
}
}
@Override
public void reset() {
checkState();
XXHashJNI.XXH32_free(state);
state = XXHashJNI.XXH32_init(seed);
}
@Override
public int getValue() {
checkState();
return XXHashJNI.XXH32_digest(state);
}
@Override
public void update(byte[] bytes, int off, int len) {
checkState();
XXHashJNI.XXH32_update(state, bytes, off, len);
}
@Override
protected void finalize() throws Throwable {
super.finalize();
// free memory
XXHashJNI.XXH32_free(state);
state = 0;
}
}

View File

@ -0,0 +1,111 @@
package net.jpountz.xxhash;
import java.util.zip.Checksum;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Streaming interface for {@link XXHash64}.
* <p>
* This API is compatible with the {@link XXHash64 block API} and the following
* code samples are equivalent:
* <pre class="prettyprint">
* long hash(XXHashFactory xxhashFactory, byte[] buf, int off, int len, long seed) {
* return xxhashFactory.hash64().hash(buf, off, len, seed);
* }
* </pre>
* <pre class="prettyprint">
* long hash(XXHashFactory xxhashFactory, byte[] buf, int off, int len, long seed) {
* StreamingXXHash64 sh64 = xxhashFactory.newStreamingHash64(seed);
* sh64.update(buf, off, len);
* return sh64.getValue();
* }
* </pre>
* <p>
* Instances of this class are <b>not</b> thread-safe.
*/
public abstract class StreamingXXHash64 {
interface Factory {
StreamingXXHash64 newStreamingHash(long seed);
}
final long seed;
StreamingXXHash64(long seed) {
this.seed = seed;
}
/**
* Get the value of the checksum.
*/
public abstract long getValue();
/**
* Update the value of the hash with buf[off:off+len].
*/
public abstract void update(byte[] buf, int off, int len);
/**
* Reset this instance to the state it had right after instantiation. The
* seed remains unchanged.
*/
public abstract void reset();
@Override
public String toString() {
return getClass().getSimpleName() + "(seed=" + seed + ")";
}
/**
* Return a {@link Checksum} view of this instance. Modifications to the view
* will modify this instance too and vice-versa.
*/
public final Checksum asChecksum() {
return new Checksum() {
@Override
public long getValue() {
return StreamingXXHash64.this.getValue();
}
@Override
public void reset() {
StreamingXXHash64.this.reset();
}
@Override
public void update(int b) {
StreamingXXHash64.this.update(new byte[] {(byte) b}, 0, 1);
}
@Override
public void update(byte[] b, int off, int len) {
StreamingXXHash64.this.update(b, off, len);
}
@Override
public String toString() {
return StreamingXXHash64.this.toString();
}
};
}
}

View File

@ -0,0 +1,71 @@
package net.jpountz.xxhash;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
final class StreamingXXHash64JNI extends StreamingXXHash64 {
static class Factory implements StreamingXXHash64.Factory {
public static final StreamingXXHash64.Factory INSTANCE = new Factory();
@Override
public StreamingXXHash64 newStreamingHash(long seed) {
return new StreamingXXHash64JNI(seed);
}
}
private long state;
StreamingXXHash64JNI(long seed) {
super(seed);
state = XXHashJNI.XXH64_init(seed);
}
private void checkState() {
if (state == 0) {
throw new AssertionError("Already finalized");
}
}
@Override
public void reset() {
checkState();
XXHashJNI.XXH64_free(state);
state = XXHashJNI.XXH64_init(seed);
}
@Override
public long getValue() {
checkState();
return XXHashJNI.XXH64_digest(state);
}
@Override
public void update(byte[] bytes, int off, int len) {
checkState();
XXHashJNI.XXH64_update(state, bytes, off, len);
}
@Override
protected void finalize() throws Throwable {
super.finalize();
// free memory
XXHashJNI.XXH64_free(state);
state = 0;
}
}

View File

@ -0,0 +1,55 @@
package net.jpountz.xxhash;
import java.nio.ByteBuffer;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* A 32-bits hash.
* <p>
* Instances of this class are thread-safe.
*/
public abstract class XXHash32 {
/**
* Compute the 32-bits hash of <code>buf[off:off+len]</code> using seed
* <code>seed</code>.
*/
public abstract int hash(byte[] buf, int off, int len, int seed);
/**
* Compute the hash of the given slice of the {@link ByteBuffer}.
* {@link ByteBuffer#position() position} and {@link ByteBuffer#limit() limit}
* are not modified.
*/
public abstract int hash(ByteBuffer buf, int off, int len, int seed);
/**
* Compute the hash of the given {@link ByteBuffer}. The
* {@link ByteBuffer#position() position} is moved in order to reflect bytes
* which have been read.
*/
public final int hash(ByteBuffer buf, int seed) {
final int hash = hash(buf, buf.position(), buf.remaining(), seed);
buf.position(buf.limit());
return hash;
}
@Override
public String toString() {
return getClass().getSimpleName();
}
}

View File

@ -0,0 +1,49 @@
package net.jpountz.xxhash;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.util.ByteBufferUtils.checkRange;
import static net.jpountz.util.SafeUtils.checkRange;
import java.nio.ByteBuffer;
final class XXHash32JNI extends XXHash32 {
public static final XXHash32 INSTANCE = new XXHash32JNI();
private static XXHash32 SAFE_INSTANCE;
@Override
public int hash(byte[] buf, int off, int len, int seed) {
checkRange(buf, off, len);
return XXHashJNI.XXH32(buf, off, len, seed);
}
@Override
public int hash(ByteBuffer buf, int off, int len, int seed) {
if (buf.isDirect()) {
checkRange(buf, off, len);
return XXHashJNI.XXH32BB(buf, off, len, seed);
} else if (buf.hasArray()) {
return hash(buf.array(), off + buf.arrayOffset(), len, seed);
} else {
XXHash32 safeInstance = SAFE_INSTANCE;
if (safeInstance == null) {
safeInstance = SAFE_INSTANCE = XXHashFactory.safeInstance().hash32();
}
return safeInstance.hash(buf, off, len, seed);
}
}
}

View File

@ -0,0 +1,55 @@
package net.jpountz.xxhash;
import java.nio.ByteBuffer;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* A 64-bits hash.
* <p>
* Instances of this class are thread-safe.
*/
public abstract class XXHash64 {
/**
* Compute the 64-bits hash of <code>buf[off:off+len]</code> using seed
* <code>seed</code>.
*/
public abstract long hash(byte[] buf, int off, int len, long seed);
/**
* Compute the hash of the given slice of the {@link ByteBuffer}.
* {@link ByteBuffer#position() position} and {@link ByteBuffer#limit() limit}
* are not modified.
*/
public abstract long hash(ByteBuffer buf, int off, int len, long seed);
/**
* Compute the hash of the given {@link ByteBuffer}. The
* {@link ByteBuffer#position() position} is moved in order to reflect bytes
* which have been read.
*/
public final long hash(ByteBuffer buf, long seed) {
final long hash = hash(buf, buf.position(), buf.remaining(), seed);
buf.position(buf.limit());
return hash;
}
@Override
public String toString() {
return getClass().getSimpleName();
}
}

View File

@ -0,0 +1,49 @@
package net.jpountz.xxhash;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import static net.jpountz.util.ByteBufferUtils.checkRange;
import static net.jpountz.util.SafeUtils.checkRange;
import java.nio.ByteBuffer;
final class XXHash64JNI extends XXHash64 {
public static final XXHash64 INSTANCE = new XXHash64JNI();
private static XXHash64 SAFE_INSTANCE;
@Override
public long hash(byte[] buf, int off, int len, long seed) {
checkRange(buf, off, len);
return XXHashJNI.XXH64(buf, off, len, seed);
}
@Override
public long hash(ByteBuffer buf, int off, int len, long seed) {
if (buf.isDirect()) {
checkRange(buf, off, len);
return XXHashJNI.XXH64BB(buf, off, len, seed);
} else if (buf.hasArray()) {
return hash(buf.array(), off + buf.arrayOffset(), len, seed);
} else {
XXHash64 safeInstance = SAFE_INSTANCE;
if (safeInstance == null) {
safeInstance = SAFE_INSTANCE = XXHashFactory.safeInstance().hash64();
}
return safeInstance.hash(buf, off, len, seed);
}
}
}

View File

@ -0,0 +1,31 @@
package net.jpountz.xxhash;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
enum XXHashConstants {
;
static final int PRIME1 = -1640531535;
static final int PRIME2 = -2048144777;
static final int PRIME3 = -1028477379;
static final int PRIME4 = 668265263;
static final int PRIME5 = 374761393;
static final long PRIME64_1 = -7046029288634856825L; //11400714785074694791
static final long PRIME64_2 = -4417276706812531889L; //14029467366897019727
static final long PRIME64_3 = 1609587929392839161L;
static final long PRIME64_4 = -8796714831421723037L; //9650029242287828579
static final long PRIME64_5 = 2870177450012600261L;
}

View File

@ -0,0 +1,220 @@
package net.jpountz.xxhash;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.lang.reflect.Field;
import java.util.Random;
import net.jpountz.util.Native;
import net.jpountz.util.Utils;
/**
* Entry point to get {@link XXHash32} and {@link StreamingXXHash32} instances.
* <p>
* This class has 3 instances<ul>
* <li>a {@link #nativeInstance() native} instance which is a JNI binding to
* <a href="http://code.google.com/p/xxhash/">the original LZ4 C implementation</a>.
* <li>a {@link #safeInstance() safe Java} instance which is a pure Java port
* of the original C library,</li>
* <li>an {@link #unsafeInstance() unsafe Java} instance which is a Java port
* using the unofficial {@link sun.misc.Unsafe} API.
* </ul>
* <p>
* Only the {@link #safeInstance() safe instance} is guaranteed to work on your
* JVM, as a consequence it is advised to use the {@link #fastestInstance()} or
* {@link #fastestJavaInstance()} to pull a {@link XXHashFactory} instance.
* <p>
* All methods from this class are very costly, so you should get an instance
* once, and then reuse it whenever possible. This is typically done by storing
* a {@link XXHashFactory} instance in a static field.
*/
public final class XXHashFactory {
private static XXHashFactory instance(String impl) {
try {
return new XXHashFactory(impl);
} catch (Exception e) {
throw new AssertionError(e);
}
}
private static XXHashFactory NATIVE_INSTANCE,
JAVA_UNSAFE_INSTANCE,
JAVA_SAFE_INSTANCE;
/** Return a {@link XXHashFactory} that returns {@link XXHash32} instances that
* are native bindings to the original C API.
* <p>
* Please note that this instance has some traps you should be aware of:<ol>
* <li>Upon loading this instance, files will be written to the temporary
* directory of the system. Although these files are supposed to be deleted
* when the JVM exits, they might remain on systems that don't support
* removal of files being used such as Windows.
* <li>The instance can only be loaded once per JVM. This can be a problem
* if your application uses multiple class loaders (such as most servlet
* containers): this instance will only be available to the children of the
* class loader which has loaded it. As a consequence, it is advised to
* either not use this instance in webapps or to put this library in the lib
* directory of your servlet container so that it is loaded by the system
* class loader.
* </ol>
*/
public static synchronized XXHashFactory nativeInstance() {
if (NATIVE_INSTANCE == null) {
NATIVE_INSTANCE = instance("JNI");
}
return NATIVE_INSTANCE;
}
/** Return a {@link XXHashFactory} that returns {@link XXHash32} instances that
* are written with Java's official API. */
public static synchronized XXHashFactory safeInstance() {
if (JAVA_SAFE_INSTANCE == null) {
JAVA_SAFE_INSTANCE = instance("JavaSafe");
}
return JAVA_SAFE_INSTANCE;
}
/** Return a {@link XXHashFactory} that returns {@link XXHash32} instances that
* may use {@link sun.misc.Unsafe} to speed up hashing. */
public static synchronized XXHashFactory unsafeInstance() {
if (JAVA_UNSAFE_INSTANCE == null) {
JAVA_UNSAFE_INSTANCE = instance("JavaUnsafe");
}
return JAVA_UNSAFE_INSTANCE;
}
/**
* Return the fastest available {@link XXHashFactory} instance which does not
* rely on JNI bindings. It first tries to load the
* {@link #unsafeInstance() unsafe instance}, and then the
* {@link #safeInstance() safe Java instance} if the JVM doesn't have a
* working {@link sun.misc.Unsafe}.
*/
public static XXHashFactory fastestJavaInstance() {
if (Utils.isUnalignedAccessAllowed()) {
try {
return unsafeInstance();
} catch (Throwable t) {
return safeInstance();
}
} else {
return safeInstance();
}
}
/**
* Return the fastest available {@link XXHashFactory} instance. If the class
* loader is the system class loader and if the
* {@link #nativeInstance() native instance} loads successfully, then the
* {@link #nativeInstance() native instance} is returned, otherwise the
* {@link #fastestJavaInstance() fastest Java instance} is returned.
* <p>
* Please read {@link #nativeInstance() javadocs of nativeInstance()} before
* using this method.
*/
public static XXHashFactory fastestInstance() {
if (Native.isLoaded()
|| Native.class.getClassLoader() == ClassLoader.getSystemClassLoader()) {
try {
return nativeInstance();
} catch (Throwable t) {
return fastestJavaInstance();
}
} else {
return fastestJavaInstance();
}
}
@SuppressWarnings("unchecked")
private static <T> T classInstance(String cls) throws NoSuchFieldException, SecurityException, ClassNotFoundException, IllegalArgumentException, IllegalAccessException {
ClassLoader loader = XXHashFactory.class.getClassLoader();
loader = loader == null ? ClassLoader.getSystemClassLoader() : loader;
final Class<?> c = loader.loadClass(cls);
Field f = c.getField("INSTANCE");
return (T) f.get(null);
}
private final String impl;
private final XXHash32 hash32;
private final XXHash64 hash64;
private final StreamingXXHash32.Factory streamingHash32Factory;
private final StreamingXXHash64.Factory streamingHash64Factory;
private XXHashFactory(String impl) throws ClassNotFoundException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException {
this.impl = impl;
hash32 = classInstance("net.jpountz.xxhash.XXHash32" + impl);
streamingHash32Factory = classInstance("net.jpountz.xxhash.StreamingXXHash32" + impl + "$Factory");
hash64 = classInstance("net.jpountz.xxhash.XXHash64" + impl);
streamingHash64Factory = classInstance("net.jpountz.xxhash.StreamingXXHash64" + impl + "$Factory");
// make sure it can run
final byte[] bytes = new byte[100];
final Random random = new Random();
random.nextBytes(bytes);
final int seed = random.nextInt();
final int h1 = hash32.hash(bytes, 0, bytes.length, seed);
final StreamingXXHash32 streamingHash32 = newStreamingHash32(seed);
streamingHash32.update(bytes, 0, bytes.length);
final int h2 = streamingHash32.getValue();
final long h3 = hash64.hash(bytes, 0, bytes.length, seed);
final StreamingXXHash64 streamingHash64 = newStreamingHash64(seed);
streamingHash64.update(bytes, 0, bytes.length);
final long h4 = streamingHash64.getValue();
if (h1 != h2) {
throw new AssertionError();
}
if (h3 != h4) {
throw new AssertionError();
}
}
/** Return a {@link XXHash32} instance. */
public XXHash32 hash32() {
return hash32;
}
/** Return a {@link XXHash64} instance. */
public XXHash64 hash64() {
return hash64;
}
/**
* Return a new {@link StreamingXXHash32} instance.
*/
public StreamingXXHash32 newStreamingHash32(int seed) {
return streamingHash32Factory.newStreamingHash(seed);
}
/**
* Return a new {@link StreamingXXHash64} instance.
*/
public StreamingXXHash64 newStreamingHash64(long seed) {
return streamingHash64Factory.newStreamingHash(seed);
}
/** Prints the fastest instance. */
public static void main(String[] args) {
System.out.println("Fastest instance is " + fastestInstance());
System.out.println("Fastest Java instance is " + fastestJavaInstance());
}
@Override
public String toString() {
return getClass().getSimpleName() + ":" + impl;
}
}

View File

@ -0,0 +1,43 @@
package net.jpountz.xxhash;
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteBuffer;
import net.jpountz.util.Native;
enum XXHashJNI {
;
static {
Native.load();
init();
}
private static native void init();
static native int XXH32(byte[] input, int offset, int len, int seed);
static native int XXH32BB(ByteBuffer input, int offset, int len, int seed);
static native long XXH32_init(int seed);
static native void XXH32_update(long state, byte[] input, int offset, int len);
static native int XXH32_digest(long state);
static native void XXH32_free(long state);
static native long XXH64(byte[] input, int offset, int len, long seed);
static native long XXH64BB(ByteBuffer input, int offset, int len, long seed);
static native long XXH64_init(long seed);
static native void XXH64_update(long state, byte[] input, int offset, int len);
static native long XXH64_digest(long state);
static native void XXH64_free(long state);
}

View File

@ -0,0 +1,65 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>xxhash hashing. This package supports both block hashing via
{@link net.jpountz.xxhash.XXHash32} and streaming hashing via
{@link net.jpountz.xxhash.StreamingXXHash32}. Have a look at
{@link net.jpountz.xxhash.XXHashFactory} to know how to get instances of these
interfaces.</p>
<p>Streaming hashing is a little slower but doesn't require to load the whole
stream into memory.</p>
<p>Sample block usage:</p>
<pre class="prettyprint">
XXHashFactory factory = XXHashFactory.fastestInstance();
byte[] data = "12345345234572".getBytes("UTF-8");
XXHash32 hash32 = factory.hash32();
int seed = 0x9747b28c; // used to initialize the hash value, use whatever
// value you want, but always the same
int hash = hash32.hash(data, 0, data.length, seed);
</pre>
<p>Sample streaming usage:</p>
<pre class="prettyprint">
XXHashFactory factory = XXHashFactory.fastestInstance();
byte[] data = "12345345234572".getBytes("UTF-8");
ByteArrayInputStream in = new ByteArrayInputStream(data);
int seed = 0x9747b28c; // used to initialize the hash value, use whatever
// value you want, but always the same
StreamingXXHash32 hash32 = factory.newStreamingHash32(seed);
byte[] buf = new byte[8]; // for real-world usage, use a larger buffer, like 8192 bytes
for (;;) {
int read = in.read(buf);
if (read == -1) {
break;
}
hash32.update(buf, 0, read);
}
int hash = hash32.getValue();
</pre>
</body>
</html>

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -1,88 +0,0 @@
#################################################
ForgeGradle 2.1-SNAPSHOT-unknown
https://github.com/MinecraftForge/ForgeGradle
#################################################
Powered by MCP unknown
http://modcoderpack.com
by: Searge, ProfMobius, Fesh0r,
R4wk, ZeuX, IngisKahn, bspkrs
#################################################
Version string 'unspecified' does not match SemVer specification
You should try SemVer : http://semver.org/
:core:compileJava
:forge:deobfCompileDummyTask
:forge:deobfProvidedDummyTask
:forge:extractDependencyATs SKIPPED
:forge:extractMcpData
:core:compileJava UP-TO-DATE
:core:processResources UP-TO-DATE
:core:classes UP-TO-DATE
:core:jar UP-TO-DATE
:forge:extractMcpData SKIPPED
:forge:extractMcpMappings SKIPPED
:forge:genSrgs SKIPPED
:forge:getVersionJson
:forge:downloadServer SKIPPED
:forge:splitServerJar SKIPPED
:forge:deobfMcMCP SKIPPED
:forge:sourceApiJava UP-TO-DATE
:forge:compileApiJava UP-TO-DATE
:forge:processApiResources UP-TO-DATE
:forge:apiClasses UP-TO-DATE
:forge:sourceMainJava UP-TO-DATE
:forge:compileJavaC:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'modid()' in type 'Mod': class file for net.minecraftforge.fml.common.Mod not found
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'name()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'version()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'acceptableRemoteVersions()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'dependencies()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'acceptedMinecraftVersions()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'value()' in type 'Instance': class file for net.minecraftforge.fml.common.Mod$Instance not found
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'serverSide()' in type 'SidedProxy': class file for net.minecraftforge.fml.common.SidedProxy not found
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'clientSide()' in type 'SidedProxy'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'modid()' in type 'Mod': class file for net.minecraftforge.fml.common.Mod not found
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'name()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'version()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'acceptableRemoteVersions()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'dependencies()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'acceptedMinecraftVersions()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'value()' in type 'Instance': class file for net.minecraftforge.fml.common.Mod$Instance not found
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'serverSide()' in type 'SidedProxy': class file for net.minecraftforge.fml.common.SidedProxy not found
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'clientSide()' in type 'SidedProxy'
Note: Writing plugin metadata to file:/C:/Users/Jesse/Desktop/OTHER/GitHub/FastAsyncWorldEdit/forge/build/classes/main/mcmod.info
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'modid()' in type 'Mod': class file for net.minecraftforge.fml.common.Mod not found
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'name()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'version()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'acceptableRemoteVersions()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'dependencies()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'acceptedMinecraftVersions()' in type 'Mod'
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'value()' in type 'Instance': class file for net.minecraftforge.fml.common.Mod$Instance not found
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'serverSide()' in type 'SidedProxy': class file for net.minecraftforge.fml.common.SidedProxy not found
C:\Users\Jesse\.gradle\caches\modules-2\files-2.1\com.sk89q.worldedit\worldedit-forge-mc1.8.9\6.1.1\dffd7e1882eba256eb2132fe315682c1d26522b1\worldedit-forge-mc1.8.9-6.1.1.jar(com/sk89q/worldedit/forge/ForgeWorldEdit.class): warning: Cannot find annotation method 'clientSide()' in type 'SidedProxy'
Note: Some input files use or override a deprecated API.
Note: Recompile with -Xlint:deprecation for details.
Note: Some input files use unchecked or unsafe operations.
Note: Recompile with -Xlint:unchecked for details.
27 warnings
:forge:processResources UP-TO-DATE
:forge:classes
:forge:jar
:forge:sourceTestJava UP-TO-DATE
:forge:compileTestJava UP-TO-DATE
:forge:processTestResources UP-TO-DATE
:forge:testClasses UP-TO-DATE
:forge:test UP-TO-DATE
:forge:reobfJar
:forge:shadowJar
:forge:reobfShadowJar
:forge:extractRangemapReplacedMain
C:\Users\Jesse\Desktop\OTHER\GitHub\FastAsyncWorldEdit\forge\build\sources\main\java
:forge:retromapReplacedMain UP-TO-DATE
:forge:sourceJar UP-TO-DATE
:forge:assemble
:forge:check UP-TO-DATE
:forge:build
BUILD SUCCESSFUL
Total time: 36.184 secs

View File

@ -1,4 +1,4 @@
org.gradle.daemon=true
org.gradle.daemon=false
org.gradle.jvmargs=-Xmx2048m -XX:MaxPermSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8
org.gradle.configureondemand=true
org.gradle.parallel=true