From a2589d44933006f4e464387e59695e80aaf2909d Mon Sep 17 00:00:00 2001 From: Jesse Boyd Date: Wed, 17 Aug 2016 12:10:03 +1000 Subject: [PATCH] More work on optimized MCA reader + Fix for 1.7.10 gson import error --- bukkit1710/build.gradle | 3 + core/src/main/java/com/boydti/fawe/Fawe.java | 15 +- .../main/java/com/boydti/fawe/config/BBC.java | 4 +- .../java/com/boydti/fawe/jnbt/MCAChunk.java | 22 +++ .../java/com/boydti/fawe/jnbt/MCAFile.java | 138 +++++++++++++++++- .../java/com/boydti/fawe/jnbt/MCAQueue.java | 138 ++++++++++++++++++ .../clipboard/DiskOptimizedClipboard.java | 2 +- .../{ => io}/BufferedRandomAccessFile.java | 4 +- .../object/io/RandomAccessInputStream.java | 28 ++++ .../com/boydti/fawe/wrappers/FakePlayer.java | 8 +- .../java/com/sk89q/jnbt/NBTInputStream.java | 53 ++++++- 11 files changed, 391 insertions(+), 24 deletions(-) create mode 100644 core/src/main/java/com/boydti/fawe/jnbt/MCAChunk.java create mode 100644 core/src/main/java/com/boydti/fawe/jnbt/MCAQueue.java rename core/src/main/java/com/boydti/fawe/object/{ => io}/BufferedRandomAccessFile.java (99%) create mode 100644 core/src/main/java/com/boydti/fawe/object/io/RandomAccessInputStream.java diff --git a/bukkit1710/build.gradle b/bukkit1710/build.gradle index 69efbae7..a7da9e8f 100644 --- a/bukkit1710/build.gradle +++ b/bukkit1710/build.gradle @@ -20,9 +20,12 @@ shadowJar { dependencies { include(dependency(':bukkit0')) include(dependency(':core')) + include(dependency('com.google.code.gson:gson:2.2.4')) } archiveName = "${parent.name}-${project.name}-${parent.version}.jar" destinationDir = file '../target' + + relocate('com.google.gson', 'com.sk89q.worldedit.internal.gson') } shadowJar.doLast { task -> diff --git a/core/src/main/java/com/boydti/fawe/Fawe.java b/core/src/main/java/com/boydti/fawe/Fawe.java index 76855fa7..e0133854 100644 --- a/core/src/main/java/com/boydti/fawe/Fawe.java +++ b/core/src/main/java/com/boydti/fawe/Fawe.java @@ -157,12 +157,7 @@ public class Fawe { INSTANCE = new Fawe(implementation); } - /** - * Write something to the console - * @param s - */ - public static void debug(Object s) { - s = BBC.PREFIX.original() + " " + s; + public static void debugPlain(String s) { if (INSTANCE != null) { INSTANCE.IMP.debug(StringMan.getString(s)); } else { @@ -170,6 +165,14 @@ public class Fawe { } } + /** + * Write something to the console + * @param s + */ + public static void debug(Object s) { + debugPlain(BBC.PREFIX.original() + " " + s); + } + /** * The platform specific implementation */ diff --git a/core/src/main/java/com/boydti/fawe/config/BBC.java b/core/src/main/java/com/boydti/fawe/config/BBC.java index 97470bba..fb55b8d4 100644 --- a/core/src/main/java/com/boydti/fawe/config/BBC.java +++ b/core/src/main/java/com/boydti/fawe/config/BBC.java @@ -367,7 +367,7 @@ public enum BBC { return; } if (actor == null) { - Fawe.debug((PREFIX.isEmpty() ? "" : PREFIX.s() + " ") + this.format(args)); + Fawe.debug(this.format(args)); } else { actor.print((PREFIX.isEmpty() ? "" : PREFIX.s() + " ") + this.format(args)); } @@ -382,7 +382,7 @@ public enum BBC { return; } if (player == null) { - Fawe.debug((PREFIX.isEmpty() ? "" : PREFIX.s() + " ") + this.format(args)); + Fawe.debug(this.format(args)); } else { player.sendMessage((PREFIX.isEmpty() ? "" : PREFIX.s() + " ") + this.format(args)); } diff --git a/core/src/main/java/com/boydti/fawe/jnbt/MCAChunk.java b/core/src/main/java/com/boydti/fawe/jnbt/MCAChunk.java new file mode 100644 index 00000000..315bf723 --- /dev/null +++ b/core/src/main/java/com/boydti/fawe/jnbt/MCAChunk.java @@ -0,0 +1,22 @@ +package com.boydti.fawe.jnbt; + +import com.boydti.fawe.example.CharFaweChunk; +import com.boydti.fawe.object.FaweQueue; + +public class MCAChunk extends CharFaweChunk { + /** + * A FaweSections object represents a chunk and the blocks that you wish to change in it. + * + * @param parent + * @param x + * @param z + */ + public MCAChunk(FaweQueue parent, int x, int z) { + super(parent, x, z); + } + + @Override + public Void getNewChunk() { + return null; + } +} diff --git a/core/src/main/java/com/boydti/fawe/jnbt/MCAFile.java b/core/src/main/java/com/boydti/fawe/jnbt/MCAFile.java index 1600b9c1..eb263880 100644 --- a/core/src/main/java/com/boydti/fawe/jnbt/MCAFile.java +++ b/core/src/main/java/com/boydti/fawe/jnbt/MCAFile.java @@ -1,18 +1,146 @@ package com.boydti.fawe.jnbt; +import com.boydti.fawe.config.Settings; +import com.boydti.fawe.object.RunnableVal3; +import com.boydti.fawe.object.io.BufferedRandomAccessFile; +import com.sk89q.jnbt.NBTInputStream; +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileNotFoundException; +import java.lang.reflect.Field; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; public class MCAFile { private final File file; - byte[] header; + private final BufferedRandomAccessFile raf; + public final byte[] locations; + private Field fieldBuf1; + private Field fieldBuf2; + private Field fieldBuf3; - public MCAFile(File regionFolder, int mcrX, int mcrZ) throws FileNotFoundException { - // TODO load NBT - this.file = new File(regionFolder, "r." + mcrX + "." + mcrZ + ".mca"); + private byte[] buffer1 = new byte[Settings.HISTORY.BUFFER_SIZE]; + private byte[] buffer2 = new byte[Settings.HISTORY.BUFFER_SIZE]; + private byte[] buffer3 = new byte[720]; + + + public MCAFile(File file) throws Exception { + this.file = file; if (!file.exists()) { throw new FileNotFoundException(file.toString()); } - this.header = new byte[4096]; + this.locations = new byte[4096]; + this.raf = new BufferedRandomAccessFile(file, "rw", Settings.HISTORY.BUFFER_SIZE); + raf.read(locations); + fieldBuf1 = BufferedInputStream.class.getDeclaredField("buf"); + fieldBuf1.setAccessible(true); + fieldBuf2 = InflaterInputStream.class.getDeclaredField("buf"); + fieldBuf2.setAccessible(true); + fieldBuf3 = NBTInputStream.class.getDeclaredField("buf"); + fieldBuf3.setAccessible(true); + } + + + public MCAFile(File regionFolder, int mcrX, int mcrZ) throws Exception { + this(new File(regionFolder, "r." + mcrX + "." + mcrZ + ".mca")); + } + + /** + * @param onEach cx, cz, offset + */ + public void forEachChunk(RunnableVal3 onEach) { + int i = 0; + for (int z = 0; z < 32; z++) { + for (int x = 0; x < 32; x++, i += 4) { + int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i+ 2] & 0xFF))); + int size = locations[i + 3] & 0xFF; + if (size != 0) { + onEach.run(x, z, offset << 12); + } + } + } + } + + public int getOffset(int cx, int cz) { + int i = (cx << 2) + (cz << 7); + int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i+ 2] & 0xFF))); + int size = locations[i + 3] & 0xFF; + return offset << 12; + } + + + private NBTStreamer getChunkReader(int offset) throws Exception { + raf.seek(offset); + int size = raf.readInt(); + int compression = raf.readByte(); + byte[] data = new byte[size]; + raf.read(data); + ByteArrayInputStream bais = new ByteArrayInputStream(data); + InflaterInputStream iis = new InflaterInputStream(bais, new Inflater(), 1); + fieldBuf2.set(iis, buffer2); + BufferedInputStream bis = new BufferedInputStream(iis, 1); + fieldBuf1.set(bis, buffer1); + NBTInputStream nis = new NBTInputStream(bis); + fieldBuf3.set(nis, buffer3); + return new NBTStreamer(nis); + } + + public int countId(int offset, final int id) throws Exception { + try { + NBTStreamer streamer = getChunkReader(offset); + NBTStreamer.ByteReader reader = new NBTStreamer.ByteReader() { + public int countId = id; + public int count = 0; + @Override + public void run(int index, int byteValue) { + if (byteValue == countId) { + count++; + } + } + }; + streamer.addReader(".Level.Sections.#.Blocks.#", reader); + streamer.readFully(); + return reader.getClass().getField("count").getInt(reader); + } catch (Exception e) { + e.printStackTrace(); + } + return 0; + } + + public static void main(String[] args) throws Exception { + File folder = new File("../../mc/world/region"); + long start = System.nanoTime(); + final AtomicInteger count = new AtomicInteger(); + final int id = 1; + for (File file : folder.listFiles()) { +// { +// File file = new File(folder, "r.0.0.mca"); + System.out.println(file); + final MCAFile mca = new MCAFile(file); + mca.forEachChunk(new RunnableVal3() { + @Override + public void run(Integer cx, Integer cz, Integer offset) { + try { + count.addAndGet(mca.countId(offset, id)); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + + } + long diff = System.nanoTime() - start; + System.out.println(diff / 1000000d); + + System.out.println("Count: " + count); + + // My results + // 496,772,342 stone + // 35,164 chunks + // 17.175 seconds + + } } diff --git a/core/src/main/java/com/boydti/fawe/jnbt/MCAQueue.java b/core/src/main/java/com/boydti/fawe/jnbt/MCAQueue.java new file mode 100644 index 00000000..26349997 --- /dev/null +++ b/core/src/main/java/com/boydti/fawe/jnbt/MCAQueue.java @@ -0,0 +1,138 @@ +package com.boydti.fawe.jnbt; + +import com.boydti.fawe.example.CharFaweChunk; +import com.boydti.fawe.example.NMSMappedFaweQueue; +import com.boydti.fawe.object.FaweChunk; +import com.boydti.fawe.object.FaweQueue; +import com.boydti.fawe.object.RunnableVal; +import com.sk89q.jnbt.CompoundTag; +import java.io.File; +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class MCAQueue extends NMSMappedFaweQueue { + + private final FaweQueue parent; + + public MCAQueue(FaweQueue parent) { + super(parent.getWorldName()); + this.parent = parent; + } + + @Override + public void setFullbright(MCAChunk sections) { + + } + + @Override + public boolean removeLighting(MCAChunk sections, RelightMode mode, boolean hasSky) { + return false; + } + + @Override + public void relight(int x, int y, int z) { + + } + + @Override + public void relightBlock(int x, int y, int z) { + + } + + @Override + public void relightSky(int x, int y, int z) { + + } + + @Override + public void setSkyLight(char[] chars, int x, int y, int z, int value) { + + } + + @Override + public void setBlockLight(char[] chars, int x, int y, int z, int value) { + + } + + @Override + public void refreshChunk(FaweChunk fs) { + + } + + @Override + public CharFaweChunk getPrevious(CharFaweChunk fs, MCAChunk sections, Map tiles, Collection[] entities, Set createdEntities, boolean all) throws Exception { + return null; + } + + @Override + public CompoundTag getTileEntity(MCAChunk mcaChunk, int x, int y, int z) { + return null; + } + + @Override + public MCAChunk getChunk(FaweQueue faweQueue, int x, int z) { + return null; + } + + @Override + public FaweQueue getImpWorld() { + return null; + } + + @Override + public boolean isChunkLoaded(FaweQueue faweQueue, int x, int z) { + return false; + } + + @Override + public boolean regenerateChunk(FaweQueue faweQueue, int x, int z) { + return false; + } + + @Override + public boolean setComponents(FaweChunk fc, RunnableVal changeTask) { + return false; + } + + @Override + public FaweChunk getFaweChunk(int x, int z) { + return null; + } + + @Override + public File getSaveFolder() { + return null; + } + + @Override + public boolean hasSky() { + return false; + } + + @Override + public boolean loadChunk(FaweQueue faweQueue, int x, int z, boolean generate) { + return false; + } + + @Override + public MCAChunk getCachedSections(FaweQueue faweQueue, int cx, int cz) { + return null; + } + + @Override + public int getCombinedId4Data(char[] chars, int x, int y, int z) { + return 0; + } + + @Override + public int getSkyLight(char[] sections, int x, int y, int z) { + return 0; + } + + @Override + public int getEmmittedLight(char[] sections, int x, int y, int z) { + return 0; + } +} diff --git a/core/src/main/java/com/boydti/fawe/object/clipboard/DiskOptimizedClipboard.java b/core/src/main/java/com/boydti/fawe/object/clipboard/DiskOptimizedClipboard.java index 403a9ca0..bc8eaa2b 100644 --- a/core/src/main/java/com/boydti/fawe/object/clipboard/DiskOptimizedClipboard.java +++ b/core/src/main/java/com/boydti/fawe/object/clipboard/DiskOptimizedClipboard.java @@ -3,7 +3,7 @@ package com.boydti.fawe.object.clipboard; import com.boydti.fawe.Fawe; import com.boydti.fawe.FaweCache; import com.boydti.fawe.config.Settings; -import com.boydti.fawe.object.BufferedRandomAccessFile; +import com.boydti.fawe.object.io.BufferedRandomAccessFile; import com.boydti.fawe.object.IntegerTrio; import com.boydti.fawe.object.RunnableVal2; import com.boydti.fawe.util.MainUtil; diff --git a/core/src/main/java/com/boydti/fawe/object/BufferedRandomAccessFile.java b/core/src/main/java/com/boydti/fawe/object/io/BufferedRandomAccessFile.java similarity index 99% rename from core/src/main/java/com/boydti/fawe/object/BufferedRandomAccessFile.java rename to core/src/main/java/com/boydti/fawe/object/io/BufferedRandomAccessFile.java index b5f290db..55c83140 100644 --- a/core/src/main/java/com/boydti/fawe/object/BufferedRandomAccessFile.java +++ b/core/src/main/java/com/boydti/fawe/object/io/BufferedRandomAccessFile.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.boydti.fawe.object; +package com.boydti.fawe.object.io; import java.io.*; import java.util.Arrays; @@ -35,7 +35,7 @@ import java.util.Arrays; * Author : Avinash Lakshman ( alakshman@facebook.com) & Prashant Malik ( pmalik@facebook.com ) */ -public final class BufferedRandomAccessFile extends RandomAccessFile +public class BufferedRandomAccessFile extends RandomAccessFile { static final int LogBuffSz_ = 16; // 64K buffer public static final int BuffSz_ = (1 << LogBuffSz_); diff --git a/core/src/main/java/com/boydti/fawe/object/io/RandomAccessInputStream.java b/core/src/main/java/com/boydti/fawe/object/io/RandomAccessInputStream.java new file mode 100644 index 00000000..9ecaec8c --- /dev/null +++ b/core/src/main/java/com/boydti/fawe/object/io/RandomAccessInputStream.java @@ -0,0 +1,28 @@ +package com.boydti.fawe.object.io; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; + +public class RandomAccessInputStream extends InputStream { + private final RandomAccessFile raf; + + public RandomAccessInputStream(RandomAccessFile raf) { + this.raf = raf; + } + + @Override + public int read() throws IOException { + return raf.read(); + } + + @Override + public int available() throws IOException { + return (int) (raf.length() - raf.getFilePointer()); + } + + @Override + public void close() throws IOException { + raf.close(); + } +} diff --git a/core/src/main/java/com/boydti/fawe/wrappers/FakePlayer.java b/core/src/main/java/com/boydti/fawe/wrappers/FakePlayer.java index f2858932..3817da9d 100644 --- a/core/src/main/java/com/boydti/fawe/wrappers/FakePlayer.java +++ b/core/src/main/java/com/boydti/fawe/wrappers/FakePlayer.java @@ -178,7 +178,7 @@ public class FakePlayer extends LocalPlayer { parent.printRaw(msg); return; } - Fawe.debug(msg); + Fawe.get().debugPlain(msg); } @Override @@ -187,7 +187,7 @@ public class FakePlayer extends LocalPlayer { parent.printDebug(msg); return; } - Fawe.debug(msg); + Fawe.get().debugPlain(msg); } @Override @@ -196,7 +196,7 @@ public class FakePlayer extends LocalPlayer { parent.print(msg); return; } - Fawe.debug(msg); + Fawe.get().debugPlain(msg); } @Override @@ -205,7 +205,7 @@ public class FakePlayer extends LocalPlayer { parent.printError(msg); return; } - Fawe.debug(msg); + Fawe.get().debugPlain(msg); } private FakeSessionKey key; diff --git a/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java b/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java index 6f90bd20..9b028c73 100644 --- a/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java +++ b/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java @@ -22,6 +22,7 @@ package com.sk89q.jnbt; import com.boydti.fawe.jnbt.NBTStreamer; import com.boydti.fawe.object.RunnableVal2; import java.io.Closeable; +import java.io.DataInput; import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; @@ -41,7 +42,7 @@ import java.util.Map; */ public final class NBTInputStream implements Closeable { - private final DataInputStream is; + private final DataInput is; /** * Creates a new {@code NBTInputStream}, which will source its data @@ -54,6 +55,10 @@ public final class NBTInputStream implements Closeable { this.is = new DataInputStream(is); } + public NBTInputStream(DataInput di) { + this.is = di; + } + /** * Reads an NBT tag from the stream. * @@ -99,6 +104,8 @@ public final class NBTInputStream implements Closeable { } } + private byte[] buf; + private void readTagPaylodLazy(int type, int depth, String node, RunnableVal2 getReader) throws IOException { switch (type) { case NBTConstants.TYPE_END: @@ -138,8 +145,40 @@ public final class NBTInputStream implements Closeable { } if (reader instanceof NBTStreamer.ByteReader) { NBTStreamer.ByteReader byteReader = (NBTStreamer.ByteReader) reader; - for (int i = 0; i < length; i++) { - byteReader.run(i, is.read()); + int i = 0; + if (is instanceof InputStream) { + DataInputStream dis = (DataInputStream) is; + if (length > 720) { + if (buf == null) { + buf = new byte[720]; + } + int left = length; + for (; left > 720; left -= 720) { + dis.read(buf); + for (byte b : buf) { + byteReader.run(i++, b & 0xFF); + } + } + } + for (; i < length; i++) { + byteReader.run(i, dis.read()); + } + } else { + if (length > 720) { + if (buf == null) { + buf = new byte[720]; + } + int left = length; + for (; left > 720; left -= 720) { + is.readFully(buf); + for (byte b : buf) { + byteReader.run(i++, b & 0xFF); + } + } + } + for (; i < length; i++) { + byteReader.run(i, is.readByte() & 0xFF); + } } } else { for (int i = 0; i < length; i++) { @@ -349,7 +388,13 @@ public final class NBTInputStream implements Closeable { @Override public void close() throws IOException { - is.close(); + if (is instanceof AutoCloseable) { + try { + ((AutoCloseable) is).close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } }