More work on optimized MCA reader

+ Fix for 1.7.10 gson import error

bukkit1710/build.gradle

@@ -20,9 +20,12 @@ shadowJar {
     dependencies {
         include(dependency(':bukkit0'))
         include(dependency(':core'))
+        include(dependency('com.google.code.gson:gson:2.2.4'))
     }
     archiveName = "${parent.name}-${project.name}-${parent.version}.jar"
     destinationDir = file '../target'
+
+    relocate('com.google.gson', 'com.sk89q.worldedit.internal.gson')
 }
 shadowJar.doLast {
     task ->

core/src/main/java/com/boydti/fawe/Fawe.java

@@ -157,12 +157,7 @@ public class Fawe {
         INSTANCE = new Fawe(implementation);
     }
 
-    /**
-     * Write something to the console
-     * @param s
-     */
-    public static void debug(Object s) {
-        s = BBC.PREFIX.original() + " " + s;
+    public static void debugPlain(String s) {
         if (INSTANCE != null) {
             INSTANCE.IMP.debug(StringMan.getString(s));
         } else {
@@ -170,6 +165,14 @@ public class Fawe {
         }
     }
 
+    /**
+     * Write something to the console
+     * @param s
+     */
+    public static void debug(Object s) {
+        debugPlain(BBC.PREFIX.original() + " " + s);
+    }
+
     /**
      * The platform specific implementation
      */

core/src/main/java/com/boydti/fawe/config/BBC.java

@@ -367,7 +367,7 @@ public enum BBC {
             return;
         }
         if (actor == null) {
-            Fawe.debug((PREFIX.isEmpty() ? "" : PREFIX.s() + " ") + this.format(args));
+            Fawe.debug(this.format(args));
         } else {
             actor.print((PREFIX.isEmpty() ? "" : PREFIX.s() + " ") + this.format(args));
         }
@@ -382,7 +382,7 @@ public enum BBC {
             return;
         }
         if (player == null) {
-            Fawe.debug((PREFIX.isEmpty() ? "" : PREFIX.s() + " ") + this.format(args));
+            Fawe.debug(this.format(args));
         } else {
             player.sendMessage((PREFIX.isEmpty() ? "" : PREFIX.s() + " ") + this.format(args));
         }

core/src/main/java/com/boydti/fawe/jnbt/MCAChunk.java

@@ -0,0 +1,22 @@
+package com.boydti.fawe.jnbt;
+
+import com.boydti.fawe.example.CharFaweChunk;
+import com.boydti.fawe.object.FaweQueue;
+
+public class MCAChunk extends CharFaweChunk<Void> {
+    /**
+     * A FaweSections object represents a chunk and the blocks that you wish to change in it.
+     *
+     * @param parent
+     * @param x
+     * @param z
+     */
+    public MCAChunk(FaweQueue parent, int x, int z) {
+        super(parent, x, z);
+    }
+
+    @Override
+    public Void getNewChunk() {
+        return null;
+    }
+}

core/src/main/java/com/boydti/fawe/jnbt/MCAFile.java

@@ -1,18 +1,146 @@
 package com.boydti.fawe.jnbt;
 
+import com.boydti.fawe.config.Settings;
+import com.boydti.fawe.object.RunnableVal3;
+import com.boydti.fawe.object.io.BufferedRandomAccessFile;
+import com.sk89q.jnbt.NBTInputStream;
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileNotFoundException;
+import java.lang.reflect.Field;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
 
 public class MCAFile {
     private final File file;
-    byte[] header;
+    private final BufferedRandomAccessFile raf;
+    public final byte[] locations;
+    private Field fieldBuf1;
+    private Field fieldBuf2;
+    private Field fieldBuf3;
 
-    public MCAFile(File regionFolder, int mcrX, int mcrZ) throws FileNotFoundException {
-        // TODO load NBT
-        this.file = new File(regionFolder, "r." + mcrX + "." + mcrZ + ".mca");
+    private byte[] buffer1 = new byte[Settings.HISTORY.BUFFER_SIZE];
+    private byte[] buffer2 = new byte[Settings.HISTORY.BUFFER_SIZE];
+    private byte[] buffer3 = new byte[720];
+
+
+    public MCAFile(File file) throws Exception {
+        this.file = file;
         if (!file.exists()) {
             throw new FileNotFoundException(file.toString());
         }
-        this.header = new byte[4096];
+        this.locations = new byte[4096];
+        this.raf = new BufferedRandomAccessFile(file, "rw", Settings.HISTORY.BUFFER_SIZE);
+        raf.read(locations);
+        fieldBuf1 = BufferedInputStream.class.getDeclaredField("buf");
+        fieldBuf1.setAccessible(true);
+        fieldBuf2 = InflaterInputStream.class.getDeclaredField("buf");
+        fieldBuf2.setAccessible(true);
+        fieldBuf3 = NBTInputStream.class.getDeclaredField("buf");
+        fieldBuf3.setAccessible(true);
+    }
+
+
+    public MCAFile(File regionFolder, int mcrX, int mcrZ) throws Exception {
+        this(new File(regionFolder, "r." + mcrX + "." + mcrZ + ".mca"));
+    }
+
+    /**
+     * @param onEach cx, cz, offset
+     */
+    public void forEachChunk(RunnableVal3<Integer, Integer, Integer> onEach) {
+        int i = 0;
+        for (int z = 0; z < 32; z++) {
+            for (int x = 0; x < 32; x++, i += 4) {
+                int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i+ 2] & 0xFF)));
+                int size = locations[i + 3] & 0xFF;
+                if (size != 0) {
+                    onEach.run(x, z, offset << 12);
+                }
+            }
+        }
+    }
+
+    public int getOffset(int cx, int cz) {
+        int i = (cx << 2) + (cz << 7);
+        int offset = (((locations[i] & 0xFF) << 16) + ((locations[i + 1] & 0xFF) << 8) + ((locations[i+ 2] & 0xFF)));
+        int size = locations[i + 3] & 0xFF;
+        return offset << 12;
+    }
+
+
+    private NBTStreamer getChunkReader(int offset) throws Exception {
+        raf.seek(offset);
+        int size = raf.readInt();
+        int compression = raf.readByte();
+        byte[] data = new byte[size];
+        raf.read(data);
+        ByteArrayInputStream bais = new ByteArrayInputStream(data);
+        InflaterInputStream iis = new InflaterInputStream(bais, new Inflater(), 1);
+        fieldBuf2.set(iis, buffer2);
+        BufferedInputStream bis = new BufferedInputStream(iis, 1);
+        fieldBuf1.set(bis, buffer1);
+        NBTInputStream nis = new NBTInputStream(bis);
+        fieldBuf3.set(nis, buffer3);
+        return new NBTStreamer(nis);
+    }
+
+    public int countId(int offset, final int id) throws Exception {
+        try {
+            NBTStreamer streamer = getChunkReader(offset);
+            NBTStreamer.ByteReader reader = new NBTStreamer.ByteReader() {
+                public int countId = id;
+                public int count = 0;
+                @Override
+                public void run(int index, int byteValue) {
+                    if (byteValue == countId) {
+                        count++;
+                    }
+                }
+            };
+            streamer.addReader(".Level.Sections.#.Blocks.#", reader);
+            streamer.readFully();
+            return reader.getClass().getField("count").getInt(reader);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return 0;
+    }
+
+    public static void main(String[] args) throws Exception {
+        File folder = new File("../../mc/world/region");
+        long start = System.nanoTime();
+        final AtomicInteger count = new AtomicInteger();
+        final int id = 1;
+        for (File file : folder.listFiles()) {
+//        {
+//            File file = new File(folder, "r.0.0.mca");
+            System.out.println(file);
+            final MCAFile mca = new MCAFile(file);
+            mca.forEachChunk(new RunnableVal3<Integer, Integer, Integer>() {
+                @Override
+                public void run(Integer cx, Integer cz, Integer offset) {
+                    try {
+                        count.addAndGet(mca.countId(offset, id));
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }
+            });
+
+        }
+        long diff = System.nanoTime() - start;
+        System.out.println(diff / 1000000d);
+
+        System.out.println("Count: " + count);
+
+        // My results
+        // 496,772,342 stone
+        // 35,164 chunks
+        // 17.175 seconds
+
+
     }
 }

core/src/main/java/com/boydti/fawe/jnbt/MCAQueue.java

@@ -0,0 +1,138 @@
+package com.boydti.fawe.jnbt;
+
+import com.boydti.fawe.example.CharFaweChunk;
+import com.boydti.fawe.example.NMSMappedFaweQueue;
+import com.boydti.fawe.object.FaweChunk;
+import com.boydti.fawe.object.FaweQueue;
+import com.boydti.fawe.object.RunnableVal;
+import com.sk89q.jnbt.CompoundTag;
+import java.io.File;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+public class MCAQueue extends NMSMappedFaweQueue<FaweQueue, MCAChunk, MCAChunk, char[]> {
+
+    private final FaweQueue parent;
+
+    public MCAQueue(FaweQueue parent) {
+        super(parent.getWorldName());
+        this.parent = parent;
+    }
+
+    @Override
+    public void setFullbright(MCAChunk sections) {
+
+    }
+
+    @Override
+    public boolean removeLighting(MCAChunk sections, RelightMode mode, boolean hasSky) {
+        return false;
+    }
+
+    @Override
+    public void relight(int x, int y, int z) {
+
+    }
+
+    @Override
+    public void relightBlock(int x, int y, int z) {
+
+    }
+
+    @Override
+    public void relightSky(int x, int y, int z) {
+
+    }
+
+    @Override
+    public void setSkyLight(char[] chars, int x, int y, int z, int value) {
+
+    }
+
+    @Override
+    public void setBlockLight(char[] chars, int x, int y, int z, int value) {
+
+    }
+
+    @Override
+    public void refreshChunk(FaweChunk fs) {
+
+    }
+
+    @Override
+    public CharFaweChunk getPrevious(CharFaweChunk fs, MCAChunk sections, Map<?, ?> tiles, Collection<?>[] entities, Set<UUID> createdEntities, boolean all) throws Exception {
+        return null;
+    }
+
+    @Override
+    public CompoundTag getTileEntity(MCAChunk mcaChunk, int x, int y, int z) {
+        return null;
+    }
+
+    @Override
+    public MCAChunk getChunk(FaweQueue faweQueue, int x, int z) {
+        return null;
+    }
+
+    @Override
+    public FaweQueue getImpWorld() {
+        return null;
+    }
+
+    @Override
+    public boolean isChunkLoaded(FaweQueue faweQueue, int x, int z) {
+        return false;
+    }
+
+    @Override
+    public boolean regenerateChunk(FaweQueue faweQueue, int x, int z) {
+        return false;
+    }
+
+    @Override
+    public boolean setComponents(FaweChunk fc, RunnableVal<FaweChunk> changeTask) {
+        return false;
+    }
+
+    @Override
+    public FaweChunk getFaweChunk(int x, int z) {
+        return null;
+    }
+
+    @Override
+    public File getSaveFolder() {
+        return null;
+    }
+
+    @Override
+    public boolean hasSky() {
+        return false;
+    }
+
+    @Override
+    public boolean loadChunk(FaweQueue faweQueue, int x, int z, boolean generate) {
+        return false;
+    }
+
+    @Override
+    public MCAChunk getCachedSections(FaweQueue faweQueue, int cx, int cz) {
+        return null;
+    }
+
+    @Override
+    public int getCombinedId4Data(char[] chars, int x, int y, int z) {
+        return 0;
+    }
+
+    @Override
+    public int getSkyLight(char[] sections, int x, int y, int z) {
+        return 0;
+    }
+
+    @Override
+    public int getEmmittedLight(char[] sections, int x, int y, int z) {
+        return 0;
+    }
+}

core/src/main/java/com/boydti/fawe/object/clipboard/DiskOptimizedClipboard.java

@@ -3,7 +3,7 @@ package com.boydti.fawe.object.clipboard;
 import com.boydti.fawe.Fawe;
 import com.boydti.fawe.FaweCache;
 import com.boydti.fawe.config.Settings;
-import com.boydti.fawe.object.BufferedRandomAccessFile;
+import com.boydti.fawe.object.io.BufferedRandomAccessFile;
 import com.boydti.fawe.object.IntegerTrio;
 import com.boydti.fawe.object.RunnableVal2;
 import com.boydti.fawe.util.MainUtil;

core/src/main/java/com/boydti/fawe/object/io/BufferedRandomAccessFile.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package com.boydti.fawe.object;
+package com.boydti.fawe.object.io;
 
 import java.io.*;
 import java.util.Arrays;
@@ -35,7 +35,7 @@ import java.util.Arrays;
  * Author : Avinash Lakshman ( alakshman@facebook.com) & Prashant Malik ( pmalik@facebook.com )
  */
 
-public final class BufferedRandomAccessFile extends RandomAccessFile
+public class BufferedRandomAccessFile extends RandomAccessFile
 {
     static final int LogBuffSz_ = 16; // 64K buffer
     public static final int BuffSz_ = (1 << LogBuffSz_);

core/src/main/java/com/boydti/fawe/object/io/RandomAccessInputStream.java

@@ -0,0 +1,28 @@
+package com.boydti.fawe.object.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.RandomAccessFile;
+
+public class RandomAccessInputStream extends InputStream {
+    private final RandomAccessFile raf;
+
+    public RandomAccessInputStream(RandomAccessFile raf) {
+        this.raf = raf;
+    }
+
+    @Override
+    public int read() throws IOException {
+        return raf.read();
+    }
+
+    @Override
+    public int available() throws IOException {
+        return (int) (raf.length() - raf.getFilePointer());
+    }
+
+    @Override
+    public void close() throws IOException {
+        raf.close();
+    }
+}

core/src/main/java/com/boydti/fawe/wrappers/FakePlayer.java

@@ -178,7 +178,7 @@ public class FakePlayer extends LocalPlayer {
             parent.printRaw(msg);
             return;
         }
-        Fawe.debug(msg);
+        Fawe.get().debugPlain(msg);
     }
 
     @Override
@@ -187,7 +187,7 @@ public class FakePlayer extends LocalPlayer {
             parent.printDebug(msg);
             return;
         }
-        Fawe.debug(msg);
+        Fawe.get().debugPlain(msg);
     }
 
     @Override
@@ -196,7 +196,7 @@ public class FakePlayer extends LocalPlayer {
             parent.print(msg);
             return;
         }
-        Fawe.debug(msg);
+        Fawe.get().debugPlain(msg);
     }
 
     @Override
@@ -205,7 +205,7 @@ public class FakePlayer extends LocalPlayer {
             parent.printError(msg);
             return;
         }
-        Fawe.debug(msg);
+        Fawe.get().debugPlain(msg);
     }
 
     private FakeSessionKey key;

core/src/main/java/com/sk89q/jnbt/NBTInputStream.java

@@ -22,6 +22,7 @@ package com.sk89q.jnbt;
 import com.boydti.fawe.jnbt.NBTStreamer;
 import com.boydti.fawe.object.RunnableVal2;
 import java.io.Closeable;
+import java.io.DataInput;
 import java.io.DataInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -41,7 +42,7 @@ import java.util.Map;
  */
 public final class NBTInputStream implements Closeable {
 
-    private final DataInputStream is;
+    private final DataInput is;
 
     /**
      * Creates a new {@code NBTInputStream}, which will source its data
@@ -54,6 +55,10 @@ public final class NBTInputStream implements Closeable {
         this.is = new DataInputStream(is);
     }
 
+    public NBTInputStream(DataInput di) {
+        this.is = di;
+    }
+
     /**
      * Reads an NBT tag from the stream.
      *
@@ -99,6 +104,8 @@ public final class NBTInputStream implements Closeable {
         }
     }
 
+    private byte[] buf;
+
     private void readTagPaylodLazy(int type, int depth, String node, RunnableVal2<String, RunnableVal2> getReader) throws IOException {
         switch (type) {
             case NBTConstants.TYPE_END:
@@ -138,8 +145,40 @@ public final class NBTInputStream implements Closeable {
                 }
                 if (reader instanceof NBTStreamer.ByteReader) {
                     NBTStreamer.ByteReader byteReader = (NBTStreamer.ByteReader) reader;
-                    for (int i = 0; i < length; i++) {
-                        byteReader.run(i, is.read());
+                    int i = 0;
+                    if (is instanceof InputStream) {
+                        DataInputStream dis = (DataInputStream) is;
+                        if (length > 720) {
+                            if (buf == null) {
+                                buf = new byte[720];
+                            }
+                            int left = length;
+                            for (; left > 720; left -= 720) {
+                                dis.read(buf);
+                                for (byte b : buf) {
+                                    byteReader.run(i++, b & 0xFF);
+                                }
+                            }
+                        }
+                        for (; i < length; i++) {
+                            byteReader.run(i, dis.read());
+                        }
+                    } else {
+                        if (length > 720) {
+                            if (buf == null) {
+                                buf = new byte[720];
+                            }
+                            int left = length;
+                            for (; left > 720; left -= 720) {
+                                is.readFully(buf);
+                                for (byte b : buf) {
+                                    byteReader.run(i++, b & 0xFF);
+                                }
+                            }
+                        }
+                        for (; i < length; i++) {
+                            byteReader.run(i, is.readByte() & 0xFF);
+                        }
                     }
                 } else {
                     for (int i = 0; i < length; i++) {
@@ -349,7 +388,13 @@ public final class NBTInputStream implements Closeable {
 
     @Override
     public void close() throws IOException {
-        is.close();
+        if (is instanceof AutoCloseable) {
+            try {
+                ((AutoCloseable) is).close();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
     }
 
 }