From 091d1ba4f4d60b8952a7b78a34b63864e29c91df Mon Sep 17 00:00:00 2001 From: Jesse Boyd Date: Wed, 23 Nov 2016 13:25:11 +1100 Subject: [PATCH] Recover incomplete or corrupt schematic files. --- core/src/main/java/com/boydti/fawe/Fawe.java | 2 +- .../fawe/jnbt/CorruptSchematicStreamer.java | 282 ++++++++++++++++++ .../object/io/ResettableFileInputStream.java | 39 +++ .../java/com/sk89q/jnbt/NBTInputStream.java | 35 ++- .../extent/clipboard/io/ClipboardFormat.java | 12 +- .../extent/clipboard/io/SchematicReader.java | 20 +- 6 files changed, 379 insertions(+), 11 deletions(-) create mode 100644 core/src/main/java/com/boydti/fawe/jnbt/CorruptSchematicStreamer.java create mode 100644 core/src/main/java/com/boydti/fawe/object/io/ResettableFileInputStream.java diff --git a/core/src/main/java/com/boydti/fawe/Fawe.java b/core/src/main/java/com/boydti/fawe/Fawe.java index 4b0c0891..1358b127 100644 --- a/core/src/main/java/com/boydti/fawe/Fawe.java +++ b/core/src/main/java/com/boydti/fawe/Fawe.java @@ -187,7 +187,7 @@ public class Fawe { if (INSTANCE != null) { INSTANCE.IMP.debug(StringMan.getString(s)); } else { - System.out.print(s); + System.out.println(s); } } diff --git a/core/src/main/java/com/boydti/fawe/jnbt/CorruptSchematicStreamer.java b/core/src/main/java/com/boydti/fawe/jnbt/CorruptSchematicStreamer.java new file mode 100644 index 00000000..85fdd7df --- /dev/null +++ b/core/src/main/java/com/boydti/fawe/jnbt/CorruptSchematicStreamer.java @@ -0,0 +1,282 @@ +package com.boydti.fawe.jnbt; + +import com.boydti.fawe.Fawe; +import com.boydti.fawe.config.Settings; +import com.boydti.fawe.object.clipboard.CPUOptimizedClipboard; +import com.boydti.fawe.object.clipboard.DiskOptimizedClipboard; +import com.boydti.fawe.object.clipboard.FaweClipboard; +import com.boydti.fawe.object.clipboard.MemoryOptimizedClipboard; +import com.sk89q.jnbt.CompoundTag; +import com.sk89q.jnbt.ListTag; +import com.sk89q.jnbt.NBTInputStream; +import com.sk89q.worldedit.Vector; +import com.sk89q.worldedit.entity.BaseEntity; +import com.sk89q.worldedit.extent.clipboard.BlockArrayClipboard; +import com.sk89q.worldedit.extent.clipboard.Clipboard; +import com.sk89q.worldedit.regions.CuboidRegion; +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.GZIPInputStream; + +public class CorruptSchematicStreamer { + + private final InputStream stream; + private final UUID uuid; + private FaweClipboard fc; + final AtomicInteger volume = new AtomicInteger(); + final AtomicInteger width = new AtomicInteger(); + final AtomicInteger height = new AtomicInteger(); + final AtomicInteger length = new AtomicInteger(); + final AtomicInteger offsetX = new AtomicInteger(); + final AtomicInteger offsetY = new AtomicInteger(); + final AtomicInteger offsetZ = new AtomicInteger(); + final AtomicInteger originX = new AtomicInteger(); + final AtomicInteger originY = new AtomicInteger(); + final AtomicInteger originZ = new AtomicInteger(); + + public CorruptSchematicStreamer(InputStream rootStream, UUID uuid) { + this.stream = rootStream; + this.uuid = uuid; + } + + public void match(String matchTag, CorruptReader reader) { + try { + stream.reset(); + stream.mark(Integer.MAX_VALUE); + DataInputStream dataInput = new DataInputStream(new BufferedInputStream(new GZIPInputStream(stream))); + byte[] match = matchTag.getBytes(); + int[] matchValue = new int[match.length]; + int matchIndex = 0; + int read; + while ((read = dataInput.read()) != -1) { + int expected = match[matchIndex]; + if (expected == -1) { + if (++matchIndex == match.length) { + break; + } + } else if (read == expected){ + if (++matchIndex == match.length) { + reader.run(dataInput); + break; + } + } else { + if (matchIndex == 2) + matchIndex = 0; + } + } + Fawe.debug(" - Recover " + matchTag + " = success"); + } catch (Throwable e) { + Fawe.debug(" - Recover " + matchTag + " = partial failure"); + e.printStackTrace(); + } + } + + public FaweClipboard setupClipboard() { + if (fc != null) { + return fc; + } + Vector dimensions = guessDimensions(volume.get(), width.get(), height.get(), length.get()); + if (width.get() == 0 || height.get() == 0 || length.get() == 0) { + Fawe.debug("No dimensions found! Estimating based on factors:" + dimensions); + } + if (Settings.CLIPBOARD.USE_DISK) { + fc = new DiskOptimizedClipboard(dimensions.getBlockX(), dimensions.getBlockY(), dimensions.getBlockZ(), uuid); + } else if (Settings.CLIPBOARD.COMPRESSION_LEVEL == 0) { + fc = new CPUOptimizedClipboard(dimensions.getBlockX(), dimensions.getBlockY(), dimensions.getBlockZ()); + } else { + fc = new MemoryOptimizedClipboard(dimensions.getBlockX(), dimensions.getBlockY(), dimensions.getBlockZ()); + } + return fc; + } + + public Clipboard recover() { + if (stream == null || !stream.markSupported()) { + throw new IllegalArgumentException("Can only recover from a marked and resettable stream!"); + } + match("Width", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + width.set(in.readShort()); + } + }); + match("Height", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + height.set(in.readShort()); + } + }); + match("Length", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + length.set(in.readShort()); + } + }); + match("WEOffsetX", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + offsetX.set(in.readInt()); + } + }); + match("WEOffsetY", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + offsetY.set(in.readInt()); + } + }); + match("WEOffsetZ", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + offsetZ.set(in.readInt()); + } + }); + match("WEOriginX", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + originX.set(in.readInt()); + } + }); + match("WEOriginY", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + originY.set(in.readInt()); + } + }); + match("WEOriginZ", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + originZ.set(in.readInt()); + } + }); + match("Blocks", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + int length = in.readInt(); + volume.set(length); + setupClipboard(); + for (int i = 0; i < length; i++) { + fc.setId(i, in.read()); + } + } + }); + match("Data", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + int length = in.readInt(); + volume.set(length); + setupClipboard(); + for (int i = 0; i < length; i++) { + fc.setData(i, in.read()); + } + } + }); + match("Add", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + int length = in.readInt(); + volume.set(length); + setupClipboard(); + for (int i = 0; i < length; i++) { + fc.setAdd(i, in.read()); + } + } + }); + Vector dimensions = guessDimensions(volume.get(), width.get(), height.get(), length.get()); + Vector min = new Vector(originX.get(), originY.get(), originZ.get()); + Vector offset = new Vector(offsetX.get(), offsetY.get(), offsetZ.get()); + Vector origin = min.subtract(offset); + CuboidRegion region = new CuboidRegion(min, min.add(dimensions.getBlockX(), dimensions.getBlockY(), dimensions.getBlockZ()).subtract(Vector.ONE)); + fc.setOrigin(offset); + final BlockArrayClipboard clipboard = new BlockArrayClipboard(region, fc); + match("TileEntities", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + int childType = in.readByte(); + int length = in.readInt(); + NBTInputStream nis = new NBTInputStream(in); + for (int i = 0; i < length; ++i) { + CompoundTag tag = (CompoundTag) nis.readTagPayload(childType, 1); + int x = tag.getInt("x"); + int y = tag.getInt("y"); + int z = tag.getInt("z"); + fc.setTile(x, y, z, tag); + } + } + }); + match("Entities", new CorruptSchematicStreamer.CorruptReader() { + @Override + public void run(DataInputStream in) throws IOException { + int childType = in.readByte(); + int length = in.readInt(); + NBTInputStream nis = new NBTInputStream(in); + for (int i = 0; i < length; ++i) { + CompoundTag tag = (CompoundTag) nis.readTagPayload(childType, 1); + int x = tag.getInt("x"); + int y = tag.getInt("y"); + int z = tag.getInt("z"); + String id = tag.getString("id"); + if (id.isEmpty()) { + return; + } + ListTag positionTag = tag.getListTag("Pos"); + ListTag directionTag = tag.getListTag("Rotation"); + BaseEntity state = new BaseEntity(id, tag); + fc.createEntity(clipboard, positionTag.asDouble(0), positionTag.asDouble(1), positionTag.asDouble(2), (float) directionTag.asDouble(0), (float) directionTag.asDouble(1), state); + } + } + }); + return clipboard; + } + + private Vector guessDimensions(int volume, int width, int height, int length) { + if (volume == 0) { + return new Vector(width, height, length); + } + if (volume == width * height * length) { + return new Vector(width, height, length); + } + if (width == 0 && height != 0 && length != 0 && volume % (height * length) == 0 && height * length <= volume) { + return new Vector(volume / (height * length), height, length); + } + if (height == 0 && width != 0 && length != 0 && volume % (width * length) == 0 && width * length <= volume) { + return new Vector(width, volume / (width * length), length); + } + if (length == 0 && height != 0 && width != 0 && volume % (height * width) == 0 && height * width <= volume) { + return new Vector(width, height, volume / (width * height)); + } + List factors = new ArrayList<>(); + for (int i = (int) Math.sqrt(volume); i > 0; i--) { + if (volume % i == 0) { + factors.add(i); + factors.add(volume/i); + } + } + int min = Integer.MAX_VALUE; + Vector dimensions = new Vector(); + for (int x = 0; x < factors.size(); x++) { + int xValue = factors.get(x); + for (int y = 0; y < factors.size(); y++) { + int yValue = factors.get(y); + long area = xValue * yValue; + if (volume % area == 0) { + int z = (int) (volume / area); + int max = Math.max(Math.max(xValue, yValue), z); + if (max < min) { + min = max; + dimensions = new Vector(xValue, z, yValue); + } + } + } + } + return dimensions; + } + + public interface CorruptReader { + void run(DataInputStream in) throws IOException; + } +} diff --git a/core/src/main/java/com/boydti/fawe/object/io/ResettableFileInputStream.java b/core/src/main/java/com/boydti/fawe/object/io/ResettableFileInputStream.java new file mode 100644 index 00000000..2772cf5a --- /dev/null +++ b/core/src/main/java/com/boydti/fawe/object/io/ResettableFileInputStream.java @@ -0,0 +1,39 @@ +package com.boydti.fawe.object.io; + +import java.io.FileInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.nio.channels.FileChannel; + +public class ResettableFileInputStream extends FilterInputStream { + private FileChannel myFileChannel; + private long mark = 0; + + public ResettableFileInputStream(FileInputStream fis) { + super(fis); + myFileChannel = fis.getChannel(); + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public synchronized void mark(int readlimit) { + try { + mark = myFileChannel.position(); + } catch (IOException ex) { + ex.printStackTrace(); + mark = -1; + } + } + + @Override + public synchronized void reset() throws IOException { + if (mark == -1) { + throw new IOException("not marked"); + } + myFileChannel.position(mark); + } +} \ No newline at end of file diff --git a/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java b/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java index 1156a634..cb58caf2 100644 --- a/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java +++ b/core/src/main/java/com/sk89q/jnbt/NBTInputStream.java @@ -55,6 +55,10 @@ public final class NBTInputStream implements Closeable { this.is = new DataInputStream(is); } + public NBTInputStream(DataInputStream dis) { + this.is = dis; + } + public NBTInputStream(DataInput di) { this.is = di; } @@ -97,7 +101,7 @@ public final class NBTInputStream implements Closeable { readTagPaylodLazy(type, 0, name, getReader); } - private String readNamedTagName(int type) throws IOException { + public String readNamedTagName(int type) throws IOException { String name; if (type != NBTConstants.TYPE_END) { int nameLength = is.readShort() & 0xFFFF; @@ -111,7 +115,7 @@ public final class NBTInputStream implements Closeable { private byte[] buf; - private void readTagPaylodLazy(int type, int depth, String node, RunnableVal2 getReader) throws IOException { + public void readTagPaylodLazy(int type, int depth, String node, RunnableVal2 getReader) throws IOException { switch (type) { case NBTConstants.TYPE_END: return; @@ -213,7 +217,8 @@ public final class NBTInputStream implements Closeable { return; case NBTConstants.TYPE_COMPOUND: depth++; - for (int i = 0;;i++) { + // 3 + for (int i = 0; ; i++) { childType = is.readByte(); if (childType == NBTConstants.TYPE_END) { return; @@ -247,6 +252,28 @@ public final class NBTInputStream implements Closeable { } } + public static int getSize(int type) { + switch (type) { + default: + case NBTConstants.TYPE_END: + case NBTConstants.TYPE_BYTE: + return 1; + case NBTConstants.TYPE_BYTE_ARRAY: + case NBTConstants.TYPE_STRING: + case NBTConstants.TYPE_LIST: + case NBTConstants.TYPE_COMPOUND: + case NBTConstants.TYPE_INT_ARRAY: + case NBTConstants.TYPE_SHORT: + return 2; + case NBTConstants.TYPE_FLOAT: + case NBTConstants.TYPE_INT: + return 4; + case NBTConstants.TYPE_DOUBLE: + case NBTConstants.TYPE_LONG: + return 8; + } + } + private Object readTagPaylodRaw(int type, int depth) throws IOException { switch (type) { case NBTConstants.TYPE_END: @@ -322,7 +349,7 @@ public final class NBTInputStream implements Closeable { * @return the tag * @throws IOException if an I/O error occurs. */ - private Tag readTagPayload(int type, int depth) throws IOException { + public Tag readTagPayload(int type, int depth) throws IOException { switch (type) { case NBTConstants.TYPE_END: if (depth == 0) { diff --git a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/ClipboardFormat.java b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/ClipboardFormat.java index e76d1ff9..85b6dc7f 100644 --- a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/ClipboardFormat.java +++ b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/ClipboardFormat.java @@ -24,6 +24,7 @@ import com.boydti.fawe.object.clipboard.AbstractClipboardFormat; import com.boydti.fawe.object.clipboard.DiskOptimizedClipboard; import com.boydti.fawe.object.clipboard.IClipboardFormat; import com.boydti.fawe.object.io.PGZIPOutputStream; +import com.boydti.fawe.object.io.ResettableFileInputStream; import com.boydti.fawe.object.schematic.FaweFormat; import com.boydti.fawe.object.schematic.PNGWriter; import com.boydti.fawe.object.schematic.Schematic; @@ -56,9 +57,14 @@ public enum ClipboardFormat { SCHEMATIC(new AbstractClipboardFormat("SCHEMATIC", "mcedit", "mce", "schematic") { @Override public ClipboardReader getReader(InputStream inputStream) throws IOException { - inputStream = new BufferedInputStream(inputStream); - NBTInputStream nbtStream = new NBTInputStream(new BufferedInputStream(new GZIPInputStream(inputStream))); - return new SchematicReader(nbtStream); + if (inputStream instanceof FileInputStream) { + inputStream = new ResettableFileInputStream((FileInputStream) inputStream); + } + BufferedInputStream buffered = new BufferedInputStream(inputStream); + NBTInputStream nbtStream = new NBTInputStream(new BufferedInputStream(new GZIPInputStream(buffered))); + SchematicReader input = new SchematicReader(nbtStream); + input.setUnderlyingStream(inputStream); + return input; } @Override diff --git a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java index 10dbccb4..8f10c8de 100644 --- a/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java +++ b/core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/SchematicReader.java @@ -19,6 +19,8 @@ package com.sk89q.worldedit.extent.clipboard.io; +import com.boydti.fawe.Fawe; +import com.boydti.fawe.jnbt.CorruptSchematicStreamer; import com.boydti.fawe.jnbt.SchematicStreamer; import com.sk89q.jnbt.CompoundTag; import com.sk89q.jnbt.NBTInputStream; @@ -26,6 +28,7 @@ import com.sk89q.jnbt.Tag; import com.sk89q.worldedit.extent.clipboard.Clipboard; import com.sk89q.worldedit.world.registry.WorldData; import java.io.IOException; +import java.io.InputStream; import java.util.Map; import java.util.UUID; import java.util.logging.Logger; @@ -40,7 +43,8 @@ import static com.google.common.base.Preconditions.checkNotNull; public class SchematicReader implements ClipboardReader { private static final Logger log = Logger.getLogger(SchematicReader.class.getCanonicalName()); - private final NBTInputStream inputStream; + private NBTInputStream inputStream; + private InputStream rootStream; /** * Create a new instance. @@ -52,13 +56,23 @@ public class SchematicReader implements ClipboardReader { this.inputStream = inputStream; } + public void setUnderlyingStream(InputStream in) { + this.rootStream = in; + } + @Override public Clipboard read(WorldData data) throws IOException { return read(data, UUID.randomUUID()); } - public Clipboard read(WorldData data, UUID clipboardId) throws IOException { - return new SchematicStreamer(inputStream, clipboardId).getClipboard(); + public Clipboard read(WorldData data, final UUID clipboardId) throws IOException { + try { + return new SchematicStreamer(inputStream, clipboardId).getClipboard(); + } catch (Exception e) { + Fawe.debug("Input is corrupt!"); + e.printStackTrace(); + return new CorruptSchematicStreamer(rootStream, clipboardId).recover(); + } } private static T requireTag(Map items, String key, Class expected) throws IOException {