Bleeding: Add compressed letter tree (experimental). Move stuff a little

for globalchat.

src/fr/neatmonster/nocheatplus/checks/chat/GlobalChat.java

@@ -4,9 +4,9 @@ import org.bukkit.entity.Player;
 
 import fr.neatmonster.nocheatplus.checks.Check;
 import fr.neatmonster.nocheatplus.checks.CheckType;
-import fr.neatmonster.nocheatplus.checks.chat.analysis.LetterEngine;
 import fr.neatmonster.nocheatplus.checks.chat.analysis.MessageLetterCount;
 import fr.neatmonster.nocheatplus.checks.chat.analysis.WordLetterCount;
+import fr.neatmonster.nocheatplus.checks.chat.analysis.engine.LetterEngine;
 import fr.neatmonster.nocheatplus.hooks.NCPExemptionManager;
 
 /**
@@ -140,13 +140,13 @@ public class GlobalChat extends Check{
 			score += wEngine;
 		}
 		
-//		System.out.println(score);
-		
 		// Wrapping it up. --------------------
 		// Add weight to frequency counts.
 		data.globalChatFrequency.add(time, score);
 		final float accumulated = cc.globalChatFrequencyWeight * data.globalChatFrequency.getScore(cc.globalChatFrequencyFactor);
 		
+//		System.out.println("Total score: " + score + " (" + accumulated + ")");
+		
 		if (score < 2.0f * cc.globalChatFrequencyWeight)
 			// Reset the VL.
 			data.globalChatVL = 0.0;

src/fr/neatmonster/nocheatplus/checks/chat/analysis/ds/LookupTree.java

@@ -0,0 +1,122 @@
+package fr.neatmonster.nocheatplus.checks.chat.analysis.ds;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import fr.neatmonster.nocheatplus.checks.chat.analysis.ds.LookupTree.Node;
+
+
+/**
+ * Tree for some sequence lookup. <br>
+ * Pretty fat, for evaluation purposes.
+ * @author mc_dev
+ *
+ */
+public class LookupTree<K, N extends Node<K>>{
+	
+	public static class Node<K>{
+		public boolean isEnd = false;
+		public Map<K, Node<K>> children = null;
+		
+		public Node(){
+			
+		}
+		
+		public Node<K> getChild(final K key, final NodeFactory<K, Node<K>> factory){
+			if (children == null){
+				if (factory != null) children = new HashMap<K, Node<K>>();
+				else return null;
+			}
+			Node<K> node = children.get(key);
+			if (node != null) return node;
+			else if (factory == null) return null;
+			else{
+				node = factory.getNewNode();
+				children.put(key, node);
+				return node;
+			}
+		}
+	}
+	
+	public static interface NodeFactory<K, N extends Node<K>>{
+		public N getNewNode();
+	}
+	
+	public static class LookupEntry<K, N extends Node<K>>{
+		/** The node, if lookup matched.*/
+		public final N node;
+		/** The node at which insertion did/would happen */
+		public final N insertion;
+		/** Depth to root from insertion point. */
+		public final int depth;
+		public LookupEntry(N node , N insertion, int depth){
+			this.node = node;
+			this.insertion = insertion;
+			this.depth = depth;
+		}
+	}
+	
+	protected final NodeFactory<K, N> factory;
+	
+	protected N root;
+	
+	public LookupTree(final Class<N> clazz){
+		this(new NodeFactory<K, N>() {
+			@Override
+			public N getNewNode() {
+				try {
+					return clazz.newInstance();
+				} catch (InstantiationException e) {
+					return null;
+				} catch (IllegalAccessException e) {
+					return null;
+				}
+			}
+		});
+	}
+	
+	public LookupTree(NodeFactory<K, N> factory){
+		this.factory = factory;
+		this.root = factory.getNewNode();
+	}
+	
+	public LookupEntry<K, N> lookup(K[] keys, final boolean create){
+		return lookup(Arrays.asList(keys), create);
+	}
+	
+	@SuppressWarnings("unchecked")
+	public LookupEntry<K, N> lookup(final List<K> keys, final boolean create){
+		N insertion = root;
+		N leaf = null;
+		int depth = 0;
+		N current = root;
+		final NodeFactory<K, N> factory = (NodeFactory<K, N>) (create ? this.factory : null);
+		for (final K key : keys){
+			final N child = (N) current.getChild(key, null);
+			if (child == null){
+				if (factory == null)
+					break;
+				else{
+					current = (N) current.getChild(key, (NodeFactory<K, Node<K>>) factory);
+				}
+			}
+			else{
+				// Node already exists, set as insertion point.
+				insertion = current = child;
+				depth ++;
+			}
+		}
+		if (create || insertion.isEnd && depth == keys.size()){
+			leaf = current;
+			if (insertion != current) current.isEnd = true;
+		}
+		return new LookupEntry<K, N>(leaf, insertion, depth);
+	}
+
+	public void clear() {
+		root = factory.getNewNode();
+		// TODO: maybe more unlinking ?
+	}
+}

src/fr/neatmonster/nocheatplus/checks/chat/analysis/engine/AbstractWordProcessor.java

@@ -1,9 +1,12 @@
-package fr.neatmonster.nocheatplus.checks.chat.analysis;
+package fr.neatmonster.nocheatplus.checks.chat.analysis.engine;
 
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
+import fr.neatmonster.nocheatplus.checks.chat.analysis.MessageLetterCount;
+import fr.neatmonster.nocheatplus.checks.chat.analysis.WordLetterCount;
+
 public abstract class AbstractWordProcessor implements WordProcessor{
 	
 	/**
@@ -46,7 +49,6 @@ public abstract class AbstractWordProcessor implements WordProcessor{
 			score += loop(ts, index, key, word);
 		}
 		score /= (float) message.words.length;
-//		System.out.println(getProcessorName() +": " + score);
 		return score;
 	}
 	

src/fr/neatmonster/nocheatplus/checks/chat/analysis/engine/CompressedChars.java

@@ -0,0 +1,84 @@
+package fr.neatmonster.nocheatplus.checks.chat.analysis.engine;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import fr.neatmonster.nocheatplus.checks.chat.analysis.MessageLetterCount;
+import fr.neatmonster.nocheatplus.checks.chat.analysis.WordLetterCount;
+import fr.neatmonster.nocheatplus.checks.chat.analysis.ds.LookupTree;
+import fr.neatmonster.nocheatplus.checks.chat.analysis.ds.LookupTree.LookupEntry;
+import fr.neatmonster.nocheatplus.checks.chat.analysis.ds.LookupTree.Node;
+import fr.neatmonster.nocheatplus.checks.chat.analysis.ds.LookupTree.NodeFactory;
+
+public class CompressedChars extends AbstractWordProcessor{
+
+	protected final LookupTree<Character, Node<Character>> tree = new LookupTree<Character, Node<Character>>(
+			new NodeFactory<Character, Node<Character>>() {
+				@Override
+				public Node<Character> getNewNode() {
+					return new Node<Character>();
+				}
+			});
+	
+	protected final int maxAdd;
+	
+	protected int added = 0;
+
+	private final boolean sort;
+	
+	public CompressedChars(int maxAdd, boolean sort) {
+		super("CompressedChars");
+		this.maxAdd = maxAdd;
+		this.sort = sort;
+	}
+	
+	@Override
+	public void start(MessageLetterCount message) {
+		if (added + message.words.length > maxAdd) tree.clear();
+	}
+
+	@Override
+	public float loop(long ts, int index, String key, WordLetterCount word) {
+		final List<Character> letters = new ArrayList<Character>(word.counts.size());
+		final List<Character> numbers = new ArrayList<Character>(word.counts.size());
+		final List<Character> other = new ArrayList<Character>(word.counts.size());
+		for (Character c : word.counts.keySet()){
+			if (Character.isLetter(c)) letters.add(c);
+			else if (Character.isDigit(c)) numbers.add(c);
+			else other.add(c);
+		}
+		if (sort){
+			Collections.sort(letters);
+			Collections.sort(numbers);
+			Collections.sort(other);
+		}
+		float score = 0;
+		float div = 0;
+		if (!letters.isEmpty()){
+			score += getScore(letters);
+			div += 1;
+		}
+		if (!numbers.isEmpty()){
+			score += getScore(numbers);
+			div += 1;
+		}
+		if (!other.isEmpty()){
+			score += getScore(other);
+			div += 1;
+		}
+		return (div == 0) ? 0 : score;
+	}
+
+	private float getScore(List<Character> chars) {
+		final int len = chars.size();
+		LookupEntry<Character, Node<Character>> entry = tree.lookup(chars, true);
+		float score = (float) (entry.depth*entry.depth) / (float) (len*len) ;
+		if (entry.depth == chars.size()){
+			score += 0.2;
+			if (entry.insertion.isEnd) score += 0.2;
+		}
+		return score;
+	}
+
+}

src/fr/neatmonster/nocheatplus/checks/chat/analysis/engine/FlatWordBuckets.java

@@ -1,7 +1,9 @@
-package fr.neatmonster.nocheatplus.checks.chat.analysis;
+package fr.neatmonster.nocheatplus.checks.chat.analysis.engine;
 
 import java.util.LinkedHashMap;
 
+import fr.neatmonster.nocheatplus.checks.chat.analysis.MessageLetterCount;
+import fr.neatmonster.nocheatplus.checks.chat.analysis.WordLetterCount;
 import fr.neatmonster.nocheatplus.utilities.ActionFrequency;
 
 /**

src/fr/neatmonster/nocheatplus/checks/chat/analysis/engine/LetterEngine.java

@@ -1,8 +1,10 @@
-package fr.neatmonster.nocheatplus.checks.chat.analysis;
+package fr.neatmonster.nocheatplus.checks.chat.analysis.engine;
 
 import java.util.ArrayList;
 import java.util.List;
 
+import fr.neatmonster.nocheatplus.checks.chat.analysis.MessageLetterCount;
+
 
 /**
  * Process words.
@@ -16,6 +18,7 @@ public class LetterEngine {
 	public LetterEngine(){
 		// Add word processors.
 		processors.add(new FlatWordBuckets(1000, 4, 1500, 0.9f));
+		processors.add(new CompressedChars(2000, false));
 	}
 	
 	public float feed(final MessageLetterCount letterCount){
@@ -23,6 +26,9 @@ public class LetterEngine {
 		// Run all processors.
 		for (final WordProcessor processor : processors){
 			final float refScore = processor.process(letterCount);
+			
+//			System.out.println(processor.getProcessorName() +": " + refScore);
+			
 			// TODO: max or sum or average or flexible (?)...
 			score = Math.max(score, refScore);
 		}

src/fr/neatmonster/nocheatplus/checks/chat/analysis/engine/WordProcessor.java

@@ -1,4 +1,6 @@
-package fr.neatmonster.nocheatplus.checks.chat.analysis;
+package fr.neatmonster.nocheatplus.checks.chat.analysis.engine;
+
+import fr.neatmonster.nocheatplus.checks.chat.analysis.MessageLetterCount;
 
 
 public interface WordProcessor{