Significant headway on the rewrite.

Handle crypto with the excellent crytography library, still a lot of TODOs to take care of but at least we can connect now

.gitignore

@@ -1,3 +1,6 @@
 *.pyc
 build/*
 src/*
+
+*.iml
+.idea/

authentication/auth.py

@@ -12,13 +12,18 @@ class Response(object):
     payload = None
 
 
+class YggdrasilError(Exception):
+
+    def __init__(self, error='', human_readable_error=''):
+        self.error = error
+        self.human_readable_error = human_readable_error
+
+
 def make_request(url, payload):
     """Makes http requests to the Yggdrasil authentication service
 
-    Returns a Response object with an error boolean, if there is an error
+    Returns a Response object.
-    then it will also contain `error` and `human_error` fields
+    If there is an error then it will raise a YggdrasilError.
-    otherwise a `payload` field will be returned with the actual response
-    from Yggdrasil
     """
     response = Response()
 
@@ -35,18 +40,21 @@ def make_request(url, payload):
         error = e.read()
         error = json.loads(error)
 
-        response.error = True
         response.human_error = error['errorMessage']
         response.error = error['error']
-        return response
+        raise YggdrasilError(error['error'], error['errorMessage'])
 
     except urllib2.URLError, e:
-        response.error = True
+        raise YggdrasilError(e.reason, e.reason)
-        response.human_error = e.reason
-        return response
 
     # ohey, everything didn't end up crashing and burning
+    if http_response == "":
+        http_response = "{}"
+    try:
         json_response = json.loads(http_response)
+    except ValueError, e:
+        raise YggdrasilError(e.message, "JSON parsing exception on data: " + http_response)
+
     response.payload = json_response
     return response
 
@@ -69,10 +77,6 @@ def login_to_minecraft(username, password):
     response = make_request(BASE_URL + "authenticate", payload)
 
     login_response = LoginResponse()
-    if response.error:
-        login_response.error = True
-        login_response.human_error = response.human_error
-    else:
     payload = response.payload
 
     login_response.error = False

network/connection.py

@@ -1,51 +1,89 @@
 from collections import deque
 from threading import Lock
+from zlib import decompress
 import threading
 import socket
 import time
 import select
+import authentication
 
 from packets import *
-from start import PROTOCOL_VERSION
 from types import VarInt
 
+PROTOCOL_VERSION = 47
 
-class Connection:
+
+class ConnectionOptions(object):
+    # TODO: allow these options to be overriden from a constructor below
+    address = None
+    port = None
+    use_encryption = True
+    compression_threshold = -1
+    compression_enabled = False
+
+
+class Connection(object):
     """This class represents a connection to a minecraft
-    server, it handles everything from connecting, sending packets,
+    server, it handles everything from connecting, sending packets to
     handling default network behaviour
     """
-    outgoing_packet_queue = deque()
+    _outgoing_packet_queue = deque()
-    write_lock = Lock()
+    _write_lock = Lock()
     networking_thread = None
+    options = ConnectionOptions()
 
     def __init__(self, address, port, login_response):
-        self.address = address
+        """Sets up an instance of this object to be able to connect to a minecraft server
-        self.port = port
+        The connect method needs to be called in order to actually begin the connection
+
+        :param address: address of the server to connect to
+        :param port(int): port of the server to connect to
+        :param login_response: login_response object as obtained from the authentication package
+        """
+        self.options.address = address
+        self.options.port = port
         self.login_response = login_response
-        self.reactor = HandshakeReactor(self)
+        self.reactor = PacketReactor(self)
 
     def _start_network_thread(self):
         self.networking_thread = NetworkingThread(self)
         self.networking_thread.start()
 
     def write_packet(self, packet, force=False):
-        if force:
+        """Writes a packet to the server.
-            self.write_lock.acquire()
+        If force is set to true, the method attempts to acquire the write lock and write the packet
-            packet.write(self.socket)
+        out immediately, and as such may block.
-            self.write_lock.release()
+        If force is false then the packet will be added to the end of the packet writing queue
-        else:
+        to be sent 'as soon as possible'
-            self.outgoing_packet_queue.append(packet)
 
-    # Mostly a convenience function, caller should make sure they have the
+        :param packet:
+        :param force(bool): Specifies if the packet write should be immediate
+        :return:
+        """
+        if force:
+            self._write_lock.acquire()
+            if self.options.compression_enabled:
+                packet.write(self.socket, self.options.compression_threshold)
+            else:
+                packet.write(self.socket)
+            self._write_lock.release()
+        else:
+            self._outgoing_packet_queue.append(packet)
+
+    def _pop_packet(self):
+        # Pops the topmost packet off the outgoing queue and writes it out through the socket
+        #
+        # Mostly an internal convenience function, caller should make sure they have the
         # write lock acquired to avoid issues caused by asynchronous access to the socket.
         # This should be the only method that removes elements from the outbound queue
-    def _pop_packet(self):
+        if len(self._outgoing_packet_queue) == 0:
-        if len(self.outgoing_packet_queue) == 0:
             return False
         else:
-            packet = self.outgoing_packet_queue.popleft()
+            packet = self._outgoing_packet_queue.popleft()
-            print "Writing out: " + hex(packet.id) + " / " + packet.name
+            print "Writing out: " + hex(packet.id) + " / " + packet.packet_name
+            if self.options.compression_enabled:
+                packet.write(self.socket, self.options.compression_threshold)
+            else:
                 packet.write(self.socket)
             return True
 
@@ -59,23 +97,31 @@ class Connection:
         self.write_packet(request_packet)
 
     def connect(self):
+        """Attempt to begin connecting to the server
+        """
         self._connect()
         self._handshake()
 
+        self.reactor = LoginReactor(self)
+        self._start_network_thread()
+        login_start_packet = LoginStartPacket()
+        login_start_packet.name = self.login_response.username
+        self.write_packet(login_start_packet)
+
     def _connect(self):
         # Connect a socket to the server and create a file object from the socket
         # The file object is used to read any and all data from the socket since it's "guaranteed"
         # to read the number of bytes specified, the socket itself will mostly be
         # used to write data upstream to the server
         self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.socket.connect((self.address, self.port))
+        self.socket.connect((self.options.address, self.options.port))
         self.file_object = self.socket.makefile()
 
     def _handshake(self, next_state=2):
         handshake = HandShakePacket()
         handshake.protocol_version = PROTOCOL_VERSION
-        handshake.server_address = self.address
+        handshake.server_address = self.options.address
-        handshake.server_port = self.port
+        handshake.server_port = self.options.port
         handshake.next_state = next_state
 
         self.write_packet(handshake)
@@ -96,15 +142,12 @@ class NetworkingThread(threading.Thread):
                 break
             # Attempt to write out as many as 300 packets as possible every 0.05 seconds (20 ticks per second)
             num_packets = 0
-            self.connection.write_lock.acquire()
+            self.connection._write_lock.acquire()
             while self.connection._pop_packet():
-
-                self.connection._pop_packet()
-
                 num_packets += 1
                 if num_packets >= 300:
                     break
-            self.connection.write_lock.release()
+            self.connection._write_lock.release()
 
             # Read and react to as many as 50 packets 
             num_packets = 0
@@ -121,7 +164,7 @@ class NetworkingThread(threading.Thread):
             time.sleep(0.05)
 
 
-class PacketReactor:
+class PacketReactor(object):
     state_name = None
     clientbound_packets = None
 
@@ -130,28 +173,122 @@ class PacketReactor:
     def __init__(self, connection):
         self.connection = connection
 
+    # Design objectives:
+    #
+    # Avoid buffering data as much as possible
+    # Wherever possible, read directly from the network stream
+    # Minimize any overheads, reading packets should be simple
     def read_packet(self, stream):
-        ready = select.select([self.connection.socket], [], [], self.TIME_OUT)
+        ready_to_read, _, __ = select.select([self.connection.socket], [], [], self.TIME_OUT)
-        if self.connection.socket in ready[0]:
+        real_stream = stream
-            length = VarInt.read_socket(self.connection.socket)
-            packet_id = VarInt.read(stream)
 
+        if self.connection.socket in ready_to_read:
+            length = VarInt.read_socket(self.connection.socket)
+
+            if self.connection.options.compression_enabled:
+                compressed_size = VarInt.read(stream)
+                #  If this is a compressed packet we'll need to decompress it into a buffer
+                #  and then pretend that that is the actual network stream
+                if compressed_size > 0:
+                    compressed_packet = stream.read(compressed_size)
+                    stream = PacketBuffer()
+                    stream.send(decompress(compressed_packet))
+                    stream.reset_cursor()
+
+            packet_id = VarInt.read_socket(self.connection.socket)
+
+            print "Reading packet: " + str(packet_id) + " / " + hex(packet_id) + " (Size: " + str(length) + ")"
+
+            # If we know the structure of the packet, attempt to parse it
+            # otherwise just skip it
             if packet_id in self.clientbound_packets:
                 packet = self.clientbound_packets[packet_id]()
                 packet.read(stream)
                 return packet
             else:
-                print "Unkown packet: " + str(packet_id) + " / " + hex(packet_id)
+                # TODO: remove debug
+                print "Unknown packet: " + str(packet_id) + " / " + hex(packet_id) + " (Size: " + str(length) + ")"
+
+                # if this is a compressed packet then we've already read it from the stream
+                # otherwise we need to skip the rest of the bytes properly
+                if self.connection.options.compression_enabled and compressed_size > 0:
+                    return Packet()
+
+                # if compression is enabled and the data isn't compressed then we need to
+                # subtract the size of the compressed_size and packet_id VarInts from the total data length
+                # to get the number of bytes to skip
+                if self.connection.options.compression_enabled and compressed_size == 0:
+                    real_stream.read(length - (VarInt.size(compressed_size) + VarInt.size(packet_id)))
+                    return Packet()
+
+                # If compression isn't enabled, just subtract the size of the packet_id VarInt we read
+                # from the total length of the packet
+                real_stream.read(length - VarInt.size(packet_id))
+
                 return Packet()
         else:
             return None
 
     def react(self, packet):
-        pass
+        raise NotImplementedError("Call to base reactor")
 
 
-class HandshakeReactor(PacketReactor):
+class LoginReactor(PacketReactor):
-    clientbound_packets = state_handshake_clientbound
+    clientbound_packets = state_login_clientbound
+
+    def react(self, packet):
+        # TODO: Add some way to bypass encryption? (connection.options.use_encryption) Not sure if it's still possible.
+        if packet.packet_name == "encryption request":
+            import encryption
+
+            secret = encryption.generate_shared_secret()
+            encrypted_token, encrypted_secret = encryption.encrypt_token_and_secret(packet.public_key,
+                                                                                    packet.verify_token, secret)
+
+            # A server id of '-' means the server is in offline mode
+            if packet.server_id != '-':
+                url = "https://sessionserver.mojang.com/session/minecraft/join"
+                server_id = encryption.generate_verification_hash(packet.server_id, secret, packet.public_key)
+
+                authentication.make_request(url, {'accessToken': self.connection.login_response.access_token,
+                                                  'selectedProfile': self.connection.login_response.profile_id,
+                                                  'serverId': server_id})
+
+            encryption_response = EncryptionResponsePacket()
+            encryption_response.shared_secret = encrypted_secret
+            encryption_response.verify_token = encrypted_token
+
+            # Forced because we don't want to send this encrypted which it will be
+            # if we put it in the queue as we'd have wrapped the socket and file object by then
+            self.connection.write_packet(encryption_response, force=True)
+
+            # Enable the encryption
+            cipher = encryption.create_AES_cipher(secret)
+            encryptor = cipher.encryptor()
+            decryptor = cipher.decryptor()
+            self.connection.socket = encryption.EncryptedSocketWrapper(self.connection.socket,
+                                                                       encryptor, decryptor)
+            self.connection.file_object = encryption.EncryptedFileObjectWrapper(self.connection.file_object,
+                                                                                decryptor)
+
+        if packet.packet_name == "disconnect":
+            print(packet.json_data)  # TODO: handle propagating this back
+
+        if packet.packet_name == "login success":
+            self.connection.reactor = PlayingReactor(self.connection)
+
+        if packet.packet_name == "set compression":
+            self.connection.options.compression_threshold = packet.threshold
+            self.connection.options.compression_enabled = True
+
+
+class PlayingReactor(PacketReactor):
+    clientbound_packets = state_playing_clientbound
+
+    def react(self, packet):
+        if packet.packet_name == "set compression":
+            self.connection.options.compression_threshold = packet.threshold
+            self.connection.options.compression_enabled = True
 
 
 class StatusReactor(PacketReactor):

network/encryption.py

@@ -0,0 +1,81 @@
+import os
+from hashlib import sha1
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15
+from cryptography.hazmat.primitives.serialization import load_der_public_key
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+
+
+def generate_shared_secret():
+    return os.urandom(16)
+
+
+def create_AES_cipher(shared_secret):
+    cipher = Cipher(algorithms.AES(shared_secret), modes.CFB8(shared_secret), backend=default_backend())
+    return cipher
+
+
+def encrypt_token_and_secret(pubkey, verification_token, shared_secret):
+    """Encrypts the verification token and shared secret with the server's public key
+
+    :param pubkey: The RSA public key provided by the server
+    :param verification_token: The verification token provided by the server
+    :param shared_secret: The generated shared secret
+    :return: A tuple containing (encrypted token, encrypted secret)
+    """
+    pubkey = load_der_public_key(pubkey, default_backend())
+
+    if not isinstance(pubkey, rsa.RSAPublicKey):
+        raise RuntimeError("Public key provided by server not an RSA key")
+
+    encrypted_token = pubkey.encrypt(verification_token, PKCS1v15())
+    encrypted_secret = pubkey.encrypt(shared_secret, PKCS1v15())
+    return encrypted_token, encrypted_secret
+
+
+def generate_verification_hash(server_id, shared_secret, public_key):
+    verification_hash = sha1()
+
+    verification_hash.update(server_id)
+    verification_hash.update(shared_secret)
+    verification_hash.update(public_key)
+
+    # Minecraft first parses the sha1 bytes as a signed number and then spits outs
+    # its hex representation
+    number = _number_from_bytes(verification_hash.digest(), signed=True)
+    return format(number, 'x')
+
+
+def _number_from_bytes(b, signed=False):
+    if len(b) == 0:
+        b = b'\x00'
+    num = int(str(b).encode('hex'), 16)
+    if signed and (ord(b[0]) & 0x80):
+        num -= 2 ** (len(b) * 8)
+    return num
+
+
+class EncryptedFileObjectWrapper(object):
+    def __init__(self, file_object, decryptor):
+        self.actual_file_object = file_object
+        self.decryptor = decryptor
+
+    def read(self, length):
+        return self.decryptor.update(self.actual_file_object.read(length))
+
+
+class EncryptedSocketWrapper(object):
+    def __init__(self, socket, encryptor, decryptor):
+        self.actual_socket = socket
+        self.encryptor = encryptor
+        self.decryptor = decryptor
+
+    def recv(self, length):
+        return self.decryptor.update(self.actual_socket.recv(length))
+
+    def send(self, data):
+        self.actual_socket.send(self.encryptor.update(data))
+
+    def fileno(self):
+        return self.actual_socket.fileno()

network/packets.py

@@ -1,4 +1,5 @@
 from io import BytesIO
+from zlib import compress
 
 from types import *
 
@@ -9,15 +10,26 @@ class PacketBuffer(object):
         self.b = BytesIO()
 
     def send(self, value):
+        """
+        Writes the given bytes to the buffer, designed to emulate socket.send
+        :param value: The bytes to write
+        """
         self.b.write(value)
 
+    def read(self, length):
+        return self.b.read(length)
+
+    def reset_cursor(self):
+        self.b.seek(0)
+
     def get_writable(self):
         return self.b.getvalue()
 
 
 class Packet(object):
 
-    name = "base"
+    packet_name = "base"
+    id = -0x01
     definition = []
 
     def __init__(self):
@@ -26,12 +38,13 @@ class Packet(object):
     def read(self, file_object):
         for field in self.definition:
             for var_name, data_type in field.iteritems():
-                setattr(self, var_name, data_type.read(file_object))
+                value = data_type.read(file_object)
+                setattr(self, var_name, value)
 
-    def write(self, socket):
+    def write(self, socket, compression_threshold=-1):
         # buffer the data since we need to know the length of each packet's payload
         packet_buffer = PacketBuffer()
-        # write off the id right off the bat
+        # write packet's id right off the bat in the header
         VarInt.send(self.id, packet_buffer)
 
         for field in self.definition:
@@ -39,6 +52,8 @@ class Packet(object):
                 data = getattr(self, var_name)
                 data_type.send(data, packet_buffer)
 
+        # TODO: implement compression
+
         VarInt.send(len(packet_buffer.get_writable()), socket)  # Packet Size
         socket.send(packet_buffer.get_writable())  # Packet Payload
 
@@ -47,7 +62,7 @@ class Packet(object):
 # ==============
 class HandShakePacket(Packet):
     id = 0x00
-    name = "handshake"
+    packet_name = "handshake"
     definition = [
         {'protocol_version': VarInt},
         {'server_address': String},
@@ -67,18 +82,17 @@ state_handshake_serverbound = {
 # ==============
 class ResponsePacket(Packet):
     id = 0x00
-    name = "response"
+    packet_name = "response"
     definition = [
         {'json_response': String}]
 
 
 class PingPacket(Packet):
     id = 0x01
-    name = "ping"
+    packet_name = "ping"
     definition = [
         {'time': Long}]
 
-
 state_status_clientbound = {
     0x00: ResponsePacket,
     0x01: PingPacket
@@ -87,71 +101,118 @@ state_status_clientbound = {
 
 class RequestPacket(Packet):
     id = 0x00
-    name = "request"
+    packet_name = "request"
     definition = []
 
 
 class PingPacket(Packet):
     id = 0x01
-    name = "ping"
+    packet_name = "ping"
     definition = [
         {'time': Long}]
 
-
 state_status_serverbound = {
     0x00: RequestPacket,
     0x01: PingPacket
 }
 
+
 # Login State
 # ==============
 class DisconnectPacket(Packet):
     id = 0x00
-    name = "disconnect"
+    packet_name = "disconnect"
     definition = [
         {'json_data': String}]
 
 
 class EncryptionRequestPacket(Packet):
     id = 0x01
-    name = "encryption request"
+    packet_name = "encryption request"
     definition = [
         {'server_id': String},
-        {'public_key': ByteArray},
+        {'public_key': VarIntPrefixedByteArray},
-        {'verify_token': ByteArray}]
+        {'verify_token': VarIntPrefixedByteArray}]
 
 
-class LoginSucessPacket(Packet):
+class LoginSuccessPacket(Packet):
     id = 0x02
-    name = "login success"
+    packet_name = "login success"
     definition = [
         {'UUID': String},
         {'Username': String}]
 
 
+class SetCompressionPacket(Packet):
+    id = 0x03
+    packet_name = "set compression"
+    definition = [
+        {'threshold': VarInt}]
+
 state_login_clientbound = {
     0x00: DisconnectPacket,
     0x01: EncryptionRequestPacket,
-    0x02: LoginSucessPacket
+    0x02: LoginSuccessPacket,
+    0x03: SetCompressionPacket
 }
 
 
 class LoginStartPacket(Packet):
     id = 0x00
-    name = "login start"
+    packet_name = "login start"
     definition = [
         {'name': String}]
 
 
 class EncryptionResponsePacket(Packet):
     id = 0x01
-    name = "encryption response"
+    packet_name = "encryption response"
     definition = [
-        {'shared_secret': ByteArray},
+        {'shared_secret': VarIntPrefixedByteArray},
-        {'verify_token': ByteArray}]
+        {'verify_token': VarIntPrefixedByteArray}]
-
 
 state_login_serverbound = {
     0x00: LoginStartPacket,
     0x01: EncryptionResponsePacket
 }
+
+
+# Playing State
+# ==============
+
+class KeepAlivePacket(Packet):
+    id = 0x00
+    packet_name = "keep alive"
+    definition = [
+        {'keep_alive_id': VarInt}]
+
+
+class JoinGamePacket(Packet):
+    id = 0x01
+    packet_name = "join game"
+    definition = [
+        {'entity_id': Integer},
+        {'game_mode': UnsignedByte},
+        {'dimension': Byte},
+        {'difficulty': UnsignedByte},
+        {'max_players': UnsignedByte},
+        {'level_type': String},
+        {'reduced_debug_info': Boolean}]
+
+
+class SetCompressionPacketPlayState(Packet):
+    id = 0x46
+    packet_name = "set compression"
+    definition = [
+        {'threshold': VarInt}]
+
+
+state_playing_clientbound = {
+    0x00: KeepAlivePacket,
+    0x01: JoinGamePacket,
+    0x46: SetCompressionPacketPlayState
+}
+
+state_playing_serverbound = {
+
+}

network/types.py

@@ -28,6 +28,16 @@ class Boolean(Type):
         socket.send(struct.pack('?', value))
 
 
+class UnsignedByte(Type):
+    @staticmethod
+    def read(file_object):
+        return struct.unpack('>B', file_object.read(1))[0]
+
+    @staticmethod
+    def send(value, socket):
+        socket.send(struct.pack('>B', value))
+
+
 class Byte(Type):
     @staticmethod
     def read(file_object):
@@ -103,6 +113,28 @@ class VarInt(Type):
                 break
         socket.send(o)
 
+    @staticmethod
+    def size(value):
+        for max_value, size in VarInt_size_table.iteritems():
+            if value < max_value:
+                return size
+
+# Maps (maximum integer value -> size of VarInt in bytes)
+VarInt_size_table = {
+    2**7: 1,
+    2**14: 2,
+    2**21: 3,
+    2**28: 4,
+    2**35: 5,
+    2**42: 6,
+    2**49: 7,
+    2**56: 8,
+    2**63: 9,
+    2**70: 10,
+    2**77: 11,
+    2**84: 12
+}
+
 
 class Long(Type):
     @staticmethod
@@ -134,7 +166,7 @@ class Double(Type):
         socket.send(struct.pack('>d', value))
 
 
-class ByteArray(Type):
+class ShortPrefixedByteArray(Type):
     @staticmethod
     def read(file_object, length=None):
         if length is None:
@@ -147,6 +179,19 @@ class ByteArray(Type):
         socket.send(value)
 
 
+class VarIntPrefixedByteArray(Type):
+    @staticmethod
+    def read(file_object, length=None):
+        if length is None:
+            length = VarInt.read(file_object)
+        return struct.unpack(str(length) + "s", file_object.read(length))[0]
+
+    @staticmethod
+    def send(value, socket):
+        VarInt.send(len(value), socket)
+        socket.send(struct.pack(str(len(value)) + "s", value))
+
+
 class String(Type):
     @staticmethod
     def read(file_object):

requirements.txt

@@ -1 +1 @@
-pycrypto>=2.5
+cryptography

start.py

@@ -5,9 +5,6 @@ from optparse import OptionParser
 import authentication
 
 
-PROTOCOL_VERSION = 5
-
-
 def main():
     parser = OptionParser()
 
@@ -28,13 +25,13 @@ def main():
     if not options.password:
         options.password = getpass.getpass("Enter your password: ")
 
+    try:
         login_response = authentication.login_to_minecraft(options.username, options.password)
         from pprint import pprint  # TODO: remove debug
 
         pprint(vars(login_response))  # TODO: remove debug
-
+    except authentication.YggdrasilError as e:
-    if login_response.error:
+        print e.human_readable_error
-        print login_response.human_error
         return
 
     print("Logged in as " + login_response.username)
@@ -53,7 +50,7 @@ def main():
     from network.connection import Connection
 
     connection = Connection(address, port, login_response)
-    connection.status()
+    connection.connect()
 
     while True:
         try: