Initial Release of IGNCore version 2.5

core/aochat/BaseModule.py

@@ -0,0 +1,17 @@
+#
+# Base class of all modules...
+# makes accessing fields which exist in all modules easier
+#
+class BaseModule:
+    module_name = ""
+    module_dir = ""
+
+    # noinspection DuplicatedCode
+    def inject(self, registry):
+        pass
+
+    def pre_start(self):
+        pass
+
+    def start(self):
+        pass

core/aochat/bot.py

@@ -0,0 +1,117 @@
+import select
+import socket
+import struct
+
+from core.aochat.client_packets import LoginRequest, LoginSelect
+from core.aochat.crypt import generate_login_key
+from core.aochat.server_packets import ServerPacket, LoginOK, LoginError, LoginCharacterList
+from core.logger import Logger
+
+
+class Bot:
+    def __init__(self):
+        self.socket = None
+        self.char_id = None
+        self.char_name = None
+        self.logger = Logger(__name__)
+
+    def connect(self, host, port):
+        self.logger.info("Connecting to '%s:%d'" % (host, port))
+        self.socket = socket.create_connection((host, port), 10)
+
+    def disconnect(self):
+        if self.socket:
+            self.socket.shutdown(socket.SHUT_RDWR)
+            self.socket.close()
+            self.socket = None
+
+    def login(self, username, password, character):
+        character = character.capitalize()
+
+        # read seed packet
+        self.logger.info("Logging in as '%s'" % character)
+        seed_packet = self.read_packet(10)
+        seed = seed_packet.seed
+
+        # send back challenge
+        key = generate_login_key(seed, username, password)
+        login_request_packet = LoginRequest(0, username, key)
+        self.send_packet(login_request_packet)
+
+        # read character list
+        character_list_packet: LoginCharacterList = self.read_packet()
+        if isinstance(character_list_packet, LoginError):
+            self.logger.error("Error logging in: %s" % character_list_packet.message)
+            return False
+        if character not in character_list_packet.names:
+            self.logger.error("Character '%s' does not exist on this account" % character)
+            return False
+        index = character_list_packet.names.index(character)
+
+        # select character
+        self.char_id = character_list_packet.char_ids[index]
+        self.char_name = character_list_packet.names[index]
+        login_select_packet = LoginSelect(self.char_id)
+        self.send_packet(login_select_packet)
+
+        # wait for OK
+        packet = self.read_packet()
+        if packet.id == LoginOK.id:
+            self.logger.info("Connected!")
+            return packet
+        else:
+            self.logger.error("Error logging in: %s" % packet.message)
+            return False
+
+    def read_packet(self, max_delay_time=1):
+        """
+        Wait for packet from server.
+        """
+
+        read, write, error = select.select([self.socket], [], [], max_delay_time)
+        if not read:
+            return None
+        else:
+            # Read data from server
+            head = self.read_bytes(4)
+            packet_type, packet_length = struct.unpack(">2H", head)
+            data = self.read_bytes(packet_length)
+
+            try:
+                return ServerPacket.get_instance(packet_type, data)
+            except Exception as e:
+                self.logger.error(f"Error parsing packet parameters for packet_type {packet_type} and payload: {data}",
+                                  e)
+                return None
+
+    def send_packet(self, packet):
+        data = packet.to_bytes()
+        data = struct.pack(">2H", packet.id, len(data)) + data
+
+        self.write_bytes(data)
+
+    def read_bytes(self, num_bytes):
+        data = bytes()
+
+        while num_bytes > 0:
+            chunk = self.socket.recv(num_bytes)
+
+            if len(chunk) == 0:
+                raise EOFError
+
+            num_bytes -= len(chunk)
+            data = data + chunk
+
+        return data
+
+    def write_bytes(self, data):
+        num_bytes = len(data)
+
+        while num_bytes > 0:
+            sent = self.socket.send(data)
+
+            if sent == 0:
+                raise EOFError
+
+            data = data[sent:]
+            num_bytes -= sent

core/aochat/client_packets.py

@@ -0,0 +1,266 @@
+from core.aochat.packets import *
+
+
+class ClientPacket(Packet):
+    def __init__(self, packet_id, types, args):
+        self.id = packet_id
+        self.types = types
+        self.args = args
+
+    def to_bytes(self):
+        return encode_args(self.types, self.args)
+
+    def __str__(self):
+        return "ClientPacket(%d): %s" % (self.id, self.args)
+
+    @classmethod
+    def get_instance(cls, packet_id, data):
+        if packet_id == LoginRequest.id:
+            LoginRequest.from_bytes(data)
+        elif packet_id == LoginSelect.id:
+            LoginSelect.from_bytes(data)
+        elif packet_id == CharacterLookup.id:
+            CharacterLookup.from_bytes(data)
+        elif packet_id == PrivateMessage.id:
+            PrivateMessage.from_bytes(data)
+        elif packet_id == BuddyAdd.id:
+            BuddyAdd.from_bytes(data)
+        elif packet_id == BuddyRemove.id:
+            BuddyRemove.from_bytes(data)
+        elif packet_id == PrivateChannelInvite.id:
+            PrivateChannelInvite.from_bytes(data)
+        elif packet_id == PrivateChannelKick.id:
+            PrivateChannelKick.from_bytes(data)
+        elif packet_id == PrivateChannelJoin.id:
+            PrivateChannelJoin.from_bytes(data)
+        elif packet_id == PrivateChannelLeave.id:
+            PrivateChannelLeave.from_bytes(data)
+        elif packet_id == PrivateChannelKickAll.id:
+            PrivateChannelKickAll.from_bytes(data)
+        elif packet_id == PrivateChannelMessage.id:
+            PrivateChannelMessage.from_bytes(data)
+        elif packet_id == PublicChannelMessage.id:
+            PublicChannelMessage.from_bytes(data)
+        elif packet_id == Ping.id:
+            Ping.from_bytes(data)
+        elif packet_id == ChatCommand.id:
+            ChatCommand.from_bytes(data)
+        else:
+            return None
+
+
+class LoginRequest(ClientPacket):
+    id = 2
+    types = "ISS"
+
+    def __init__(self, unknown, username, key):
+        self.unknown = unknown
+        self.username = username
+        self.key = key
+        super().__init__(self.id, self.types, [self.unknown, self.username, self.key])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class LoginSelect(ClientPacket):
+    id = 3
+    types = "I"
+
+    def __init__(self, char_id):
+        self.char_id = char_id
+        super().__init__(self.id, self.types, [self.char_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class CharacterLookup(ClientPacket):
+    id = 21
+    types = "S"
+
+    def __init__(self, name):
+        self.name = name
+        super().__init__(self.id, self.types, [self.name])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateMessage(ClientPacket):
+    id = 30
+    types = "ISS"
+
+    def __init__(self, char_id, message, blob):
+        self.char_id = char_id
+        self.message = message
+        self.blob = blob
+        super().__init__(self.id, self.types, [self.char_id, self.message, self.blob])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class BuddyAdd(ClientPacket):
+    id = 40
+    types = "IS"
+
+    def __init__(self, char_id, status):
+        self.char_id = char_id
+        self.status = status
+        super().__init__(self.id, self.types, [self.char_id, self.status])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class BuddyRemove(ClientPacket):
+    id = 41
+    types = "I"
+
+    def __init__(self, char_id):
+        self.char_id = char_id
+        super().__init__(self.id, self.types, [self.char_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelInvite(ClientPacket):
+    id = 50
+    types = "I"
+
+    def __init__(self, char_id):
+        self.char_id = char_id
+        super().__init__(self.id, self.types, [self.char_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelKick(ClientPacket):
+    id = 51
+    types = "I"
+
+    def __init__(self, char_id):
+        self.char_id = char_id
+        super().__init__(self.id, self.types, [self.char_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelJoin(ClientPacket):
+    id = 52
+    types = "I"
+
+    def __init__(self, private_channel_id):
+        self.private_channel_id = private_channel_id
+        super().__init__(self.id, self.types, [self.private_channel_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelLeave(ClientPacket):
+    id = 53
+    types = "I"
+
+    def __init__(self, private_channel_id):
+        self.private_channel_id = private_channel_id
+        super().__init__(self.id, self.types, [self.private_channel_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelKickAll(ClientPacket):
+    id = 54
+    types = ""
+
+    def __init__(self):
+        super().__init__(self.id, self.types, [])
+
+    @classmethod
+    def from_bytes(cls, data):
+        return cls()
+
+
+class PrivateChannelMessage(ClientPacket):
+    id = 57
+    types = "ISS"
+
+    def __init__(self, private_channel_id, message, blob):
+        self.private_channel_id = private_channel_id
+        self.message = message
+        self.blob = blob
+        super().__init__(self.id, self.types, [self.private_channel_id, self.message, self.blob])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PublicChannelMessage(ClientPacket):
+    id = 65
+    types = "GSS"
+
+    def __init__(self, channel_id, message, blob):
+        self.channel_id = channel_id
+        self.message = message
+        self.blob = blob
+        super().__init__(self.id, self.types, [self.channel_id, self.message, self.blob])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class Ping(ClientPacket):
+    id = 100
+    types = "S"
+
+    def __init__(self, blob):
+        self.blob = blob
+        super().__init__(self.id, self.types, [self.blob])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class ChatCommand(ClientPacket):
+    id = 120
+    types = "s"
+
+    def __init__(self, commands):
+        self.commands = commands
+        super().__init__(self.id, self.types, [self.commands])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)

core/aochat/crypt.py

@@ -0,0 +1,110 @@
+#  Relevant parts of original copyright notice of AOChat.php:
+
+# Copyright (C) 2005 by Jürgen A. Erhard
+# Copyright (C) 2002-2004  Oskari Saarenmaa <auno@auno.org>.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+
+
+import random
+import socket
+import struct
+
+
+# This is 'half' Diffie-Hellman key exchange.
+# 'Half' as in we already have the server's key ($dhY)
+# $dhN is a prime and $dhG is generator for it.
+#
+# http://en.wikipedia.org/wiki/Diffie-Hellman_key_exchange
+
+
+# noinspection LongLine
+def generate_login_key(server_key, username, password):
+    dhY = 0x9c32cc23d559ca90fc31be72df817d0e124769e809f936bc14360ff4bed758f260a0d596584eacbbc2b88bdd410416163e11dbf62173393fbc0c6fefb2d855f1a03dec8e9f105bbad91b3437d8eb73fe2f44159597aa4053cf788d2f9d7012fb8d7c4ce3876f7d6cd5d0c31754f4cd96166708641958de54a6def5657b9f2e92
+    dhN = 0xeca2e8c85d863dcdc26a429a71a9815ad052f6139669dd659f98ae159d313d13c6bf2838e10a69b6478b64a24bd054ba8248e8fa778703b418408249440b2c1edd28853e240d8a7e49540b76d120d3b1ad2878b1b99490eb4a2a5e84caa8a91cecbdb1aa7c816e8be343246f80c637abc653b893fd91686cf8d32d6cfe5f2a6f
+    dhG = 0x5
+    dhx = random.randrange(0, 2 ** 256)
+
+    dhX = pow(dhG, dhx, dhN)
+    dhK = pow(dhY, dhx, dhN)
+
+    dhK = "%x" % dhK
+    if len(dhK) > 32:
+        dhK = dhK[:32]
+
+    dhK = eval("0x" + dhK)
+
+    challenge = "%s|%s|%s" % (username, server_key, password)
+
+    # prefix is an 8 bytes of randomness
+    prefix_bytes = random.randrange(0, 2 ** 64)
+    prefix = struct.pack(">Q", prefix_bytes)
+
+    length = 8 + 4 + len(challenge)  # prefix, int, ...
+    pad = " " * ((8 - length % 8) % 8)
+    challenge_len = struct.pack(">I", len(challenge))
+
+    plain = prefix + challenge_len + challenge.encode('ascii') + pad.encode('ascii')
+    crypted = aochat_crypt(dhK, plain)
+
+    if not crypted:
+        raise Exception("panic")
+
+    return ("%0x" % dhX) + "-" + crypted
+
+
+def aochat_crypt(key, data):
+    if len(data) % 8 != 0:
+        return None
+
+    cycle = [0, 0]
+    result = [0, 0]
+    ret = ""
+
+    key_arr = [socket.ntohl(int(s, 16)) for s in
+               struct.unpack("8s" * (len("%s" % key) // 8), ("%x" % key).encode('ascii'))]
+    data_arr = struct.unpack("I" * (len(data) // 4), data)
+
+    i = 0
+    while i < len(data_arr):
+        cycle[0] = data_arr[i] ^ result[0]
+        cycle[1] = data_arr[i + 1] ^ result[1]
+        result = aochat_tea_encrypt(cycle, key_arr)
+
+        p = "%08x%08x" % (socket.htonl(result[0]) & 0xffffffff, socket.htonl(result[1]) & 0xffffffff)
+
+        ret += p
+
+        i += 2
+
+    return ret
+
+
+def aochat_tea_encrypt(cycle, key):
+    a, b = cycle
+    total = 0
+    delta = 0x9e3779b9
+    i = 32
+
+    while i:
+        total = (total + delta) & 0xffffffff
+        a += (((b << 4 & 0xfffffff0) + key[0]) ^ (b + total) ^ ((b >> 5 & 0x7ffffff) + key[1])) & 0xffffffff
+        a &= 0xffffffff
+        b += (((a << 4 & 0xfffffff0) + key[2]) ^ (a + total) ^ ((a >> 5 & 0x7ffffff) + key[3])) & 0xffffffff
+        b &= 0xffffffff
+        i -= 1
+
+    return a, b

core/aochat/delay_queue.py

@@ -0,0 +1,34 @@
+import time
+
+
+class DelayQueue:
+    def __init__(self, recovery: int, burst=0):
+        self.recovery = recovery
+        self.burst = burst
+        self.items = []
+        self.next_packet = 0
+
+    def enqueue(self, item):
+        self.items.insert(0, item)
+
+    def dequeue(self):
+        if self.items:
+            t = time.time()
+            time_with_burst = t - (self.burst * self.recovery)
+            if self.next_packet < time_with_burst:
+                self.next_packet = time_with_burst
+
+            if t >= self.next_packet:
+                self.next_packet += self.recovery
+                return self.items.pop()
+        else:
+            return None
+
+    def __len__(self):
+        return len(self.items)
+
+    def clear(self):
+        self.items = []
+
+    def is_empty(self):
+        return len(self.items) == 0

core/aochat/extended_message.py

@@ -0,0 +1,20 @@
+class ExtendedMessage:
+    def __init__(self, category_id, instance_id, template, params):
+        self.category_id = category_id
+        self.instance_id = instance_id
+        self.template = template
+        self.params = params
+
+    def get_message(self):
+        try:
+            return self.template % tuple(self.params)
+        except TypeError:
+            # sometimes params are sent even tho the template does not include param placeholders
+            # ex: ExtendedMessage:
+            # [20000, 134870373,
+            # 'Your ability to send private messages has been revoked temporarily with a GM gag.',
+            # [1000]]
+            return self.template
+
+    def __str__(self):
+        return str([self.category_id, self.instance_id, self.template, self.params, self.get_message()])

core/aochat/mmdb_parser.py

@@ -0,0 +1,137 @@
+import struct
+
+from core.logger import Logger
+
+
+class MMDBParser:
+    def __init__(self, filename):
+        self.filename = filename
+        self.logger = Logger(__name__)
+
+    def get_message_string(self, category_id, instance_id):
+        with open(self.filename, mode="rb") as file:
+            categories = self.get_categories(file)
+
+            try:
+                category = next(categories)
+                while category["id"] != category_id:
+                    category = next(categories)
+                next_category = next(categories)
+            except StopIteration:
+                return None
+
+            instance = self.find_entry(file, instance_id, category["offset"], next_category["offset"])
+
+            if instance:
+                file.seek(instance["offset"])
+                return self.read_string(file)
+            else:
+                return None
+
+    def get_all_message_strings(self):
+        with open(self.filename, mode="rb") as file:
+            categories = iter(list(self.get_categories(file)))
+            next_category = next(categories)
+
+            while True:
+                try:
+                    category = next_category
+                    next_category = next(categories)
+                except StopIteration:
+                    break
+
+                max_offset = next_category["offset"]
+                file.seek(category["offset"])
+
+                instances = []
+                while file.tell() < max_offset:
+                    entry = self.read_entry(file)
+                    instances.append(entry)
+
+                for instance in instances:
+                    file.seek(instance["offset"])
+                    message_string = self.read_string(file)
+                    print([category["id"], instance["id"], message_string])
+
+    def find_entry(self, file, entry_id, min_offset, max_offset):
+        file.seek(min_offset)
+        entry = self.read_entry(file)
+        while file.tell() <= max_offset:
+            if entry["id"] == entry_id:
+                return entry
+            entry = self.read_entry(file)
+
+        return None
+
+    def get_categories(self, file):
+        file.seek(4)
+        num_categories = self.read_int(file)
+        for i in range(0, num_categories):
+            yield self.read_entry(file)
+
+    def read_entry(self, file):
+        return {"id": self.read_int(file), "offset": self.read_int(file)}
+
+    def read_int(self, file):
+        return int.from_bytes(file.read(4), byteorder="little")
+
+    def read_string(self, file):
+        message = bytearray()
+        char = file.read(1)
+        i = 0
+        while char and char != b'\x00':
+            i += 1
+            message.append(ord(char))
+            char = file.read(1)
+
+        return message.decode("utf-8")
+
+    def read_base_85(self, num_str):
+        n = 0
+        for i in range(0, 5):
+            n = n * 85 + num_str[i] - 33
+        return n
+
+    def parse_params(self, param_arr):
+        args = []
+        while len(param_arr) > 0:
+            data_type = chr(param_arr[0])
+            param_arr = param_arr[1:]
+            if data_type == "S":
+                size = param_arr[0] * 256 + param_arr[1]
+                args.append(param_arr[2:2 + size].decode("utf-8"))
+                param_arr = param_arr[2 + size:]
+            elif data_type == "s":
+                size = param_arr[0] - 1  # size is 1 less than indicated
+                args.append(param_arr[1:1 + size].decode("utf-8"))
+                param_arr = param_arr[1 + size:]
+            elif data_type == "I":
+                args.append(struct.unpack(">I", param_arr[:4])[0])
+                param_arr = param_arr[4:]
+            elif data_type == "i" or data_type == "u":
+                args.append(self.read_base_85(param_arr[:5]))
+                param_arr = param_arr[5:]
+            elif data_type == "R":
+                category_id = self.read_base_85(param_arr[:5])
+                instance_id = self.read_base_85(param_arr[5:10])
+                message = self.get_message_string(category_id, instance_id)
+                if not message:
+                    raise Exception(f"Could not find message string for category "
+                                    f"'{category_id}' and instance '{instance_id}'")
+                args.append(message)
+                param_arr = param_arr[10:]
+            elif data_type == "l":
+                category_id = 20000
+                instance_id = struct.unpack(">I", param_arr[:4])[0]
+                message = self.get_message_string(category_id, instance_id)
+                if not message:
+                    raise Exception(f"Could not find message string for category "
+                                    f"'{category_id}' and instance '{instance_id}'")
+                args.append(message)
+                param_arr = param_arr[4:]
+            elif data_type == "~":
+                break
+            else:
+                raise Exception("Unknown argument type '%s'" % data_type)
+
+        return args

core/aochat/packets.py

@@ -0,0 +1,95 @@
+import struct
+
+
+class UnknownArgumentType(Exception):
+    pass
+
+
+class PacketMissingArgument(Exception):
+    pass
+
+
+def decode_args(types, data):
+    args = []
+    for argtype in types:
+        if argtype == "I":
+            elem, data = data[:4], data[4:]
+            result = struct.unpack(">I", elem)[0]
+
+        elif argtype == "S":
+            length = struct.unpack(">H", data[:2])[0]
+            result = data[2:2 + length].decode("utf-8", "ignore")
+            data = data[2 + length:]
+
+        elif argtype == "B":
+            length = struct.unpack(">H", data[:2])[0]
+            result = data[2:2 + length]
+            data = data[2 + length:]
+
+        elif argtype == "G":
+            result, data = data[:5], data[5:]
+            # Convert result (5 bytes) to a long.  Can't use
+            # struct.unpack(">Q", "\x00"*3 + result), since we
+            # can't rely on "long long" being available.
+            high, low = struct.unpack(">BI", result)
+            result = (high << 32) + low
+
+        elif argtype == "i":
+            length = struct.unpack(">H", data[:2])[0]
+            result = struct.unpack(">%sI" % length, data[2:2 + 4 * length])
+            data = data[2 + 4 * length:]
+
+        elif argtype == "s":
+            length = struct.unpack(">H", data[:2])[0]
+            data = data[2:]
+            result = []
+            while length:
+                slength = struct.unpack(">H", data[:2])[0]
+                result.append(data[2:2 + slength].decode("utf-8"))
+                data = data[2 + slength:]
+                length -= 1
+
+        else:
+            raise UnknownArgumentType(argtype)
+
+        args.append(result)
+
+    return args
+
+
+def encode_args(types, args):
+    data = b""
+
+    for argtype in types:
+        if not args:
+            raise PacketMissingArgument
+
+        it = args[0]
+        del args[0]
+
+        if argtype == "I":
+            data += struct.pack(">I", it)
+
+        elif argtype == "S":
+            encoded = it.encode("utf-8")
+            data += struct.pack(">H", len(encoded))
+            data += encoded
+
+        elif argtype == "G":
+            data += struct.pack(">BI", it >> 32, it & 0xffffffff)
+
+        elif argtype == "s":
+            data += struct.pack(">H", len(it))
+            for it_elem in it:
+                encoded = it_elem.encode("utf-8")
+                data += struct.pack(">H", len(encoded))
+                data += encoded
+
+        else:
+            raise UnknownArgumentType(argtype)
+
+    return data
+
+
+class Packet:
+    pass

core/aochat/server_packets.py

@@ -0,0 +1,456 @@
+from core.aochat.extended_message import ExtendedMessage
+from core.aochat.packets import *
+
+
+class ServerPacket(Packet):
+    def __init__(self, packet_id, types, args):
+        self.id = packet_id
+        self.types = types
+        self.args = args
+
+    def to_bytes(self):
+        return encode_args(self.types, self.args)
+
+    def __str__(self):
+        return "ServerPacket(%d): %s" % (self.id, self.args)
+
+    @classmethod
+    def get_instance(cls, packet_id, data):
+        if packet_id == LoginSeed.id:
+            return LoginSeed.from_bytes(data)
+        elif packet_id == LoginOK.id:
+            return LoginOK.from_bytes(data)
+        elif packet_id == LoginError.id:
+            return LoginError.from_bytes(data)
+        elif packet_id == LoginCharacterList.id:
+            return LoginCharacterList.from_bytes(data)
+        elif packet_id == CharacterUnknown.id:
+            return CharacterUnknown.from_bytes(data)
+        elif packet_id == CharacterName.id:
+            return CharacterName.from_bytes(data)
+        elif packet_id == CharacterLookup.id:
+            return CharacterLookup.from_bytes(data)
+        elif packet_id == PrivateMessage.id:
+            return PrivateMessage.from_bytes(data)
+        elif packet_id == VicinityMessage.id:
+            return VicinityMessage.from_bytes(data)
+        elif packet_id == BroadcastMessage.id:
+            return BroadcastMessage.from_bytes(data)
+        elif packet_id == SimpleSystemMessage.id:
+            return SimpleSystemMessage.from_bytes(data)
+        elif packet_id == SystemMessage.id:
+            return SystemMessage.from_bytes(data)
+        elif packet_id == BuddyAdded.id:
+            return BuddyAdded.from_bytes(data)
+        elif packet_id == BuddyRemoved.id:
+            return BuddyRemoved.from_bytes(data)
+        elif packet_id == PrivateChannelInvited.id:
+            return PrivateChannelInvited.from_bytes(data)
+        elif packet_id == PrivateChannelKicked.id:
+            return PrivateChannelKicked.from_bytes(data)
+        elif packet_id == PrivateChannelLeft.id:
+            return PrivateChannelLeft.from_bytes(data)
+        elif packet_id == PrivateChannelClientJoined.id:
+            return PrivateChannelClientJoined.from_bytes(data)
+        elif packet_id == PrivateChannelClientLeft.id:
+            return PrivateChannelClientLeft.from_bytes(data)
+        elif packet_id == PrivateChannelMessage.id:
+            return PrivateChannelMessage.from_bytes(data)
+        elif packet_id == PrivateChannelInviteRefused.id:
+            return PrivateChannelInviteRefused.from_bytes(data)
+        elif packet_id == PublicChannelJoined.id:
+            return PublicChannelJoined.from_bytes(data)
+        elif packet_id == PublicChannelLeft.id:
+            return PublicChannelLeft.from_bytes(data)
+        elif packet_id == PublicChannelMessage.id:
+            return PublicChannelMessage.from_bytes(data)
+        elif packet_id == Pong.id:
+            return Pong.from_bytes(data)
+        else:
+            return None
+
+
+class LoginSeed(ServerPacket):
+    id = 0
+    types = "S"
+
+    def __init__(self, seed):
+        self.seed = seed
+        super().__init__(self.id, self.types, [self.seed])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class LoginOK(ServerPacket):
+    id = 5
+    types = ""
+
+    def __init__(self):
+        super().__init__(self.id, self.types, [])
+
+    @classmethod
+    def from_bytes(cls, data):
+        return cls()
+
+
+class LoginError(ServerPacket):
+    id = 6
+    types = "S"
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.id, self.types, [self.message])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class LoginCharacterList(ServerPacket):
+    id = 7
+    types = "isii"
+
+    def __init__(self, char_ids, names, levels, online_statuses):
+        self.char_ids = char_ids
+        self.names = names
+        self.levels = levels
+        self.online_statuses = online_statuses
+        super().__init__(self.id, self.types, [self.char_ids, self.names, self.levels, self.online_statuses])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class CharacterUnknown(ServerPacket):
+    id = 10
+    types = "I"
+
+    def __init__(self, char_id):
+        self.char_id = char_id
+        super().__init__(self.id, self.types, [self.char_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class CharacterName(ServerPacket):
+    id = 20
+    types = "IS"
+
+    def __init__(self, char_id, name):
+        self.char_id = char_id
+        self.name = name
+        super().__init__(self.id, self.types, [self.char_id, self.name])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class CharacterLookup(ServerPacket):
+    id = 21
+    types = "IS"
+
+    def __init__(self, char_id, name):
+        self.char_id = char_id
+        self.name = name
+        super().__init__(self.id, self.types, [self.char_id, self.name])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateMessage(ServerPacket):
+    id = 30
+    types = "ISS"
+
+    def __init__(self, char_id, message, blob):
+        self.char_id = char_id
+        self.message = message
+        self.blob = blob
+        super().__init__(self.id, self.types, [self.char_id, self.message, self.blob])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class VicinityMessage(ServerPacket):
+    id = 34
+    types = "ISS"
+
+    def __init__(self, char_id, message, blob):
+        self.char_id = char_id
+        self.message = message
+        self.blob = blob
+        super().__init__(self.id, self.types, [self.char_id, self.message, self.blob])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class BroadcastMessage(ServerPacket):
+    id = 35
+    types = "SSS"
+
+    def __init__(self, text, message, blob):
+        self.text = text
+        self.message = message
+        self.blob = blob
+        super().__init__(self.id, self.types, [self.text, self.message, self.blob])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class SimpleSystemMessage(ServerPacket):
+    id = 36
+    types = "S"
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.id, self.types, [self.message])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class SystemMessage(ServerPacket):
+    id = 37
+    types = "IIIB"
+
+    def __init__(self, client_id, window_id, message_id, message_args):
+        self.client_id = client_id
+        self.window_id = window_id
+        self.message_id = message_id
+        self.message_args = message_args
+        # noinspection PyTypeChecker
+        self.extended_message: ExtendedMessage = None
+        super().__init__(self.id, self.types, [self.client_id, self.window_id, self.message_id, self.message_args])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+    def __str__(self):
+        return super().__str__() + ", ExtendedMessage: %s" % self.extended_message
+
+
+class BuddyAdded(ServerPacket):
+    id = 40
+    types = "IIS"
+
+    def __init__(self, char_id, online, status):
+        self.char_id = char_id
+        self.online = online
+        self.status = status
+        super().__init__(self.id, self.types, [self.char_id, self.online, self.status])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class BuddyRemoved(ServerPacket):
+    id = 41
+    types = "I"
+
+    def __init__(self, char_id):
+        self.char_id = char_id
+        super().__init__(self.id, self.types, [self.char_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelInvited(ServerPacket):
+    id = 50
+    types = "I"
+
+    def __init__(self, private_channel_id):
+        self.private_channel_id = private_channel_id
+        super().__init__(self.id, self.types, [self.private_channel_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelKicked(ServerPacket):
+    id = 51
+    types = "I"
+
+    def __init__(self, private_channel_id):
+        self.private_channel_id = private_channel_id
+        super().__init__(self.id, self.types, [self.private_channel_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelLeft(ServerPacket):
+    id = 53
+    types = "I"
+
+    def __init__(self, private_channel_id):
+        self.private_channel_id = private_channel_id
+        super().__init__(self.id, self.types, [self.private_channel_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelClientJoined(ServerPacket):
+    id = 55
+    types = "II"
+
+    def __init__(self, private_channel_id, char_id):
+        self.private_channel_id = private_channel_id
+        self.char_id = char_id
+        super().__init__(self.id, self.types, [self.private_channel_id, self.char_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelClientLeft(ServerPacket):
+    id = 56
+    types = "II"
+
+    def __init__(self, private_channel_id, char_id):
+        self.private_channel_id = private_channel_id
+        self.char_id = char_id
+        super().__init__(self.id, self.types, [self.private_channel_id, self.char_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelMessage(ServerPacket):
+    id = 57
+    types = "IISS"
+
+    def __init__(self, private_channel_id, char_id, message, blob):
+        self.private_channel_id = private_channel_id
+        self.char_id = char_id
+        self.message = message
+        self.blob = blob
+        super().__init__(self.id, self.types, [self.private_channel_id, self.char_id, self.message, self.blob])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PrivateChannelInviteRefused(ServerPacket):
+    id = 58
+    types = "II"
+
+    def __init__(self, private_channel_id, char_id):
+        self.private_channel_id = private_channel_id
+        self.char_id = char_id
+        super().__init__(self.id, self.types, [self.private_channel_id, self.char_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PublicChannelJoined(ServerPacket):
+    id = 60
+    types = "GSIS"
+
+    def __init__(self, channel_id, name, unknown, flags):
+        self.channel_id = channel_id
+        self.name = name
+        self.unknown = unknown
+        self.flags = flags
+        super().__init__(self.id, self.types, [self.channel_id, self.name, self.unknown, self.flags])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PublicChannelLeft(ServerPacket):
+    id = 61
+    types = "G"
+
+    def __init__(self, channel_id):
+        self.channel_id = channel_id
+        super().__init__(self.id, self.types, [self.channel_id])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+
+class PublicChannelMessage(ServerPacket):
+    id = 65
+    types = "GISS"
+
+    def __init__(self, channel_id, char_id, message, blob):
+        self.channel_id = channel_id
+        self.char_id = char_id
+        self.message = message
+        self.blob = blob
+        # noinspection PyTypeChecker
+        self.extended_message: ExtendedMessage = None
+        super().__init__(self.id, self.types, [self.channel_id, self.char_id, self.message, self.blob])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)
+
+    def __str__(self):
+        return super().__str__() + ", ExtendedMessage: %s" % self.extended_message
+
+
+class Pong(ServerPacket):
+    id = 100
+    types = "S"
+
+    def __init__(self, blob):
+        self.blob = blob
+        super().__init__(self.id, self.types, [self.blob])
+
+    @classmethod
+    def from_bytes(cls, data):
+        args = decode_args(cls.types, data)
+        return cls(*args)

core/bot_status.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class BotStatus(Enum):
+    SHUTDOWN = 0
+    RUN = 1
+    RESTART = 2
+    ERROR = 4

core/buddy_service.py

@@ -0,0 +1,149 @@
+from core.aochat import client_packets
+from core.aochat import server_packets
+from core.conn import Conn
+from core.decorators import instance
+from core.logger import Logger
+from core.lookup.character_service import CharacterService
+
+
+@instance()
+class BuddyService:
+    BUDDY_LOGON_EVENT = "buddy_logon"
+    BUDDY_LOGOFF_EVENT = "buddy_logoff"
+
+    def __init__(self):
+        self.buddy_list = {}
+        self.buddy_list_size = 0
+        self.logger = Logger(__name__)
+
+    def inject(self, registry):
+        self.character_service: CharacterService = registry.get_instance("character_service")
+        self.bot = registry.get_instance("bot")
+        self.event_service = registry.get_instance("event_service")
+
+    def pre_start(self):
+        self.bot.register_packet_handler(server_packets.BuddyAdded.id, self.handle_add)
+        self.bot.register_packet_handler(server_packets.BuddyRemoved.id, self.handle_remove)
+        self.bot.register_packet_handler(server_packets.LoginOK.id, self.handle_login_ok)
+        self.event_service.register_event_type(self.BUDDY_LOGON_EVENT)
+        self.event_service.register_event_type(self.BUDDY_LOGOFF_EVENT)
+
+    def handle_add(self, conn: Conn, packet):
+        buddy = self.buddy_list[conn.id].get(packet.char_id, {"types": [], "conn_id": conn.id})
+        buddy["online"] = packet.online
+        self.buddy_list[conn.id][packet.char_id] = buddy
+
+        # verify that buddy does not exist on any other conn
+        for conn_id, conn_buddy_list in self.buddy_list.items():
+            if conn.id != conn_id:
+                buddy = conn_buddy_list.get(packet.char_id, None)
+                if buddy:
+                    if buddy["online"] is None:
+                        # remove from other conn list
+                        del conn_buddy_list[packet.char_id]
+                    else:
+                        # remove from this conn
+                        self.logger.warning(f"Removing char '{packet.char_id}' from conn '{conn.id}' "
+                                            f"since it already exists on another conn")
+                        conn.send_packet(client_packets.BuddyRemove(packet.char_id))
+
+        if packet.online == 1:
+            self.event_service.fire_event(self.BUDDY_LOGON_EVENT, packet)
+        else:
+            self.event_service.fire_event(self.BUDDY_LOGOFF_EVENT, packet)
+
+    def handle_remove(self, conn: Conn, packet):
+        conn_buddy_list = self.buddy_list[conn.id]
+        if packet.char_id in conn_buddy_list:
+            if len(conn_buddy_list[packet.char_id]["types"]) > 0:
+                self.logger.warning(f"Removing buddy {packet.char_id} that still has "
+                                    f"types {conn_buddy_list[packet.char_id]['types']}")
+
+            del conn_buddy_list[packet.char_id]
+
+    def handle_login_ok(self, conn: Conn, _):
+        self.buddy_list_size += 1000
+        self.buddy_list[conn.id] = {}
+        self.buddy_list[conn.id][conn.char_id] = {"online": True, "types": [], "conn_id": conn.id}
+
+    def add_buddy(self, char_id, _type):
+        if not char_id:
+            return False
+
+        # check if we are trying to add a conn as a buddy
+        if self.is_conn_char_id(char_id):
+            return False
+
+        buddy = self.get_buddy(char_id)
+        if buddy:
+            buddy["types"].append(_type)
+        else:
+            conn = self.get_conn_for_new_buddy()
+            if conn.char_id != char_id:
+                conn.send_packet(client_packets.BuddyAdd(char_id, "\1"))
+                self.buddy_list[conn.id][char_id] = {"online": None, "types": [_type], "conn_id": conn.id}
+
+        return True
+
+    def is_conn_char_id(self, char_id):
+        for _id, conn in self.bot.conns.items():
+            if conn.char_id == char_id:
+                return True
+
+        return False
+
+    def remove_buddy(self, char_id, _type, force_remove=False):
+        if char_id:
+            buddy = self.get_buddy(char_id)
+            if not buddy:
+                return False
+
+            if _type in buddy["types"]:
+                buddy["types"].remove(_type)
+
+            if len(buddy["types"]) == 0 or force_remove:
+                if not self.is_conn_char_id(char_id):
+                    conn = self.bot.conns[buddy["conn_id"]]
+                    conn.send_packet(client_packets.BuddyRemove(char_id))
+
+            return True
+        else:
+            return False
+
+    def get_buddy(self, char_id):
+        for conn_id, conn_buddy_list in self.buddy_list.items():
+            if char_id in conn_buddy_list:
+                return conn_buddy_list[char_id]
+        return None
+
+    def is_online(self, char_id):
+        buddy = self.get_buddy(char_id)
+        if buddy is None:
+            return None
+        else:
+            return buddy.get("online", None)
+
+    def get_all_buddies(self):
+        result = {}
+        for conn_id, conn_buddy_list in self.buddy_list.items():
+            for char_id, buddy in conn_buddy_list.items():
+                result[char_id] = buddy
+
+        return result
+
+    def get_buddy_list_size(self):
+        count = 0
+        for conn_id, conn_buddy_list in self.buddy_list.items():
+            count += len(conn_buddy_list)
+
+        return count
+
+    def get_conn_for_new_buddy(self):
+        buddy_list_size = None
+        _id = None
+        for conn_id, conn_buddy_list in self.buddy_list.items():
+            if buddy_list_size is None or len(conn_buddy_list) < buddy_list_size:
+                buddy_list_size = len(conn_buddy_list)
+                _id = conn_id
+
+        return self.bot.conns.get(_id, None)

core/cache_service.py

@@ -0,0 +1,34 @@
+import os
+from pathlib import Path
+
+from core.decorators import instance
+from core.dict_object import DictObject
+from core.logger import Logger
+
+
+@instance()
+class CacheService:
+    CACHE_DIR = os.sep + os.path.join("data", "cache")
+
+    def __init__(self):
+        Path(os.getcwd() + self.CACHE_DIR).mkdir(parents=True, exist_ok=True)
+        self.logger = Logger(__name__)
+
+    def store(self, group, filename, contents):
+        base_path = os.getcwd() + self.CACHE_DIR + os.sep + group
+        Path(base_path).mkdir(exist_ok=True)
+
+        with open(base_path + os.sep + filename, mode="w", encoding="UTF-8") as f:
+            f.write(contents)
+
+    def retrieve(self, group, filename):
+        base_path = os.getcwd() + self.CACHE_DIR + os.sep + group
+
+        full_path = base_path + os.sep + filename
+
+        try:
+            with open(full_path, mode="r", encoding="UTF-8") as f:
+                last_modified = int(os.path.getmtime(full_path))
+                return DictObject({"data": f.read(), "last_modified": last_modified})
+        except FileNotFoundError:
+            return None

core/chat_blob.py

@@ -0,0 +1,19 @@
+class ChatBlob:
+    def __init__(self, title, msg):
+        self.title = title
+        self.msg = msg.strip("\n")
+        self.page_prefix = ""
+        self.page_postfix = ""
+
+    def __str__(self):
+        return f"ChatBlob('{self.title}', '{self.msg}')"
+
+    def __repr__(self):
+        return self.__str__()
+
+    def __eq__(self, obj):
+        return isinstance(obj, ChatBlob) and \
+               obj.title == self.title and \
+               obj.msg == self.msg and \
+               obj.page_prefix == self.page_prefix and \
+               obj.page_postfix == self.page_postfix

core/command_alias_service.py

@@ -0,0 +1,48 @@
+from core.decorators import instance
+from core.logger import Logger
+
+
+@instance()
+class CommandAliasService:
+    def __init__(self):
+        self.logger = Logger(__name__)
+
+    def inject(self, registry):
+        self.db = registry.get_instance("db")
+
+    def check_for_alias(self, command_str):
+        row = self.get_alias(command_str)
+        if row and row.enabled:
+            return row.command
+        else:
+            return None
+
+    def get_alias(self, alias):
+        return self.db.query_single("SELECT alias, command, enabled FROM command_alias WHERE alias = ?", [alias])
+
+    def add_alias(self, alias, command, force_enable=False):
+        """Call during start"""
+        row = self.get_alias(alias)
+        if row:
+            if row.enabled:
+                return False
+            elif force_enable:
+                self.db.exec("UPDATE command_alias SET command = ?, enabled = 1 WHERE alias = ?", [command, alias])
+            return True
+        else:
+            self.db.exec("INSERT INTO command_alias (alias, command, enabled) VALUES (?, ?, 1)", [alias, command])
+            return True
+
+    def remove_alias(self, alias):
+        row = self.get_alias(alias)
+        if row:
+            if row.enabled:
+                self.db.exec("UPDATE command_alias SET enabled = 0 WHERE alias = ?", [alias])
+                return True
+            else:
+                return False
+        else:
+            return False
+
+    def get_enabled_aliases(self):
+        return self.db.query("SELECT alias, command FROM command_alias WHERE enabled = 1 ORDER BY alias")

core/command_param_types.py

@@ -0,0 +1,362 @@
+import re
+
+from core.dict_object import DictObject
+from core.registry import Registry
+from core.sender_obj import SenderObj
+
+
+class CommandParam:
+    def get_regex(self):
+        pass
+
+    def get_name(self):
+        pass
+
+
+class Const(CommandParam):
+    def __init__(self, name, is_optional=False):
+        self.name = name
+        self.is_optional = is_optional
+        if " " in name:
+            raise Exception("One or more spaces found in command param '%s'." % name)
+
+    def get_regex(self):
+        regex = r"(\s+" + self.name + ")"
+        return regex + ("?" if self.is_optional else "")
+
+    def get_name(self):
+        if self.is_optional:
+            return "[" + self.name + "]"
+        else:
+            return self.name
+
+    def process_matches(self, params):
+        val = params.pop(0)
+        if val is None:
+            return None
+        else:
+            return val.lstrip()
+
+
+class Int(CommandParam):
+    def __init__(self, name, is_optional=False):
+        self.name = name
+        self.is_optional = is_optional
+        if " " in name:
+            raise Exception("One or more spaces found in command param '%s'." % name)
+
+    def get_regex(self):
+        regex = r"(\s+[0-9]+)"
+        return regex + ("?" if self.is_optional else "")
+
+    def get_name(self):
+        if self.is_optional:
+            return "<highlight>[%s]</highlight>" % self.name
+        else:
+            return "<highlight>%s</highlight>" % self.name
+
+    def process_matches(self, params):
+        val = params.pop(0)
+        if val is None:
+            return None
+        else:
+            return int(val.lstrip())
+
+
+class SignedInt(Int):
+    def __init__(self, name, is_optional=False):
+        super().__init__(name, is_optional)
+
+    def get_regex(self):
+        regex = r"(\s+\-?[0-9]+)"
+        return regex + ("?" if self.is_optional else "")
+
+
+class Decimal(CommandParam):
+    def __init__(self, name, is_optional=False):
+        self.name = name
+        self.is_optional = is_optional
+        if " " in name:
+            raise Exception("One or more spaces found in command param '%s'." % name)
+
+    def get_regex(self):
+        regex = r"(\s+[0-9]*\.?[0-9]+)"
+        return regex + ("?" if self.is_optional else "")
+
+    def get_name(self):
+        if self.is_optional:
+            return "<highlight>[%s]</highlight>" % self.name
+        else:
+            return "<highlight>%s</highlight>" % self.name
+
+    def process_matches(self, params):
+        val = params.pop(0)
+        if val is None:
+            return None
+        else:
+            return float(val.lstrip())
+
+
+class Any(CommandParam):
+    def __init__(self, name, is_optional=False, allowed_chars="."):
+        self.name = name
+        self.is_optional = is_optional
+        self.allowed_chars = allowed_chars
+        if " " in name:
+            raise Exception("One or more spaces found in command param '%s'." % name)
+
+    def get_regex(self):
+        regex = r"(\s+%s+?)" % self.allowed_chars
+        return regex + ("?" if self.is_optional else "")
+
+    def get_name(self):
+        if self.is_optional:
+            return "<highlight>[%s]</highlight>" % self.name
+        else:
+            return "<highlight>%s</highlight>" % self.name
+
+    def process_matches(self, params):
+        val = params.pop(0)
+        if val is None:
+            return None
+        else:
+            return val.lstrip()
+
+
+class Regex(CommandParam):
+    def __init__(self, name, regex, is_optional=False, num_groups=1):
+        self.name = name
+        self.regex = regex
+        self.is_optional = is_optional
+        self.num_groups = num_groups
+        if " " in name:
+            raise Exception("One or more spaces found in command param '%s'." % name)
+
+    def get_regex(self):
+        return self.regex
+
+    def get_name(self):
+        if self.is_optional:
+            return "<highlight>[%s]</highlight>" % self.name
+        else:
+            return "<highlight>%s</highlight>" % self.name
+
+    def process_matches(self, params):
+        p = []
+        for i in range(self.num_groups):
+            p.append(params.pop(0))
+        return p
+
+
+class Options(CommandParam):
+    def __init__(self, options, is_optional=False):
+        self.options = options
+        self.is_optional = is_optional
+        for name in options:
+            if " " in name:
+                raise Exception("One or more spaces found in command param option '%s'." % name)
+
+    def get_regex(self):
+        regex = r"(" + "|".join(map(lambda x: r"\s+" + re.escape(x), self.options)) + ")"
+        return regex + ("?" if self.is_optional else "")
+
+    def get_name(self):
+        if self.is_optional:
+            return "[" + "|".join(self.options) + "]"
+        else:
+            return "|".join(self.options)
+
+    def process_matches(self, params):
+        val = params.pop(0)
+        if val is None:
+            return None
+        else:
+            return val.lstrip()
+
+
+class Time(CommandParam):
+    def __init__(self, name, is_optional=False):
+        self.name = name
+        self.is_optional = is_optional
+        if " " in name:
+            raise Exception("One or more spaces found in command param '%s'." % name)
+
+    def get_regex(self):
+        regex = r"(\s+(([0-9]+)([a-z]+))+)"
+        return regex + ("?" if self.is_optional else "")
+
+    def get_name(self):
+        if self.is_optional:
+            return "<highlight>[%s]</highlight>" % self.name
+        else:
+            return "<highlight>%s</highlight>" % self.name
+
+    def process_matches(self, params):
+        budatime_str = params.pop(0)
+        params.pop(0)
+        params.pop(0)
+        params.pop(0)
+
+        if budatime_str is None:
+            return None
+        else:
+            util = Registry.get_instance("util")
+            return util.parse_time(budatime_str.lstrip())
+
+
+class Item(CommandParam):
+    def __init__(self, name, is_optional=False):
+        self.name = name
+        self.is_optional = is_optional
+        if " " in name:
+            raise Exception("One or more spaces found in command param '%s'." % name)
+
+    def get_regex(self):
+        regex = r"""(\s*<a href=["']itemref:\/\/(\d+)\/(\d+)\/(\d+)["']>(.+?)<\/a>)"""
+        return regex + ("?" if self.is_optional else "")
+
+    def get_name(self):
+        if self.is_optional:
+            return "<highlight>[%s]</highlight>" % self.name
+        else:
+            return "<highlight>%s</highlight>" % self.name
+
+    def process_matches(self, params):
+        if params.pop(0):
+            return DictObject({
+                "low_id": int(params.pop(0)),
+                "high_id": int(params.pop(0)),
+                "ql": int(params.pop(0)),
+                "name": params.pop(0)
+            })
+        else:
+            params.pop(0)
+            params.pop(0)
+            params.pop(0)
+            params.pop(0)
+            return None
+
+
+class Character(Any):
+    def __init__(self, name, is_optional=False):
+        super().__init__(name, is_optional)
+
+    def get_regex(self):
+        regex = r"(\s+[\d+a-z-]+)"
+        return regex + ("?" if self.is_optional else "")
+
+    def process_matches(self, params):
+        val = super().process_matches(params)
+
+        if val is None:
+            return None
+        else:
+            character_service = Registry.get_instance("character_service")
+            access_service = Registry.get_instance("access_service")
+            char_id = character_service.resolve_char_to_id(val)
+            if char_id is None:
+                return SenderObj(char_id, val.capitalize(), None)
+            else:
+                return SenderObj(char_id, val.capitalize(), access_service.get_access_level(char_id))
+
+
+# Note: NamedParameters should always go at the end of the command parameter list
+# Note: NamedParameters need to be validated manually to ensure they have valid values
+class NamedParameters(CommandParam):
+    def __init__(self, names):
+        self.names = names
+        for name in names:
+            if " " in name:
+                raise Exception("One or more spaces found in command named param option '%s'." % name)
+
+    def get_regex(self):
+        regex = "((" + "|".join(map(lambda x: r"\s+--%s=.+?" % x, self.names)) + ")*)"
+        return regex
+
+    def get_name(self):
+        return " ".join(map(lambda x: f"[--{x}=<highlight>{x}</highlight>]", self.names))
+
+    def process_matches(self, params):
+        v = params.pop(0)
+        params.pop(0)
+
+        regex = "^(" + "|".join(map(lambda x: r"(\s+--(%s)=(.+?))" % x, self.names)) + ")*$"
+        p = re.compile(regex)
+        results = p.findall(v)[0][1:]
+        values = DictObject()
+        for name in self.names:
+            values[name] = results[2]
+            results = results[3:]
+        return values
+
+
+# Note: NamedFlagParameters should always go at the end of the command parameter list
+class NamedFlagParameters(CommandParam):
+    def __init__(self, names):
+        super().__init__()
+        self.names = names
+        for name in names:
+            if " " in name:
+                raise Exception("One or more spaces found in command named param option '%s'." % name)
+
+    def get_regex(self):
+        regex = "((" + "|".join(map(lambda x: r"\s+--%s" % x, self.names)) + ")*)"
+        return regex
+
+    def get_name(self):
+        return " ".join(map(lambda x: "[--%s]" % x, self.names))
+
+    def process_matches(self, params):
+        v = params.pop(0)
+        params.pop(0)
+
+        regex = "^(" + "|".join(map(lambda x: r"(\s+--(%s))" % x, self.names)) + ")*$"
+        p = re.compile(regex)
+        results = p.findall(v)[0][1:]
+        values = DictObject()
+        for name in self.names:
+            values[name] = True if results[1] else False
+            results = results[2:]
+        return values
+
+
+# Note: cannot be used with Any due to eagerness!
+class Multiple(CommandParam):
+    def __init__(self, inner_type, min_num=1, max_num=None):
+        if type(inner_type) is Any:
+            # Any type ignores is_optional and allowed_chars params, and can only capture
+            # single words (no spaces) when used with Multiple
+            def get_regex():
+                regex = r"(\s+[^ ]+)"
+                return regex
+
+            inner_type.get_regex = get_regex
+
+        self.inner_type = inner_type
+        self.min = min_num or ""
+        self.max = max_num or ""
+
+    def get_regex(self):
+        regex = "(" + self.inner_type.get_regex() + "{%s,%s})" % (self.min, self.max)
+        return regex
+
+    def get_name(self):
+        return self.inner_type.get_name() + "*"
+
+    def process_matches(self, params):
+        v = params.pop(0)
+
+        # remove unused params
+        self.inner_type.process_matches(params)
+
+        results = []
+        p = re.compile(self.inner_type.get_regex(), re.IGNORECASE | re.DOTALL)
+
+        matches = p.search(v)
+        while matches:
+            v = v[matches.end():]
+            a = self.inner_type.process_matches(list(matches.groups()))
+            results.append(a)
+            matches = p.search(v)
+
+        return results

core/command_request.py

@@ -0,0 +1,6 @@
+class CommandRequest:
+    def __init__(self, conn, channel, sender, reply):
+        self.conn = conn
+        self.channel = channel
+        self.sender = sender
+        self.reply = reply

core/command_service.py

@@ -0,0 +1,481 @@
+import collections
+import inspect
+import re
+from threading import Thread
+
+from core.aochat import server_packets
+from core.chat_blob import ChatBlob
+from core.command_request import CommandRequest
+from core.conn import Conn
+from core.decorators import instance
+from core.dict_object import DictObject
+from core.functions import flatmap, get_attrs
+from core.logger import Logger
+from core.lookup.character_service import CharacterService
+from core.message_hub_service import MessageHubService
+from core.registry import Registry
+from core.sender_obj import SenderObj
+from core.setting_service import SettingService
+from modules.core.accounting.services.access_service import AccessService
+
+
+# noinspection SqlResolve,PyAttributeOutsideInit,PyUnresolvedReferences,PyTypeChecker
+@instance()
+class CommandService:
+    PRIVATE_CHANNEL = "priv"
+    ORG_CHANNEL = "org"
+    PRIVATE_MESSAGE_CHANNEL = "msg"
+
+    def __init__(self):
+        self.ignore = []
+        self.handlers = collections.defaultdict(list)
+        self.logger = Logger(__name__)
+        self.channels = {}
+        self.pre_processors = []
+        self.ignore_regexes = [
+            re.compile(r" is AFK \(Away from keyboard\) since ", re.IGNORECASE),
+            re.compile(r"I am away from my keyboard right now", re.IGNORECASE),
+            re.compile(r"Unknown command or access denied!", re.IGNORECASE),
+            re.compile(r"I am responding", re.IGNORECASE),
+            re.compile(r"I only listen", re.IGNORECASE),
+            re.compile(r"Error!", re.IGNORECASE),
+            re.compile(r"Unknown command input", re.IGNORECASE),
+            re.compile(r"You have been auto invited", re.IGNORECASE),
+            re.compile(r"^<font")
+        ]
+
+    def inject(self, registry):
+        self.db = registry.get_instance("db")
+        self.util = registry.get_instance("util")
+        self.access_service: AccessService = registry.get_instance("access_service")
+        self.bot = registry.get_instance("bot")
+        self.character_service: CharacterService = registry.get_instance("character_service")
+        self.setting_service: SettingService = registry.get_instance("setting_service")
+        self.command_alias_service = registry.get_instance("command_alias_service")
+        self.usage_service = registry.get_instance("usage_service")
+        self.public_channel_service = registry.get_instance("public_channel_service")
+        self.ban_service = registry.get_instance("ban_service")
+        self.getresp = registry.get_instance("translation_service").get_response
+        self.relay_hub_service: MessageHubService = registry.get_instance("message_hub_service")
+
+    def pre_start(self):
+        self.bot.register_packet_handler(server_packets.PrivateMessage.id, self.handle_private_message)
+        self.bot.register_packet_handler(server_packets.PrivateChannelMessage.id, self.handle_private_channel_message)
+        # Org channel is not supported
+        self.bot.register_packet_handler(server_packets.PublicChannelMessage.id, self.handle_public_channel_message)
+        self.register_command_channel("Private Message", self.PRIVATE_MESSAGE_CHANNEL)
+        self.register_command_channel("Org Channel", self.ORG_CHANNEL)
+        self.register_command_channel("Private Channel", self.PRIVATE_CHANNEL)
+
+    def start(self):
+        access_levels = {}
+
+        # process decorators
+        for _, inst in Registry.get_all_instances().items():
+            for name, method in get_attrs(inst).items():
+                if hasattr(method, "command"):
+                    key = Registry.get_module_name(inst).split(".")
+                    if key[0] not in self.bot.modules:
+                        continue
+                    cmd_name, params, access_level, description, help_file, sub_command, extended_description = getattr(
+                        method, "command")
+                    handler = getattr(inst, name)
+                    help_text = self.get_help_file(inst.module_name, help_file)
+
+                    command_key = self.get_command_key(cmd_name.lower(), sub_command.lower() if sub_command else "")
+                    al = access_levels.get(command_key, None)
+                    if al is not None and al != access_level.lower():
+                        print(handler)
+                        raise Exception("Different access levels specified for forms of command '%s'" % command_key)
+                    access_levels[command_key] = access_level
+
+                    self.register(handler, cmd_name, params, access_level, description, inst.module_name, help_text,
+                                  sub_command, extended_description)
+
+    def register(self, handler, command, params, access_level, description, module, help_text=None, sub_command=None,
+                 extended_description=None, check_access=None):
+        """
+        Call during pre_start
+
+        Args:
+            handler: (request, param1, param2, ...) -> str|ChatBlob|None
+            command: str
+            params: [CommandParam...]
+            access_level: str
+            description: str
+            module: str
+            help_text: str
+            sub_command: str
+            extended_description: str
+            check_access: (char, access_level_label) -> bool
+        """
+
+        if len(inspect.signature(handler).parameters) != len(params) + 1:
+            raise Exception(
+                "Incorrect number of arguments for handler '%s.%s()'" % (handler.__module__, handler.__name__))
+
+        command = command.lower()
+        if sub_command:
+            sub_command = sub_command.lower()
+        else:
+            sub_command = ""
+        access_level = access_level.lower()
+        module = module.lower()
+        command_key = self.get_command_key(command, sub_command)
+
+        if help_text is None:
+            help_text = self.generate_help(command, description, params, extended_description)
+
+        if check_access is None:
+            check_access = self.access_service.check_access
+
+        if not self.access_service.get_access_level_by_label(access_level):
+            self.logger.error("Could not add command '%s': could not find access level '%s'" % (command, access_level))
+            return
+
+        for channel, label in self.channels.items():
+            row = self.db.query_single("SELECT access_level, module, enabled, verified "
+                                       "FROM command_config "
+                                       "WHERE command = ? AND sub_command = ? AND channel = ?",
+                                       [command, sub_command, channel])
+
+            if row is None:
+                # add new command
+                self.db.exec(
+                    "INSERT INTO command_config "
+                    "(command, sub_command, access_level, channel, module, enabled, verified) "
+                    "VALUES (?, ?, ?, ?, ?, 1, 1)",
+                    [command, sub_command, access_level, channel, module])
+            elif row.verified:
+                if row.module != module:
+                    self.logger.warning("module different for different forms of command '%s' and sub_command '%s'" % (
+                        command, sub_command))
+            else:
+                # mark command as verified
+                self.db.exec("UPDATE command_config SET verified = 1, module = ? "
+                             "WHERE command = ? AND sub_command = ? AND channel = ?",
+                             [module, command, sub_command, channel])
+
+        # save reference to command handler
+        r = re.compile(self.get_regex_from_params(params), re.IGNORECASE | re.DOTALL)
+        self.handlers[command_key].append(
+            {"regex": r, "callback": handler, "help": help_text, "description": description, "params": params,
+             "check_access": check_access})
+
+    def register_command_pre_processor(self, pre_processor):
+        """
+        Call during start
+
+        Args:
+            pre_processor: (context) -> bool
+        """
+
+        self.pre_processors.append(pre_processor)
+
+    def register_command_channel(self, label, value):
+        """
+        Call during pre_start
+
+        Args:
+            label: str
+            value: str
+        """
+
+        if value in self.channels:
+            self.logger.error("Could not register command channel '%s': command channel already registered" % value)
+            return
+
+        self.logger.debug("Registering command channel '%s'" % value)
+        self.channels[value] = label
+
+    def is_command_channel(self, channel):
+        return channel in self.channels
+
+    def process_command(self, message: str, channel: str, char_id, reply, conn, followup=True):
+        try:
+            context = DictObject({"message": message, "char_id": char_id, "channel": channel, "reply": reply})
+            for pre_processor in self.pre_processors:
+                if pre_processor(context) is False:
+                    return
+
+            for regex in self.ignore_regexes:
+                if regex.search(message):
+                    self.logger.info(f"Ignoring message from {char_id}: {message}")
+                    return
+            if not followup:
+                if channel == "msg":
+                    if message.lower().startswith("mail"):
+                        self.relay_hub_service.send_message("tell_logger", char_id, "mail |usage hidden|",
+                                                            "mail |usage hidden|")
+                    else:
+                        self.relay_hub_service.send_message("tell_logger", char_id, message, message)
+                else:
+                    if message.lower().startswith("mail"):
+                        self.relay_hub_service.send_message("dc_relay_log", char_id, "mail |usage hidden|",
+                                                            "mail |usage hidden|")
+                    else:
+                        self.relay_hub_service.send_message("dc_relay_log", char_id, message, message)
+            # message = html.unescape(message)
+            command_str, command_args = self.get_command_parts(message)
+
+            # check for command alias
+            command_alias = self.command_alias_service.check_for_alias(command_str)
+
+            alias_depth_count = 0
+            while command_alias:
+                alias_depth_count += 1
+                command_str, command_args = self.get_command_parts(command_alias + command_args)
+                command_alias = self.command_alias_service.check_for_alias(command_str)
+
+                if alias_depth_count > 20:
+                    raise Exception("Command alias infinite recursion detected for command '%s'" % message)
+
+            cmd_configs = self.get_command_configs(command_str, channel)
+            access_level = self.access_service.get_access_level(char_id)
+            sender = SenderObj(char_id, self.character_service.resolve_char_to_name(char_id, "Unknown(%d)" % char_id),
+                               access_level)
+            if cmd_configs:
+                # given a list of cmd_configs that are enabled, see if one has regex that matches incoming command_str
+                cmd_config, matches, handler = self.get_matches(cmd_configs, command_args)
+                if matches:
+                    if handler["check_access"](char_id, cmd_config.access_level):
+                        try:
+                            response = handler["callback"](CommandRequest(conn, channel, sender, reply),
+                                                           *self.process_matches(matches, handler["params"]))
+                            if response is not None:
+                                reply(response)
+                        except Exception as e:
+                            self.logger.error("error processing command: %s" % message, e)
+                            self.relay_hub_service.send_message("access_denied_logger", sender,
+                                                                f"[ERROR] {sender.name}: {message}",
+                                                                f"[ERROR] {sender.name}:  {message}")
+                            self.bot.send_mass_message(char_id, self.getresp("global", "error_processing"))
+
+                        # record command usage
+                        self.usage_service.add_usage(command_str, handler["callback"].__qualname__, char_id, channel)
+                    else:
+                        self.access_denied_response(message, sender, cmd_config, reply)
+                else:
+                    # handlers were found, but no handler regex matched
+                    help_text = self.get_help_text(char_id, command_str, channel)
+                    if help_text:
+                        reply(self.format_help_text(command_str, help_text))
+                    else:
+                        # the command is known, but no help is returned, therefore user does not have access to command
+                        if access_level['label'] != "all":
+                            self.relay_hub_service.send_message("access_denied_logger", sender,
+                                                                f"[DENIED] {sender.name}: {message}",
+                                                                f"[DENIED] {sender.name}:  {message}")
+                            self.bot.send_mass_message(char_id, self.getresp("global", "access_denied"))
+            else:
+                self.handle_unknown_command(command_str, command_args, channel, sender, reply)
+        except Exception as e:
+            self.logger.error("error processing command: %s" % message, e)
+            sender = SenderObj(char_id, self.character_service.resolve_char_to_name(char_id, "Unknown(%d)" % char_id),
+                               0)
+            self.relay_hub_service.send_message("access_denied_logger", sender, f"[ERROR] {sender.name}: {message}",
+                                                f"[ERROR] {sender.name}:  {message}")
+            self.bot.send_mass_message(char_id, self.getresp("global", "error_processing"))
+
+    def handle_unknown_command(self, command_str, command_args, channel, sender, reply):
+        self.relay_hub_service.send_message("access_denied_logger", sender,
+                                            f"[UNKNOWN] {sender.name}: {command_str} {command_args}",
+                                            f"[UNKNOWN] {sender.name}: {command_str} {command_args}")
+        if sender.access_level["label"] != "all":
+            self.bot.send_mass_message(sender.char_id, self.getresp("global", "unknown_command", {"cmd": command_str}))
+
+    def access_denied_response(self, message, sender, cmd_config, reply):
+        self.relay_hub_service.send_message("access_denied_logger", sender, f"[DENIED] {sender.name}: {message}",
+                                            f"[DENIED] {sender.name}:  {message}")
+        if sender.access_level["label"] != "all":
+            self.bot.send_mass_message(sender.char_id, self.getresp("global", "access_denied"))
+
+    def get_command_parts(self, message):
+        parts = message.split(" ", 1)
+        if len(parts) == 2:
+            return parts[0].lower(), " " + parts[1]
+        else:
+            return parts[0].lower(), ""
+
+    def get_command_configs(self, command, channel=None, enabled=1, sub_command=None):
+        sql = "SELECT command, sub_command, access_level, channel, enabled FROM command_config WHERE command = ?"
+        params = [command]
+        if channel:
+            sql += " AND channel = ?"
+            params.append(channel)
+        if enabled:
+            sql += " AND enabled = ?"
+            params.append(enabled)
+        if sub_command:
+            sql += " AND sub_command = ?"
+            params.append(sub_command)
+
+        sql += " ORDER BY sub_command, channel"
+
+        return self.db.query(sql, params)
+
+    def get_matches(self, cmd_configs, command_args):
+        for row in cmd_configs:
+            command_key = self.get_command_key(row.command, row.sub_command)
+            handlers = self.handlers[command_key]
+            for handler in handlers:
+                # add leading space to search string to normalize input for command params
+                matches = handler["regex"].search(command_args)
+                if matches:
+                    return row, matches, handler
+        return None, None, None
+
+    def process_matches(self, matches, params):
+        groups = list(matches.groups())
+
+        processed = []
+        for param in params:
+            processed.append(param.process_matches(groups))
+        return processed
+
+    def get_help_text(self, char, command_str, channel, show_regex=False):
+        data = self.db.query("SELECT command, sub_command, access_level FROM command_config "
+                             "WHERE command = ? AND channel = ? AND enabled = 1",
+                             [command_str, channel])
+
+        # filter out commands that character does not have access level for
+        data = filter(lambda row: self.access_service.check_access(char, row.access_level), data)
+
+        def get_regex(params):
+            if show_regex:
+                return "\n" + self.get_regex_from_params(params)
+            else:
+                return ""
+
+        def read_help_text(row):
+            command_key = self.get_command_key(row.command, row.sub_command)
+            return filter(lambda x: x is not None, map(lambda handler: handler["help"] + get_regex(handler["params"]),
+                                                       self.handlers[command_key]))
+
+        content = "\n\n".join(flatmap(read_help_text, data))
+        return content if content else None
+
+    def format_help_text(self, topic, help_text):
+        return ChatBlob("Help (" + topic + ")", help_text)
+
+    def get_help_file(self, module, help_file):
+        if help_file:
+            try:
+                help_file = "./" + module.replace(".", "/") + "/" + help_file
+                with open(help_file, mode="r", encoding="UTF-8") as f:
+                    return f.read().strip()
+            except FileNotFoundError as e:
+                self.logger.error("Error reading help file", e)
+        return None
+
+    def get_command_key(self, command, sub_command):
+        if sub_command:
+            return command + ":" + sub_command
+        else:
+            return command
+
+    def get_command_key_parts(self, command_str):
+        parts = command_str.split(":", 1)
+        if len(parts) == 2:
+            return parts[0], parts[1]
+        else:
+            return parts[0], ""
+
+    def get_regex_from_params(self, params):
+        # params must be wrapped with line-beginning and line-ending anchors in order to match
+        # when no params are specified (eg. "^$")
+        return "^" + "".join(map(lambda x: x.get_regex(), params)) + "$"
+
+    def generate_help(self, command, description, params, extended_description=None):
+        help_text = description + ":\n" + "<tab><symbol>" + command + " " + " ".join(
+            map(lambda x: x.get_name(), params))
+        if extended_description:
+            help_text += "\n" + extended_description
+
+        return help_text
+
+    def get_handlers(self, command_key):
+        return self.handlers.get(command_key, None)
+
+    def handle_private_message(self, conn: Conn, packet: server_packets.PrivateMessage):
+        if not self.bot.is_ready():
+            if packet.char_id not in self.ignore:
+                self.ignore.append(packet.char_id)
+                self.bot.send_mass_message(packet.char_id,
+                                           "Your command has not been proceeded, bot is still starting.")
+            return
+        if not self.setting_service.get("accept_commands_from_slave_bots").get_value() and conn.id != "main":
+            return
+
+        # since the command symbol is not required for private messages,
+        # the command_str must have length of at least 1 in order to be valid,
+        # otherwise it is ignored
+        if len(packet.message) < 1:
+            return
+
+        # ignore leading space
+        message = packet.message.lstrip()
+
+        def i():
+            self.process_command(
+                self.trim_command_symbol(message),
+                self.PRIVATE_MESSAGE_CHANNEL,
+                packet.char_id,
+                lambda msg: self.bot.send_private_message(packet.char_id, msg, conn_id=conn.id),
+                conn,
+                False)
+
+        t: Thread = Thread(target=i, daemon=True)
+        t.run()
+
+    def handle_private_channel_message(self, conn: Conn, packet: server_packets.PrivateChannelMessage):
+        if not self.setting_service.get("accept_commands_from_slave_bots").get_value() and conn.id != "main":
+            return
+
+        # since the command symbol is required in the private channel,
+        # the command_str must have length of at least 2 in order to be valid,
+        # otherwise it is ignored
+        if len(packet.message) < 2:
+            return
+
+        # ignore leading space
+        message = packet.message.lstrip()
+        if message.startswith(self.setting_service.get("symbol").get_value()) \
+                and packet.private_channel_id == self.bot.get_char_id():
+            Thread(target=self.process_command(
+                self.trim_command_symbol(message),
+                self.PRIVATE_CHANNEL,
+                packet.char_id,
+                lambda msg: self.bot.send_private_channel_message(msg, private_channel_id=conn.char_id,
+                                                                  conn_id=conn.id),
+                conn,
+                False), daemon=True).run()
+
+    def handle_public_channel_message(self, conn: Conn, packet: server_packets.PublicChannelMessage):
+        if not self.setting_service.get("accept_commands_from_slave_bots").get_value() and conn.id != "main":
+            return
+
+        # since the command symbol is required in the org channel,
+        # the command_str must have length of at least 2 in order to be valid,
+        # otherwise it is ignored
+        if len(packet.message) < 2:
+            return
+
+        # ignore leading space
+        message = packet.message.lstrip()
+
+        if message.startswith(self.setting_service.get("symbol").get_value()) \
+                and self.public_channel_service.is_org_channel_id(packet.channel_id):
+            Thread(target=self.process_command(
+                self.trim_command_symbol(message),
+                self.ORG_CHANNEL,
+                packet.char_id,
+                lambda msg: self.bot.send_org_message(msg, conn_id=conn.id),
+                conn,
+                False), daemon=True).run()
+
+    def trim_command_symbol(self, s):
+        symbol = self.setting_service.get("symbol").get_value()
+        if s.startswith(symbol):
+            s = s[len(symbol):]
+        return s

core/conn.py

@@ -0,0 +1,63 @@
+import threading
+import time
+
+from core.aochat.bot import Bot
+from core.aochat.client_packets import Ping
+from core.aochat.delay_queue import DelayQueue
+
+
+class Conn(Bot):
+    def __init__(self, _id, failure_callback):
+        super().__init__()
+        self.id = _id
+        self.packet_queue = DelayQueue(2, 2.5)
+        self.packet_last_received_timestamp = time.time()
+        self.failure_callback = failure_callback
+        self.send_lock = threading.Lock()
+
+    def read_packet(self, max_delay_time=1):
+        self.check_outgoing_message_queue()
+        packet = super().read_packet(max_delay_time)
+        if not packet:
+            time_since = time.time() - self.packet_last_received_timestamp
+            if time_since > 90:
+                self.logger.error(f"no packet received in 90 seconds for conn {self.id}")
+                self.failure_callback()
+            if time_since > 60:
+                self.send_packet(Ping("tyrbot_aochat"))
+        else:
+            self.packet_last_received_timestamp = time.time()
+        return packet
+
+    def send_packet(self, packet):
+        # synchronize sending packets
+        # noinspection PyBroadException
+        try:
+            with self.send_lock:
+                super().send_packet(packet)
+        except Exception as e:
+            self.failure_callback()
+
+    def add_packet_to_queue(self, packet):
+        self.packet_queue.enqueue(packet)
+        self.check_outgoing_message_queue()
+
+    def check_outgoing_message_queue(self):
+        # check packet queue for outgoing packets
+        outgoing_packet = self.packet_queue.dequeue()
+        while outgoing_packet:
+            self.send_packet(outgoing_packet)
+            outgoing_packet = self.packet_queue.dequeue()
+
+        num_messages = len(self.packet_queue)
+        if num_messages > 30:
+            self.logger.warning("automatically clearing outgoing message queue (%d messages)" % num_messages)
+            self.packet_queue.clear()
+        elif num_messages > 10:
+            self.logger.warning("%d messages in outgoing message queue" % num_messages)
+
+    def __str__(self):
+        return self.id
+
+    def __repr__(self):
+        return self.__str__()

core/db.py

@@ -0,0 +1,278 @@
+import os
+import re
+import sys
+import threading
+import time
+
+import mariadb
+# noinspection PyProtectedMember
+from mariadb._mariadb import ConnectionPool, OperationalError
+from mysql.connector.cursor import CursorBase
+from pkg_resources import parse_version
+
+from conf.config import BotConfig
+from core.decorators import instance
+from core.dict_object import DictObject
+from core.logger import Logger
+
+
+@instance()
+class DB:
+    MYSQL = "mysql"
+    MARIADB = "mariadb-pool"
+
+    def __init__(self):
+        self.pool_size = 4
+        # noinspection PyTypeChecker
+        self.pool: ConnectionPool = None
+        self.enhanced_like_regex = re.compile(r"(\s+)(\S+)\s+<EXTENDED_LIKE=(\d+)>\s+\?(\s*)", re.IGNORECASE)
+        self.lastrowid = None
+        self.logger = Logger(__name__)
+        self.type = None
+        self.lock = threading.Semaphore(self.pool_size)
+        self.transaction_level = 0
+        # noinspection PyTypeChecker
+        self.shared: DB = None
+        mod = __import__(f'conf.{sys.argv[1]}', fromlist=['BotConfig'])
+        config: BotConfig = getattr(mod, 'BotConfig')
+        self.name = config.character
+
+    def connect_mariadb(self, host, port, username, password, database_name):
+        self.type = self.MARIADB
+        self.connect_detail = {'host': host, 'port': port, 'user': username,
+                               'password': password, 'database': database_name, 'autocommit': True}
+        self.pool = mariadb.ConnectionPool(pool_name=database_name, pool_size=self.pool_size,
+                                           pool_reset_connection=False,
+                                           host=host, port=port, user=username, password=password,
+                                           database=database_name, autocommit=True)
+        self.exec("SET collation_connection = 'utf8_general_ci'")
+        self.exec("SET sql_mode = 'TRADITIONAL,ANSI'")
+        self.create_db_version_table()
+
+    def create_db_version_table(self):
+        self.exec("CREATE TABLE IF NOT EXISTS db_version ("
+                  "file VARCHAR(255) NOT NULL, "
+                  "version VARCHAR(255) NOT NULL, "
+                  "verified SMALLINT NOT NULL, "
+                  "bot varchar(32))")
+
+    def _execute_wrapper(self, sql, params, callback):
+        with self.lock:
+            start_time = time.time()
+            if self.pool.pool_size < self.pool_size - 1:
+                self.pool.add_connection(mariadb.connect(**self.connect_detail))
+
+            with self.pool.get_connection() as conn:
+                conn.auto_reconnect = True
+                conn.autocommit = True
+                with conn.cursor(dictionary=True) as cur:
+                    try:
+                        string: str = sql.upper()
+
+                        if string.__contains__("UPDATE ") or string.__contains__("INSERT "):
+                            cur.execute("START TRANSACTION;")
+                        cur.execute(sql.replace("?", "%s"), params)
+                        if string.__contains__("UPDATE ") or string.__contains__("INSERT "):
+                            conn.commit()
+                    except Exception as e:
+                        raise SqlException("SQL Error: '%s' for '%s' [%s]" % (
+                            str(e), sql, ", ".join(map(lambda x: str(x), params)))) from e
+                    elapsed = time.time() - start_time
+                    result = callback(cur)
+            if elapsed > 5:
+                self.logger.warning("slow query (%fs) '%s' for params: %s" % (elapsed, sql, str(params)))
+
+        return result
+
+    def query_single(self, sql, params=None, extended_like=False) -> DictObject:
+        if params is None:
+            params = []
+
+        if extended_like:
+            sql, params = self.handle_extended_like(sql, params)
+
+        sql, params = self.format_sql(sql, params)
+
+        def map_result(cur):
+            row = cur.fetchone()
+            return DictObject(row) if row else None
+
+        return self._execute_wrapper(sql, params, map_result)
+
+    def query(self, sql, params=None, extended_like=False) -> list[DictObject]:
+        if params is None:
+            params = []
+
+        if extended_like:
+            sql, params = self.handle_extended_like(sql, params)
+
+        sql, params = self.format_sql(sql, params)
+
+        def map_result(cur):
+            return list(map(lambda row: DictObject(row), cur.fetchall()))
+
+        return self._execute_wrapper(sql, params, map_result)
+
+    def exec(self, sql, params=None, extended_like=False) -> int:
+        if params is None:
+            params = []
+
+        if extended_like:
+            sql, params = self.handle_extended_like(sql, params)
+
+        sql, params = self.format_sql(sql, params)
+
+        def map_result(cur):
+            return [cur.rowcount, cur.lastrowid]
+
+        row_count, lastrowid = self._execute_wrapper(sql, params, map_result)
+        self.lastrowid = lastrowid
+        return row_count
+
+    def last_insert_id(self) -> int:
+        return self.lastrowid
+
+    def format_sql(self, sql, params=None) -> [str, list]:
+        return sql, params
+
+    def handle_extended_like(self, sql, params):
+        original_params = params.copy()
+        params = list(map(lambda x: [x], params))
+
+        for match in self.enhanced_like_regex.finditer(sql):
+            field = match.group(2)
+            index = int(match.group(3))
+            extra_sql, vals = self._get_extended_params(field, original_params[index].split(" "))
+
+            sql = self.enhanced_like_regex.sub(match.group(1) + "(" + " AND ".join(extra_sql) + ")" + match.group(4),
+                                               sql, 1)
+            # remove current param and add generated params in its place
+            del params[index]
+            params.insert(index, vals)
+        return sql, [item for sublist in params for item in sublist]
+
+    def _get_extended_params(self, field, params) -> [str, list]:
+        extra_sql = []
+        vals = []
+        for p in params:
+            if p.startswith("-") and p != "-":
+                vals.append("%" + p[1:] + "%")
+                extra_sql.append(field + " NOT LIKE ?")
+            else:
+                vals.append("%" + p + "%")
+                extra_sql.append(field + " LIKE ?")
+        return extra_sql, vals
+
+    def create_view(self, table) -> None:
+        if self.shared == self:
+            return
+        self.exec(f"DROP TABLE if exists {table};")
+        self.exec(f"CREATE OR REPLACE SQL SECURITY INVOKER VIEW {table} AS "
+                  f"SELECT * FROM `{self.shared.pool.pool_name}`.{table};")
+
+    def load_sql_file(self, sql_file: str, force_update=False, per_bot=False, pre_optimized=False) -> None:
+        filename = sql_file.replace("\\", "/")
+        bot = "global"
+        if per_bot:
+            bot = self.name
+        db_version = self.shared.get_db_version(filename, bot)
+        file_version = self.get_file_version(filename)
+        if db_version:
+            if parse_version(file_version) > parse_version(db_version) or force_update:
+                self.logger.debug("loading sql file '%s'" % sql_file)
+                self._load_file(filename, pre_optimized)
+                self.exec("UPDATE db_version SET version = ?, verified = 1 WHERE file = ? and bot = ?",
+                          [int(file_version), filename, bot])
+        else:
+            self.logger.debug("loading sql file '%s'" % sql_file)
+            self._load_file(filename, pre_optimized)
+            self.exec("INSERT INTO db_version (file, version, bot, verified) VALUES (?, ?, ?, 1)",
+                      [filename, int(file_version), bot])
+
+    def get_file_version(self, filename) -> str:
+        return str(int(os.path.getmtime(filename)))
+
+    def get_db_version(self, filename, bot) -> int or None:
+
+        row = self.query_single("SELECT version FROM db_version WHERE file = ? and bot = ?", [filename, bot])
+        if row:
+            return row.version
+        else:
+            return None
+
+    def _load_optimized_file(self, filename):
+        start = time.time()
+        with open(filename, mode="r", encoding="UTF-8") as f:
+            with self.shared.pool.get_connection() as conn:
+                with conn.cursor() as cur:
+                    for line in f.readlines():
+                        line = line.strip()
+
+                        if line != "":
+                            if line.startswith("#") or line.startswith("--"):
+                                continue
+                            cur.execute(line)
+        print(f"Runtime: {time.time() - start: .2f} for {filename}")
+
+    def _load_file(self, filename, pre_optimized=False) -> None:
+        if pre_optimized:
+            self._load_optimized_file(filename)
+            return
+        start = time.time()
+        # Short version... instead of executing 90 000 inserts for the itemDB, just do one,
+        # while providing all values during one query
+        with open(filename, mode="r", encoding="UTF-8") as f:
+            insert_batches = []
+            inserts = []
+            others = []
+            stat = ""
+            for i in f.readlines():
+                i = i.strip()
+                if i == "" or i == " ":
+                    continue
+                if i.startswith("INSERT INTO"):
+                    match2 = re.match("(INSERT INTO .+? VALUES) (\(.+?\));", i)
+                    if match2:
+                        r2 = match2[2].replace("NULL", "None")
+                        r2 = r2.replace("null", "None")
+                        query = match2[1] + f" ({', ?' * len(eval(r2))})"
+                        query = query.replace("(, ", "(")
+                        if stat != query:
+                            if stat != "" or len(inserts) != 0:
+                                insert_batches.append([stat, inserts])
+                            inserts = []
+                            stat = query
+                        inserts.append(eval(r2))
+                else:
+                    if i.startswith("--"):
+                        continue
+                    others.append(i)
+            insert_batches.append([stat, inserts])
+            with self.shared.lock:
+                with self.shared.pool.get_connection() as conn:
+                    with conn.cursor() as cur:
+                        cur: CursorBase
+                        if others:
+                            for statement in others:
+                                try:
+                                    cur.execute(statement)
+                                except OperationalError:
+                                    pass
+                            for sql, param in insert_batches:
+                                if sql == "INSERT INTO trickle (id, group_name, name, amount_agility, " \
+                                          "amount_intelligence, amount_psychic, amount_stamina, " \
+                                          "amount_strength, amount_sense) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)":
+                                    for row in param:
+                                        cur.execute(sql, row)
+                                    continue
+                                cur.executemany(sql, param)
+
+        print(f"Runtime: {time.time() - start: .2f} for {filename}")
+
+    def get_type(self) -> str:
+        return self.type
+
+
+class SqlException(Exception):
+    def __init__(self, message):
+        super().__init__(message)

core/decorators.py

@@ -0,0 +1,61 @@
+from core.dict_object import DictObject
+from core.registry import Registry
+
+
+# taken from: https://stackoverflow.com/a/26151604/280574
+def parameterized(dec):
+    def layer(*args, **kwargs):
+        def repl(f):
+            return dec(f, *args, **kwargs)
+
+        return repl
+
+    return layer
+
+
+@parameterized
+def instance(cls, name=None, override=False):
+    instance_name = name if name else cls.__name__
+    Registry.add_instance(instance_name, cls(), override)
+    return cls
+
+
+# noinspection PyShadowingNames
+@parameterized
+def command(handler, command, params, access_level, description,
+            sub_command=None, help_file=None, extended_description=None):
+    handler.command = [command, params, access_level, description, help_file, sub_command, extended_description]
+    return handler
+
+
+@parameterized
+def event(handler, event_type, description, is_hidden=False, is_enabled=True):
+    handler.event = DictObject({"event_type": event_type,
+                                "description": description,
+                                "is_hidden": is_hidden,
+                                "is_enabled": is_enabled})
+    return handler
+
+
+@parameterized
+def timerevent(handler, budatime, description, is_hidden=False, is_enabled=True, run_at_startup=False):
+    util = Registry.get_instance("util")
+    t = util.parse_time(budatime)
+    handler.event = DictObject({"event_type": "timer:" + str(t),
+                                "description": description,
+                                "is_hidden": is_hidden,
+                                "is_enabled": is_enabled,
+                                "run_at_startup": run_at_startup})
+    return handler
+
+
+@parameterized
+def setting(handler, name, value, description, extended_description=None):
+    obj = handler(None)
+
+    def new_handler(self):
+        return obj
+
+    new_handler.setting = [name, value, description, extended_description, obj]
+    new_handler.__module__ = handler.__module__
+    return new_handler

core/dict_object.py

@@ -0,0 +1,25 @@
+class DictObject(dict):
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+
+    def get_value(self, name):
+        val = self[name]
+        # convert dict to DictObject
+        if not isinstance(val, DictObject) and isinstance(val, dict):
+            self[name] = DictObject(val)
+            val = self[name]
+
+        # convert list of dicts to list of DictObjects
+        elif isinstance(val, list):
+            for k, v in enumerate(val):
+                if not isinstance(v, DictObject) and isinstance(v, dict):
+                    self[name][k] = DictObject(v)
+            val = self[name]
+
+        return val
+
+    def __getattr__(self, name):
+        return self.get_value(name)
+
+    def __setattr__(self, key, value):
+        self[key] = value

core/event_service.py

@@ -0,0 +1,212 @@
+import inspect
+import time
+from threading import Thread
+
+from core.decorators import instance
+from core.functions import get_attrs
+from core.logger import Logger
+from core.registry import Registry
+
+
+@instance()
+class EventService:
+    def __init__(self):
+        self.handlers = {}
+        self.logger = Logger(__name__)
+        self.event_types = []
+        self.db_cache = {}
+
+    def inject(self, registry):
+        self.db = registry.get_instance("db")
+        self.bot = registry.get_instance("bot")
+        self.util = registry.get_instance("util")
+
+    def pre_start(self):
+        self.register_event_type("timer")
+
+    def start(self):
+        # process decorators
+        for _, inst in Registry.get_all_instances().items():
+            for name, method in get_attrs(inst).items():
+                if hasattr(method, "event"):
+                    key = Registry.get_module_name(inst).split(".")
+                    # We dont want to load events, if their modules not enabled in our config...
+                    if key[0] not in self.bot.modules:
+                        continue
+                    attrs = getattr(method, "event")
+                    handler = getattr(inst, name)
+                    self.register(handler, attrs.event_type, attrs.description, inst.module_name, attrs.is_hidden,
+                                  attrs.is_enabled)
+
+    def register_event_type(self, event_type):
+        """
+        Call during pre_start
+
+        Args:
+            event_type (str)
+        """
+
+        event_type = event_type.lower()
+
+        if event_type in self.event_types:
+            self.logger.error("Could not register event type '%s': event type already registered" % event_type)
+            return
+
+        self.logger.debug("Registering event type '%s'" % event_type)
+        self.event_types.append(event_type)
+
+    def is_event_type(self, event_base_type):
+        return event_base_type in self.event_types
+
+    def register(self, handler, event_type, description, module, is_hidden, is_enabled):
+        """
+        Call during pre_start
+
+        Args:
+            handler: (event_type, event_data) -> void
+            event_type: str
+            description: str
+            module: str
+            is_hidden: bool
+            is_enabled: bool
+        """
+        if len(inspect.signature(handler).parameters) != 2:
+            raise Exception(
+                "Incorrect number of arguments for handler '%s.%s()'" % (handler.__module__, handler.__name__))
+
+        event_base_type, event_sub_type = self.get_event_type_parts(event_type)
+        module = module.lower()
+        handler_name = self.util.get_handler_name(handler)
+        is_hidden = 1 if is_hidden else 0
+        is_enabled = 1 if is_enabled else 0
+        if event_base_type not in self.event_types:
+            self.logger.error("Could not register handler '%s' for event type '%s': event type does not exist" % (
+                handler_name, event_type))
+            return
+
+        if not description:
+            self.logger.warning("No description for event_type '%s' and handler '%s'" % (event_type, handler_name))
+
+        row = self.db.query_single("SELECT 1 FROM event_config WHERE event_type = ? AND handler = ?",
+                                   [event_base_type, handler_name])
+        if row is None:
+            # add new event commands
+            self.db.exec(
+                "INSERT INTO event_config (event_type, event_sub_type, handler, "
+                "description, module, enabled, verified, is_hidden) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                [event_base_type, event_sub_type, handler_name, description, module, is_enabled, 1, is_hidden])
+            if event_base_type == "timer":
+                self.db.exec(
+                    "INSERT INTO timer_event (event_type, event_sub_type, handler, next_run) VALUES (?, ?, ?, ?)",
+                    [event_base_type, event_sub_type, handler_name, int(time.time())])
+        else:
+            # mark command as verified
+            self.db.exec(
+                "UPDATE event_config SET verified = ?, module = ?, description = ?, event_sub_type = ?, is_hidden = ? "
+                "WHERE event_type = ? AND handler = ?",
+                [1, module, description, event_sub_type, is_hidden, event_base_type, handler_name])
+
+            if event_base_type == "timer":
+                self.db.exec("UPDATE timer_event SET event_sub_type = ? WHERE event_type = ? AND handler = ?",
+                             [event_sub_type, event_base_type, handler_name])
+
+        # load command handler
+        self.handlers[handler_name] = handler
+
+    def fire_event(self, event_type, event_data=None):
+        event_base_type, event_sub_type = self.get_event_type_parts(event_type)
+
+        if event_base_type not in self.event_types:
+            self.logger.error("Could not fire event type '%s': event type does not exist" % event_type)
+            return
+
+        data = self.get_handlers(event_base_type, event_sub_type)
+        for row in data:
+            if event_type != "connect":
+                t = Thread(target=self.call_handler, args=(row.handler, event_type, event_data), daemon=True)
+                t.run()
+            else:
+                self.call_handler(row.handler, event_type, event_data)
+
+    def call_handler(self, handler_method, event_type, event_data):
+        handler = self.handlers.get(handler_method, None)
+        if not handler:
+            self.logger.error(
+                "Could not find handler callback for event type '%s' and handler '%s'" % (event_type, handler_method))
+            return
+
+        try:
+            handler(event_type, event_data)
+        except Exception as e:
+            self.logger.error("error processing event '%s'" % event_type, e)
+
+    def get_event_type_parts(self, event_type):
+        parts = event_type.lower().split(":", 1)
+        if len(parts) == 2:
+            return parts[0], parts[1]
+        else:
+            return parts[0], ""
+
+    def get_event_type_key(self, event_base_type, event_sub_type):
+        return event_base_type + ":" + event_sub_type
+
+    def check_for_timer_events(self, current_timestamp):
+        data = self.db.query("SELECT e.event_type, e.event_sub_type, e.handler, t.next_run FROM timer_event t "
+                             "JOIN event_config e ON t.event_type = e.event_type AND t.handler = e.handler "
+                             "WHERE t.next_run <= ? AND e.enabled = 1", [current_timestamp])
+        for row in data:
+            self.execute_timed_event(row, current_timestamp)
+
+    def execute_timed_event(self, row, current_timestamp):
+        event_type_key = self.get_event_type_key(row.event_type, row.event_sub_type)
+
+        # timer event run times should be consistent, so we base the next run time off the last run time,
+        # instead of the current timestamp
+        next_run = row.next_run + int(row.event_sub_type)
+
+        # prevents timer events from getting too far behind, or having a large "catch-up" after
+        # the bot has been offline for a time
+        if next_run < current_timestamp:
+            next_run = current_timestamp + int(row.event_sub_type)
+
+        self.db.exec("UPDATE timer_event SET next_run = ? WHERE event_type = ? AND handler = ?",
+                     [next_run, row.event_type, row.handler])
+
+        self.call_handler(row.handler, event_type_key, None)
+
+    def update_event_status(self, event_base_type, event_sub_type, event_handler, enabled_status):
+        # clear cache
+        self.db_cache[event_base_type + ":" + event_sub_type] = None
+
+        return self.db.exec(
+            "UPDATE event_config SET enabled = ? WHERE event_type = ? AND event_sub_type = ? AND handler LIKE ?",
+            [enabled_status, event_base_type, event_sub_type, event_handler])
+
+    def get_event_types(self):
+        return self.event_types
+
+    def get_handlers(self, event_base_type, event_sub_type):
+        # check first in cache
+        result = self.db_cache.get(event_base_type + ":" + event_sub_type, None)
+        if result is not None:
+            return result
+        else:
+            result = self.db.query(
+                "SELECT handler FROM event_config WHERE event_type = ? AND event_sub_type = ? AND enabled = 1",
+                [event_base_type, event_sub_type])
+
+            # store result in cache
+            self.db_cache[event_base_type + ":" + event_sub_type] = result
+
+            return result
+
+    def run_timer_events_at_startup(self):
+        t = int(time.time())
+        data = self.db.query("SELECT e.event_type, e.event_sub_type, e.handler, t.next_run FROM timer_event t "
+                             "JOIN event_config e ON t.event_type = e.event_type AND t.handler = e.handler "
+                             "WHERE e.event_type = ? AND e.enabled = 1", ["timer"])
+        for row in data:
+            handler = self.handlers[row.handler]
+            attrs = getattr(handler, "event")
+            if attrs.get("run_at_startup", False):
+                self.execute_timed_event(row, t)

core/fifo_queue.py

@@ -0,0 +1,38 @@
+import time
+from queue import Queue
+
+
+class FifoQueue(Queue):
+    def get_or_default(self, block=True, timeout=None, default=None):
+        """Remove and return an item from the queue.
+
+        This differs from get() in that it will return `default` instead of
+        raising Empty exception.
+
+        If optional args 'block' is true and 'timeout' is None (the default),
+        block if necessary until an item is available. If 'timeout' is
+        a non-negative number, it blocks at most 'timeout' seconds and raises
+        the Empty exception if no item was available within that time.
+        Otherwise ('block' is false), return an item if one is immediately
+        available, else raise the Empty exception ('timeout' is ignored
+        in that case).
+        """
+        with self.not_empty:
+            if not block:
+                if not self._qsize():
+                    return default
+            elif timeout is None:
+                while not self._qsize():
+                    self.not_empty.wait()
+            elif timeout < 0:
+                raise ValueError("'timeout' must be a non-negative number")
+            else:
+                endtime = time.time() + timeout
+                while not self._qsize():
+                    remaining = endtime - time.time()
+                    if remaining <= 0.0:
+                        return default
+                    self.not_empty.wait(remaining)
+            item = self._get()
+            self.not_full.notify()
+            return item

core/functions.py

@@ -0,0 +1,21 @@
+import itertools
+
+from core.dict_object import DictObject
+
+
+def flatmap(func, *iterable):
+    return itertools.chain.from_iterable(map(func, *iterable))
+
+
+# taken from: https://stackoverflow.com/a/8529229/280574 and modified
+def get_attrs(obj):
+    attrs = {}
+    for cls in obj.__class__.__mro__:
+        attrs.update(cls.__dict__.items())
+    attrs.update(obj.__class__.__dict__.items())
+    return attrs
+
+
+def merge_dicts(dict1, dict2):
+    res = DictObject({**dict1, **dict2})
+    return res

core/global.msg

@@ -0,0 +1,18 @@
+{
+  "char_not_found": {
+    "en_US": "Could not find character <highlight>{char}</highlight>.",
+    "de_DE": "Der Charakter <highlight>{char}</highlight> wurde nicht gefunden."
+  },
+  "access_denied": {
+    "en_US": "Access denied.",
+    "de_DE": "Zugriff verweigert."
+  },
+  "unknown_command": {
+    "en_US": "Error! Unknown command <highlight>{cmd}</highlight>.",
+    "de_DE": "Error! Den Befehl <highlight>{cmd}</highlight> kenne ich nicht."
+  },
+  "error_processing": {
+    "en_US": "There was an error processing your request.",
+    "de_DE": "Es gab einen Fehler beim bearbeiten deiner Anfrage."
+  }
+}

core/job_scheduler.py

@@ -0,0 +1,73 @@
+import inspect
+import time
+
+from core.decorators import instance
+from core.logger import Logger
+
+
+@instance()
+class JobScheduler:
+    def __init__(self):
+        self.logger = Logger(__name__)
+        self.jobs = []
+        self.job_id_index = 0
+
+    def check_for_scheduled_jobs(self, timestamp):
+        while self.jobs and self.jobs[0]["time"] <= timestamp:
+            try:
+                job = self.jobs.pop(0)
+                job["callback"](job["time"], *job["args"], **job["kwargs"])
+            except Exception as e:
+                self.logger.warning("Error processing scheduled job", e)
+
+    def delayed_job(self, callback, delay, *args, **kwargs):
+        """
+        Args:
+            callback: (time: Int, *args, *kwargs) -> void)
+            delay: int
+            *args
+            **kwargs
+        """
+
+        return self.scheduled_job(callback, int(time.time()) + delay, *args, **kwargs)
+
+    def scheduled_job(self, callback, scheduled_time, *args, **kwargs):
+        """
+        Args:
+            callback: (time: Int, *args, *kwargs) -> void)
+            scheduled_time: int
+            *args
+            **kwargs
+        """
+
+        if len(inspect.signature(callback).parameters) < 1:
+            raise Exception(f"Incorrect number of arguments for handler '{callback.__module__}.{callback.__name__}()'")
+
+        job_id = self._get_next_job_id()
+        new_job = {
+            "id": job_id,
+            "callback": callback,
+            "args": args,
+            "kwargs": kwargs,
+            "time": scheduled_time
+        }
+
+        self._insert_job(new_job)
+        return job_id
+
+    def cancel_job(self, job_id):
+        for index, job in enumerate(self.jobs):
+            if job["id"] == job_id:
+                return self.jobs.pop(index)
+        return None
+
+    def _insert_job(self, new_job):
+        for index, job in enumerate(self.jobs):
+            if job["time"] > new_job["time"]:
+                self.jobs.insert(index, new_job)
+                return
+        self.jobs.append(new_job)
+
+    def _get_next_job_id(self):
+        self.job_id_index += 1
+        return self.job_id_index

core/logger.py

@@ -0,0 +1,44 @@
+import logging
+import logging.handlers
+import re
+import traceback
+
+
+class Logger:
+    def __init__(self, name):
+        self.logger = logging.getLogger(name)
+
+    def warning(self, msg, obj: Exception = None):
+        self.logger.warning(self.format_message(msg, obj))
+
+    def info(self, msg, obj: Exception = None):
+        self.logger.info(self.format_message(msg, obj))
+
+    def error(self, msg, obj: Exception = None):
+        self.logger.error(self.format_message(msg, obj))
+
+    def debug(self, msg, obj: Exception = None):
+        self.logger.debug(self.format_message(msg, obj))
+
+    def log_chat(self, conn_id, channel, sender, msg):
+        if sender:
+            self.info("(%s) [%s] %s: %s" % (conn_id, channel, sender, self.format_chat_message(msg)))
+        else:
+            self.info("(%s) [%s] %s" % (conn_id, channel, self.format_chat_message(msg)))
+
+    def log_tell(self, conn_id, direction, sender, msg):
+        self.info("(%s) %s %s: %s" % (conn_id, direction.capitalize(), sender, self.format_chat_message(msg)))
+
+    def format_chat_message(self, msg):
+        msg = re.sub(r"<a\s+href=\".+?[^\\]\">", "[link]", msg, flags=re.UNICODE | re.DOTALL)
+        msg = re.sub(r"<a\s+href='.+?'>", "[link]", msg, flags=re.UNICODE | re.DOTALL)
+        msg = re.sub(r"<font\s+.+?>", "", msg, flags=re.UNICODE)
+        msg = re.sub("</font>", "", msg, flags=re.UNICODE)
+        msg = re.sub("</a>", "[/link]", msg, flags=re.UNICODE)
+        return msg
+
+    def format_message(self, msg, obj):
+        if obj:
+            return msg + "\n" + traceback.format_exc()
+        else:
+            return msg

core/lookup/character_history_service.py

@@ -0,0 +1,67 @@
+import json
+import time
+
+import requests
+from requests import ReadTimeout
+from torpy.http.requests import do_request
+
+from core.db import DB
+from core.decorators import instance
+from core.dict_object import DictObject
+from core.logger import Logger
+
+
+@instance()
+class CharacterHistoryService:
+    CACHE_GROUP = "history"
+    CACHE_MAX_AGE = 86400
+    users = []
+
+    def __init__(self):
+        self.logger = Logger(__name__)
+
+    def inject(self, registry):
+        self.bot = registry.get_instance("bot")
+        self.db: DB = registry.get_instance("db")
+        self.cache_service = registry.get_instance("cache_service")
+
+    def get_character_history(self, name, server_num):
+        cache_key = "%s.%d.json" % (name, server_num)
+
+        t = int(time.time())
+
+        # check cache for fresh value
+        cache_result = self.cache_service.retrieve(self.CACHE_GROUP, cache_key)
+        if cache_result and cache_result.last_modified > (t - self.CACHE_MAX_AGE):
+            result = json.loads(cache_result.data)
+        else:
+            url = self.get_pork_url(server_num, name)
+            try:
+                r = do_request(url)
+                # with TorRequests() as tor_request:
+                #     with tor_request.get_session(1) as session:
+                #         r = session.get(url, timeout=5)
+                r = requests.get(url, timeout=5, headers={"User-Agent": self.bot.major_version})
+                result = r.json()
+            except ReadTimeout:
+                self.logger.warning("Timeout while requesting '%s'" % url)
+                result = None
+            except Exception as e:
+                self.logger.error("Error requesting history for url '%s'" % url, e)
+                result = None
+
+            if result:
+                # store result in cache
+                self.cache_service.store(self.CACHE_GROUP, cache_key, json.dumps(result))
+            elif cache_result:
+                # check cache for any value, even expired
+                result = json.loads(cache_result.data)
+
+        if result:
+            return map(lambda x: DictObject(x), result)
+        else:
+            return None
+
+    def get_pork_url(self, dimension, char_name):
+        # noinspection HttpUrlsUsage
+        return f"http://pork.budabot.jkbff.com/pork/history.php?server={dimension:d}&name={char_name}"

core/lookup/character_service.py

@@ -0,0 +1,99 @@
+import threading
+import time
+
+from core.aochat import server_packets
+from core.aochat.client_packets import CharacterLookup
+from core.decorators import instance
+
+
+@instance()
+class CharacterService:
+    def __init__(self):
+        self.name_to_id = {}
+        self.id_to_name = {}
+        self.waiting_for_response = set()
+        self.notify_on_receive = {}
+
+    def inject(self, registry):
+        self.bot = registry.get_instance("bot")
+        self.db = registry.get_instance("db")
+
+    def pre_start(self):
+        self.bot.register_packet_handler(server_packets.CharacterLookup.id, self.update)
+        self.bot.register_packet_handler(server_packets.CharacterName.id, self.update)
+
+    def start(self):
+        self.db.shared.exec("CREATE TABLE IF NOT EXISTS `all_orgs` (`org_id` INT(11) NOT NULL, "
+                            "`org_name` VARCHAR(64) NOT NULL COLLATE 'utf8mb4_general_ci', "
+                            "`member_count` INT(11) NULL DEFAULT NULL, "
+                            "`faction` VARCHAR(16) NULL DEFAULT NULL COLLATE 'utf8mb4_general_ci', "
+                            "`last_seen` INT(11) NOT NULL, "
+                            "PRIMARY KEY (`org_id`) USING BTREE, "
+                            "INDEX `org_name` (`org_name`) USING BTREE)")
+        self.db.create_view("all_orgs")
+
+    def _wait_for_char_id(self, char_name):
+        # char_name must be .capitalize()'ed
+
+        packet = self.bot.iterate(1)
+        while packet and char_name not in self.name_to_id:
+            packet = self.bot.iterate(1)
+
+        return self.name_to_id.get(char_name, None)
+
+    # FeatureFlags.THREADING
+    def _wait_for_char_id_threading(self, char_name):
+        # char_name must be .capitalize()'ed
+
+        event = self.notify_on_receive.get(char_name, None)
+        if event is None:
+            event = threading.Event()
+            self.notify_on_receive[char_name] = event
+
+        if char_name not in self.name_to_id:
+            event.wait(10)
+
+        return self.name_to_id.get(char_name, None)
+
+    def resolve_char_to_id(self, char):
+        if isinstance(char, int):
+            return char
+        elif char.isdigit():
+            return int(char)
+        else:
+            char_name = char.capitalize()
+            if char_name in self.name_to_id:
+                return self.name_to_id[char_name]
+            else:
+                self._send_lookup_if_needed(char_name)
+                return self._wait_for_char_id(char_name)
+
+    def resolve_char_to_name(self, char, default=None):
+        if isinstance(char, int) or char.isdigit():
+            char_name = self.get_char_name(char)
+            return char_name if char_name else default
+        else:
+            return char
+
+    def get_char_name(self, char_id):
+        return self.id_to_name.get(char_id, None)
+
+    def update(self, conn, packet):
+        self.waiting_for_response.discard(packet.name)
+
+        if packet.char_id == 4294967295:
+            self.name_to_id[packet.name] = None
+        else:
+            self.id_to_name[packet.char_id] = packet.name
+            self.name_to_id[packet.name] = packet.char_id
+            # self._update_name_history(packet.name, packet.char_id)
+
+    def _update_name_history(self, char_name, char_id):
+        params = [char_name, char_id, int(time.time())]
+        self.db.exec("INSERT IGNORE INTO name_history (name, char_id, created_at) VALUES (?, ?, ?)", params)
+
+    def _send_lookup_if_needed(self, char_name):
+        # char_name must be .capitalize()'ed
+        if char_name not in self.name_to_id and char_name not in self.waiting_for_response:
+            self.waiting_for_response.add(char_name)
+            self.bot.send_packet(CharacterLookup(char_name))

core/lookup/org_pork_service.py

@@ -0,0 +1,187 @@
+import datetime
+import json
+import time
+
+import requests
+from requests import ReadTimeout
+
+from core.db import DB
+from core.decorators import instance
+from core.dict_object import DictObject
+from core.logger import Logger
+
+
+@instance()
+class OrgPorkService:
+    CACHE_GROUP = "org_roster"
+    CACHE_MAX_AGE = 86400
+
+    def __init__(self):
+        self.logger = Logger(__name__)
+
+    def inject(self, registry):
+        self.bot = registry.get_instance("bot")
+        self.db: DB = registry.get_instance("db")
+        self.character_service = registry.get_instance("character_service")
+        self.pork_service = registry.get_instance("pork_service")
+        self.cache_service = registry.get_instance("cache_service")
+
+    def get_org_info(self, org_id):
+        cache_key = f"{org_id:d}.{self.bot.dimension:d}.json"
+
+        t = int(time.time())
+
+        # check cache for fresh value
+        cache_result = self.cache_service.retrieve(self.CACHE_GROUP, cache_key)
+
+        is_cache = False
+        if cache_result and cache_result.last_modified > (t - self.CACHE_MAX_AGE):
+            result = json.loads(cache_result.data)
+            is_cache = True
+        else:
+            url = self.get_pork_url(self.bot.dimension, org_id)
+
+            try:
+                r = requests.get(url, timeout=5)
+                result = r.json()
+
+                # if data is invalid
+                if result[0]["ORG_INSTANCE"] != org_id:
+                    result = None
+            except ReadTimeout:
+                self.logger.warning("Timeout while requesting '%s'" % url)
+                result = None
+            except ValueError as e:
+                # noinspection PyUnboundLocalVariable
+                self.logger.warning("Error marshalling value as json for url '%s': %s" % (url, r.text), e)
+                result = None
+
+            if result:
+                # store result in cache
+                self.cache_service.store(self.CACHE_GROUP, cache_key, json.dumps(result))
+            elif cache_result:
+                # check cache for any value, even expired
+                result = json.loads(cache_result.data)
+                is_cache = True
+
+        if not result:
+            return None
+
+        org_info = result[0]
+        org_members = result[1]
+        last_updated = result[2]
+
+        new_org_info = DictObject({
+            "counts": {
+                "gender": {
+                    "Female": org_info["FEMALECOUNT"],
+                    "Male": org_info["MALECOUNT"],
+                    "Neuter": org_info["NEUTERCOUNT"],
+                },
+                "breed": {
+                    "Atrox": org_info["ATROXCOUNT"],
+                    "Nanomage": org_info["NANORACECOUNT"],
+                    "Opifex": org_info["OPIFEXCOUNT"],
+                    "Solitus": org_info["SOLITUSCOUNT"],
+                },
+                "profession": {
+                    "Monster": org_info["MONSTERCOUNT"],
+                    "Adventurer": org_info["ADVENTURERCOUNT"],
+                    "Agent": org_info["AGENTCOUNT"],
+                    "Bureaucrat": org_info["BTCOUNT"],
+                    "Doctor": org_info["DOCTORCOUNT"],
+                    "Enforcer": org_info["ENFCOUNT"],
+                    "Engineer": org_info["ENGINEEERCOUNT"],
+                    "Fixer": org_info["FIXERCOUNT"],
+                    "Keeper": org_info["KEEPERCOUNT"],
+                    "Martial Artist": org_info["MACOUNT"],
+                    "Meta-Physicist": org_info["METACOUNT"],
+                    "Nano-Technician": org_info["NANOCOUNT"],
+                    "Shade": org_info["SHADECOUNT"],
+                    "Soldier": org_info["SOLIDERCOUNT"],
+                    "Trader": org_info["TRADERCOUNT"],
+                }
+            },
+            "min_level": org_info["MINLVL"],
+            "num_members": org_info["NUMMEMBERS"],
+            "dimension": org_info["ORG_DIMENSION"],
+            "governing_type": org_info["GOVERNINGNAME"],
+            "max_level": org_info["MAXLVL"],
+            "org_id": org_info["ORG_INSTANCE"],
+            "objective": org_info["OBJECTIVE"],
+            "description": org_info["DESCRIPTION"],
+            "history": org_info["HISTORY"],
+            "avg_level": org_info["AVGLVL"],
+            "name": org_info["NAME"],
+            "faction": org_info["SIDE_NAME"],
+            "faction_id": org_info["SIDE"],
+        })
+        members = {}
+        data = []
+        for org_member in org_members:
+            char_info = DictObject({
+                "name": org_member["NAME"],
+                "char_id": org_member["CHAR_INSTANCE"],
+                "first_name": org_member["FIRSTNAME"],
+                "last_name": org_member["LASTNAME"],
+                "level": org_member["LEVELX"],
+                "breed": org_member["BREED"],
+                "dimension": org_member["CHAR_DIMENSION"],
+                "gender": org_member["SEX"],
+                "faction": org_info["SIDE_NAME"],
+                "profession": org_member["PROF"],
+                "profession_title": org_member["PROF_TITLE"],
+                "ai_rank": org_member["DEFENDER_RANK_TITLE"],
+                "ai_level": org_member["ALIENLEVEL"],
+                "pvp_rating": org_member["PVPRATING"],
+                "pvp_title": org_member["PVPTITLE"] or "",
+                "head_id": org_member["HEADID"],
+                "org_id": org_info.get("ORG_INSTANCE", 0),
+                "org_name": org_info.get("NAME", ""),
+                "org_rank_name": org_member.get("RANK_TITLE", ""),
+                "org_rank_id": org_member.get("RANK", 0),
+                "source": "people.anarchy-online.com"
+            })
+
+            if not is_cache:
+                data.append(
+                    (char_info.char_id, char_info.name, char_info.first_name, char_info.last_name, char_info.level,
+                     char_info.breed, char_info.gender, char_info.faction, char_info.profession,
+                     char_info.profession_title,
+                     char_info.ai_rank, char_info.ai_level, char_info.org_id, char_info.org_name,
+                     char_info.org_rank_name,
+                     char_info.org_rank_id, char_info.dimension, char_info.head_id, char_info.pvp_rating,
+                     char_info.pvp_title, char_info.source, int(time.time())))
+
+                # prefetch char ids from chat server
+            # noinspection PyProtectedMember
+            self.character_service._send_lookup_if_needed(char_info.name)
+
+            members[char_info.char_id] = char_info
+
+        if len(data) > 0:
+            with self.db.lock:
+                with self.db.pool.get_connection() as conn:
+                    with conn.cursor() as cur:
+                        cur.executemany(
+                            "INSERT IGNORE INTO player ( char_id, name, first_name, last_name, level, breed, gender, "
+                            "faction, profession, profession_title, "
+                            "ai_rank, ai_level, org_id, org_name, "
+                            "org_rank_name, org_rank_id, dimension, "
+                            "head_id, pvp_rating, pvp_title, source, last_updated) "
+                            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", data)
+                        conn.commit()
+
+        if len(members) == 0:
+            return None
+        else:
+            return DictObject({"last_modified": cache_result.last_modified if is_cache else t,
+                               "org_info": new_org_info,
+                               "org_members": members,
+                               "last_updated": int(
+                                   datetime.datetime.strptime(last_updated, "%Y/%m/%d %H:%M:%S").timestamp())})
+
+    def get_pork_url(self, dimension, org_id):
+        # Dont use SSL, as its rather slow compared to normal requests....
+        # noinspection HttpUrlsUsage
+        return f"http://people.anarchy-online.com/org/stats/d/{dimension}/name/{org_id}/basicstats.xml?data_type=json"

core/lookup/pork_service.py

@@ -0,0 +1,267 @@
+import time
+
+import requests
+from mysql.connector.cursor import CursorBase
+from requests import ReadTimeout
+
+from core.aochat import server_packets
+from core.db import DB
+from core.decorators import instance, timerevent
+from core.dict_object import DictObject
+from core.logger import Logger
+
+
+@instance()
+class PorkService:
+    def __init__(self):
+        self.logger = Logger(__name__)
+        self.updates = []
+
+    def inject(self, registry):
+        self.bot = registry.get_instance("bot")
+        self.db: DB = registry.get_instance("db")
+        self.character_service = registry.get_instance("character_service")
+
+    def pre_start(self):
+        self.bot.register_packet_handler(server_packets.CharacterLookup.id, self.update)
+        self.bot.register_packet_handler(server_packets.CharacterName.id, self.update)
+
+    def start(self):
+        self.db.shared.exec(
+            "CREATE TABLE IF NOT EXISTS player ("
+            "char_id BIGINT PRIMARY KEY, "
+            "first_name VARCHAR(30) NOT NULL, "
+            "name VARCHAR(20) NOT NULL, "
+            "last_name VARCHAR(30) NOT NULL, "
+            "level SMALLINT NOT NULL, "
+            "breed VARCHAR(20) NOT NULL, "
+            "gender VARCHAR(20) NOT NULL, "
+            "faction VARCHAR(20) NOT NULL, "
+            "profession VARCHAR(20) NOT NULL, "
+            "profession_title VARCHAR(50) NOT NULL, "
+            "ai_rank VARCHAR(20) NOT NULL, "
+            "ai_level SMALLINT, "
+            "org_id INT DEFAULT NULL, "
+            "org_name VARCHAR(255) NOT NULL, "
+            "org_rank_name VARCHAR(20) NOT NULL, "
+            "org_rank_id SMALLINT NOT NULL, "
+            "dimension SMALLINT NOT NULL, "
+            "head_id INT NOT NULL, "
+            "pvp_rating SMALLINT NOT NULL, "
+            "pvp_title VARCHAR(20) NOT NULL, "
+            "source VARCHAR(50) NOT NULL, "
+            "last_updated INT NOT NULL, "
+            "invalid int DEFAULT 0, "
+            "INDEX `name` (`name`) USING BTREE, "
+            "INDEX `org_id` (`org_id`) USING BTREE, "
+            "INDEX `org_name` (`org_name`) USING BTREE,"
+            "INDEX `org_rank_name` (`org_rank_name`) USING BTREE, "
+            "INDEX `org_rank_id` (`org_rank_id`) USING BTREE, "
+            "INDEX `profession` (`profession`) USING BTREE, "
+            "INDEX `level` (`level`) USING BTREE, "
+            "INDEX `ai_level` (`ai_level`) USING BTREE)")
+
+        self.db.create_view("player")
+
+    # forces a lookup from remote PoRK server
+    # this should not be called directly unless you are requesting info for a char on a different server
+    # since cache will not be used and the result will also update the cache
+    def request_char_info(self, char_name, server_num):
+        url = self.get_pork_url(server_num, char_name)
+
+        try:
+            r = requests.get(url, timeout=5)
+            result = r.json()
+        except ReadTimeout:
+            self.logger.warning("Timeout while requesting '%s'" % url)
+            result = None
+        except ValueError as e:
+            # noinspection PyUnboundLocalVariable
+            self.logger.debug("Error marshalling value as json for url '%s': %s" % (url, r.text), e)
+            result = None
+
+        char_info = None
+        if result:
+            char_info_json = result[0]
+            org_info_json = result[1] if result[1] else {}
+
+            char_info = DictObject({
+                "name": char_info_json["NAME"],
+                "char_id": char_info_json["CHAR_INSTANCE"],
+                "first_name": char_info_json["FIRSTNAME"],
+                "last_name": char_info_json["LASTNAME"],
+                "level": char_info_json["LEVELX"],
+                "breed": char_info_json["BREED"],
+                "dimension": char_info_json["CHAR_DIMENSION"],
+                "gender": char_info_json["SEX"],
+                "faction": char_info_json["SIDE"],
+                "profession": char_info_json["PROF"],
+                "profession_title": char_info_json["PROFNAME"],
+                "ai_rank": char_info_json["RANK_name"],
+                "ai_level": char_info_json["ALIENLEVEL"],
+                "pvp_rating": char_info_json["PVPRATING"],
+                "pvp_title": char_info_json["PVPTITLE"] or "",
+                "head_id": char_info_json["HEADID"],
+                "org_id": org_info_json.get("ORG_INSTANCE", 0),
+                "org_name": org_info_json.get("NAME", ""),
+                "org_rank_name": org_info_json.get("RANK_TITLE", ""),
+                "org_rank_id": org_info_json.get("RANK", 0),
+                "source": "people.anarchy-online.com",
+                "cache_age": 0
+            })
+
+        return char_info
+
+    # standard method to get character pork data when character is on the same server
+    def get_character_info(self, char, max_cache_age=86400):
+        char_id = self.character_service.resolve_char_to_id(char)
+        char_name = self.character_service.resolve_char_to_name(char)
+
+        t = int(time.time())
+
+        # if there is an entry in database and it is within the cache time, use that
+        db_char_info = self.get_from_database(char_id=char_id, char_name=char_name)
+        if db_char_info:
+            db_char_info.cache_age = t - db_char_info.last_updated
+
+            if db_char_info.cache_age < max_cache_age and db_char_info.source != "chat_server":
+                return db_char_info
+
+        # if we can't resolve to a char_name, we can't make a call to pork
+        if not char_name:
+            return db_char_info
+
+        char_info = self.request_char_info(char_name, self.bot.dimension)
+
+        if char_info and char_info.char_id == char_id:
+            self.save_character_info(char_info)
+
+            return char_info
+        else:
+            # return cached info from database, even tho it's old, and set cache_age (if it exists)
+            if db_char_info:
+                db_char_info.cache_age = t - db_char_info.last_updated
+
+            return db_char_info
+
+    # forces a skeleton object into the player table in the case that PoRK does not return any data
+    # call this method if you don't need the data now but want to ensure there is a record in the database
+    def load_character_info(self, char_id, char_name=None, skeleton_only=False):
+        char_info = self.get_character_info(char_id)
+        if not skeleton_only and (not char_info and char_name):
+            char_info = self.get_character_info(char_name)
+        if not char_info:
+            char_info = DictObject({
+                "name": "Unknown:" + str(char_id),
+                "char_id": char_id,
+                "first_name": "",
+                "last_name": "",
+                "level": 0,
+                "breed": "",
+                "dimension": self.bot.dimension,
+                "gender": "",
+                "faction": "",
+                "profession": "",
+                "profession_title": "",
+                "ai_rank": "",
+                "ai_level": 0,
+                "pvp_rating": 0,
+                "pvp_title": "",
+                "head_id": 0,
+                "org_id": 0,
+                "org_name": "",
+                "org_rank_name": "",
+                "org_rank_id": 6,
+                "source": "stub"
+            })
+            self.save_character_info(char_info)
+
+    def save_character_info(self, char_info):
+        if char_info["dimension"] != self.bot.dimension:
+            return
+        with self.db.pool.get_connection() as conn:
+            with conn.cursor() as cur:
+                # cur.execute("DELETE FROM player WHERE char_id = ?", [char_info["char_id"]])
+
+                insert_sql = """
+                    REPLACE INTO player (char_id, name, first_name, last_name, level, 
+                    breed, gender, faction, profession, profession_title, ai_rank, ai_level, 
+                    org_id, org_name, org_rank_name, org_rank_id, dimension, head_id,
+                    pvp_rating, pvp_title, source, last_updated)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """
+
+                cur.execute(insert_sql,
+                            [char_info["char_id"], char_info["name"], char_info["first_name"], char_info["last_name"],
+                             char_info["level"], char_info["breed"],
+                             char_info["gender"], char_info["faction"], char_info["profession"],
+                             char_info["profession_title"], char_info["ai_rank"], char_info["ai_level"],
+                             char_info["org_id"], char_info["org_name"], char_info["org_rank_name"],
+                             char_info["org_rank_id"], char_info["dimension"], char_info["head_id"],
+                             char_info["pvp_rating"], char_info["pvp_title"], char_info["source"], int(time.time())])
+
+    def get_from_database(self, char_id=None, char_name=None):
+        if char_id:
+            return self.db.query_single(
+                "SELECT char_id, name, first_name, last_name, level, breed, gender, faction, profession, "
+                "profession_title, ai_rank, ai_level, org_id, org_name, org_rank_name, org_rank_id, "
+                "dimension, head_id, pvp_rating, pvp_title, source, last_updated "
+                "FROM player WHERE char_id = ?", [char_id])
+        elif char_name:
+            return self.db.query_single(
+                "SELECT char_id, name, first_name, last_name, level, breed, gender, faction, profession, "
+                "profession_title, ai_rank, ai_level, org_id, org_name, org_rank_name, org_rank_id, "
+                "dimension, head_id, pvp_rating, pvp_title, source, last_updated "
+                "FROM player WHERE name = ?", [char_name])
+        else:
+            return None
+
+    def update(self, conn, packet):
+        # don't update if we didn't get a valid response
+        if packet.char_id == 4294967295:
+            return
+        self.updates.append(packet)
+
+    @timerevent(budatime="1min", description="Save player changes", is_hidden=True)
+    def batch_update(self, event_type, event_data):
+        update = "UPDATE player SET name = ? WHERE char_id = ?"
+        insert = "INSERT IGNORE INTO player ( char_id, name, first_name, last_name, level, breed, gender, faction, " \
+                 "profession, profession_title, ai_rank, ai_level, org_id, org_name, org_rank_name, org_rank_id, " \
+                 "dimension, head_id, pvp_rating, pvp_title, source, last_updated) " \
+                 "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+        updates, inserts = [], []
+        with self.db.pool.get_connection() as conn:
+            with conn.cursor(dictionary=True) as cur:
+                for packet in self.updates:
+                    cur: CursorBase
+                    cur.execute(
+                        "SELECT char_id, name, first_name, last_name, level, breed, gender, faction, profession, "
+                        "profession_title, ai_rank, ai_level, org_id, org_name, org_rank_name, org_rank_id, "
+                        "dimension, head_id, pvp_rating, pvp_title, source, last_updated "
+                        "FROM player WHERE char_id = ?",
+                        [packet.char_id])
+                    data = cur.fetchone()
+                    character = None
+                    if data:
+                        character = DictObject(data)
+                    if character:
+                        if character.name != packet.name:
+                            updates.append((packet.name, packet.char_id))
+                    else:
+                        inserts.append((packet.char_id, packet.name, "", "", 0, "", "", "", "", "", "", 0, 0, "", "", 6,
+                                        self.bot.dimension, 0, 0, "", "chat_server", int(time.time())))
+                if inserts:
+                    cur.executemany(insert, inserts)
+                if updates:
+                    cur.executemany(update, updates)
+        self.updates = []
+
+    # noinspection SqlResolve
+    def find_orgs(self, search):
+        return self.db.query("SELECT DISTINCT org_name, org_id FROM all_orgs WHERE org_name <EXTENDED_LIKE=0> ?",
+                             [search], extended_like=True)
+
+    def get_pork_url(self, dimension, char_name):
+        # Dont use SSL, as its rather slow compared to normal requests....
+        # noinspection HttpUrlsUsage
+        return f"http://people.anarchy-online.com/character/bio/d/{dimension}/name/{char_name}/bio.xml?data_type=json"

core/message_hub_service.py

@@ -0,0 +1,116 @@
+import inspect
+
+from core.decorators import instance
+from core.dict_object import DictObject
+from core.logger import Logger
+from core.lookup.character_service import CharacterService
+from core.text import Text
+
+
+@instance()
+class MessageHubService:
+    def __init__(self):
+        self.logger = Logger(__name__)
+        self.hub = {}
+        self.sources = []
+
+    def inject(self, registry):
+        self.bot = registry.get_instance("bot")
+        self.setting_service = registry.get_instance("setting_service")
+        self.character_service: CharacterService = registry.get_instance("character_service")
+        self.text: Text = registry.get_instance("text")
+        self.db = registry.get_instance("db")
+
+    def pre_start(self):
+        self.db.exec("CREATE TABLE IF NOT EXISTS message_hub_subscriptions ( "
+                     "destination VARCHAR(50) NOT NULL,"
+                     "source VARCHAR(50) NOT NULL"
+                     ")")
+
+    def register_message_source(self, source):
+        """Call during pre_start"""
+        if source not in self.sources:
+            self.sources.append(source)
+
+    def register_message_destination(self, destination, callback, default_sources, invalid_sources=None):
+        """
+        Call during start
+
+        Args:
+            destination: str
+            callback: (ctx) -> void
+            default_sources: [str...]
+            invalid_sources: [str...]
+        """
+
+        if invalid_sources is None:
+            invalid_sources = []
+        if len(inspect.signature(callback).parameters) != 1:
+            raise Exception(
+                "Incorrect number of arguments for handler '%s.%s()'" % (callback.__module__, callback.__name__))
+
+        if destination in self.hub:
+            raise Exception("Message hub destination '%s' already subscribed" % destination)
+
+        for source in default_sources:
+            if source not in self.sources:
+                self.logger.warning(
+                    "Could not subscribe destination '%s' to source '%s' because source does not exist" % (
+                        destination, source))
+
+        self.hub[destination] = (DictObject({"name": destination,
+                                             "callback": callback,
+                                             "sources": default_sources,
+                                             "invalid_sources": invalid_sources}))
+
+        self.reload_mapping(destination)
+
+    def reload_mapping(self, destination):
+        data = self.db.query("SELECT source FROM message_hub_subscriptions WHERE destination = ?", [destination])
+        if data:
+            self.hub[destination].sources = list(map(lambda x: x.source, data))
+
+    def send_message(self, source, sender, message, formatted_message):
+        ctx = DictObject({"source": source,
+                          "sender": sender,
+                          "message": message,
+                          "formatted_message": formatted_message})
+
+        for _, c in self.hub.items():
+            if source in c.sources:
+                try:
+                    c.callback(ctx)
+                except Exception as e:
+                    self.logger.error("", e)
+
+    def subscribe_to_source(self, destination, source):
+        if source not in self.sources:
+            raise Exception("Message hub source '%s' doeselecs not exist" % source)
+
+        obj = self.hub.get(destination, None)
+        if not obj:
+            raise Exception("Message hub destination '%s' does not exist" % destination)
+
+        if source not in obj.sources:
+            self.db.exec("DELETE FROM message_hub_subscriptions WHERE destination = ?", [destination])
+
+            obj.sources.append(source)
+            for source in obj.sources:
+                self.db.exec("INSERT INTO message_hub_subscriptions (destination, source)"
+                             "VALUES (?, ?)", [destination, source])
+
+    def unsubscribe_from_source(self, destination, source):
+        # if source not in self.sources:
+        #    raise Exception("Message hub source '%s' does not exist" % source)
+
+        obj = self.hub.get(destination, None)
+        if not obj:
+            raise Exception("Message hub destination '%s' does not exist" % destination)
+
+        if source in obj.sources:
+            self.db.exec("DELETE FROM message_hub_subscriptions WHERE destination = ?", [destination])
+
+            obj.sources.remove(source)
+            for source in obj.sources:
+                self.db.exec("INSERT INTO message_hub_subscriptions (destination, source)"
+                             "VALUES (?, ?)", [destination, source])

core/private_channel_service.py

@@ -0,0 +1,76 @@
+from core.aochat import server_packets, client_packets
+from core.conn import Conn
+from core.decorators import instance
+from core.logger import Logger
+
+
+@instance()
+class PrivateChannelService:
+    PRIVATE_CHANNEL_MESSAGE_EVENT = "private_channel_message"
+    JOINED_PRIVATE_CHANNEL_EVENT = "private_channel_joined"
+    LEFT_PRIVATE_CHANNEL_EVENT = "private_channel_left"
+
+    def __init__(self):
+        self.logger = Logger(__name__)
+        self.private_channel_chars = {}
+
+    def inject(self, registry):
+        self.bot = registry.get_instance("bot")
+        self.event_service = registry.get_instance("event_service")
+        self.character_service = registry.get_instance("character_service")
+        self.access_service = registry.get_instance("access_service")
+
+    def pre_start(self):
+        self.event_service.register_event_type(self.JOINED_PRIVATE_CHANNEL_EVENT)
+        self.event_service.register_event_type(self.LEFT_PRIVATE_CHANNEL_EVENT)
+        self.event_service.register_event_type(self.PRIVATE_CHANNEL_MESSAGE_EVENT)
+
+        self.bot.register_packet_handler(server_packets.PrivateChannelClientJoined.id,
+                                         self.handle_private_channel_client_joined)
+        self.bot.register_packet_handler(server_packets.PrivateChannelClientLeft.id,
+                                         self.handle_private_channel_client_left)
+        # priority must be above that of CommandService in order for relaying of commands to work correctly
+        self.bot.register_packet_handler(server_packets.PrivateChannelMessage.id,
+                                         self.handle_private_channel_message, priority=30)
+
+        self.access_service.register_access_level("guest", 95, self.in_private_channel)
+
+    def handle_private_channel_message(self, conn: Conn, packet: server_packets.PrivateChannelMessage):
+        if conn.id != "main":
+            return
+
+        if packet.private_channel_id == self.bot.get_char_id():
+            self.event_service.fire_event(self.PRIVATE_CHANNEL_MESSAGE_EVENT, packet)
+
+    def handle_private_channel_client_joined(self, conn: Conn, packet: server_packets.PrivateChannelClientJoined):
+        if conn.id != "main":
+            return
+
+        if packet.private_channel_id == self.bot.get_char_id():
+            self.private_channel_chars[packet.char_id] = packet
+            self.event_service.fire_event(self.JOINED_PRIVATE_CHANNEL_EVENT, packet)
+
+    def handle_private_channel_client_left(self, conn: Conn, packet: server_packets.PrivateChannelClientLeft):
+        if conn.id != "main":
+            return
+
+        if packet.private_channel_id == self.bot.get_char_id():
+            del self.private_channel_chars[packet.char_id]
+            self.event_service.fire_event(self.LEFT_PRIVATE_CHANNEL_EVENT, packet)
+
+    def invite(self, char_id):
+        if char_id != self.bot.get_char_id():
+            self.bot.send_packet(client_packets.PrivateChannelInvite(char_id))
+
+    def kick(self, char_id):
+        if char_id != self.bot.get_char_id() and char_id in self.private_channel_chars:
+            self.bot.send_packet(client_packets.PrivateChannelKick(char_id))
+
+    def kickall(self):
+        self.bot.send_packet(client_packets.PrivateChannelKickAll())
+
+    def in_private_channel(self, char_id):
+        return char_id in self.private_channel_chars
+
+    def get_all_in_private_channel(self):
+        return self.private_channel_chars

core/public_channel_service.py

@@ -0,0 +1,116 @@
+from core.aochat import server_packets
+from core.aochat.BaseModule import BaseModule
+from core.conn import Conn
+from core.decorators import instance
+from core.logger import Logger
+from core.setting_service import SettingService
+from core.setting_types import NumberSettingType, TextSettingType
+
+
+@instance()
+class PublicChannelService(BaseModule):
+    ORG_CHANNEL_MESSAGE_EVENT = "org_channel_message"
+    ORG_MSG_EVENT = "org_msg"
+
+    ORG_MSG_CHANNEL_ID = 42949672961
+
+    def __init__(self):
+        self.logger = Logger(__name__)
+        self.name_to_id = {}
+        self.id_to_name = {}
+        self.org_channel_id = None
+        self.org_id = None
+        self.org_name = None
+
+    def inject(self, registry):
+        self.bot = registry.get_instance("bot")
+        self.event_service = registry.get_instance("event_service")
+        self.character_service = registry.get_instance("character_service")
+        self.setting_service: SettingService = registry.get_instance("setting_service")
+
+    def pre_start(self):
+        self.bot.register_packet_handler(server_packets.PublicChannelJoined.id, self.add)
+        self.bot.register_packet_handler(server_packets.PublicChannelLeft.id, self.remove)
+        # priority must be above that of CommandService in order for relaying of commands to work correctly
+        self.bot.register_packet_handler(server_packets.PublicChannelMessage.id, self.public_channel_message,
+                                         priority=30)
+        self.event_service.register_event_type(self.ORG_CHANNEL_MESSAGE_EVENT)
+        self.event_service.register_event_type(self.ORG_MSG_EVENT)
+        self.setting_service.register_new('core.system', 'org_id', 0,
+                                          NumberSettingType(), 'OrgID used for roster')
+        self.setting_service.register_new('core.system', 'org_name', "",
+                                          TextSettingType(allow_empty=True), 'OrgName used for roster')
+
+    def start(self):
+        org_id_setting = self.setting_service.get("org_id")
+        if org_id_setting and org_id_setting.get_value() and org_id_setting.get_value() != "0":
+            self.org_id = org_id_setting.get_value()
+
+        org_name_setting = self.setting_service.get("org_name")
+        if org_name_setting and org_name_setting.get_value():
+            self.org_name = org_name_setting.get_value()
+
+    def get_channel_id(self, channel_name):
+        return self.name_to_id.get(channel_name)
+
+    def get_channel_name(self, channel_id):
+        return self.id_to_name.get(channel_id, None)
+
+    def add(self, conn: Conn, packet: server_packets.PublicChannelJoined):
+        if conn.id != "main":
+            return
+
+        self.id_to_name[packet.channel_id] = packet.name
+        self.name_to_id[packet.name] = packet.channel_id
+        if not self.org_id and self.is_org_channel_id(packet.channel_id):
+            self.org_channel_id = packet.channel_id
+            self.org_id = 0x00ffffffff & packet.channel_id
+            if packet.name != "Clan (name unknown)":
+                self.setting_service.get("org_name").set_value(packet.name)
+                self.org_name = packet.name
+            else:
+                data = self.event_service.db.query_single('SELECT org_name from all_orgs where org_id=?', [self.org_id])
+                self.org_name = data.org_name if data else 'Unknown Org'
+            self.logger.info("Org Id: %d" % self.org_id)
+            self.logger.info("Org Name: %s" % self.org_name)
+
+    def remove(self, conn: Conn, packet: server_packets.PublicChannelLeft):
+        if conn.id != "main":
+            return
+
+        channel_name = self.get_channel_name(packet.channel_id)
+        del self.id_to_name[packet.channel_id]
+        del self.name_to_id[channel_name]
+
+    def public_channel_message(self, conn: Conn, packet: server_packets.PublicChannelMessage):
+        if conn.id != "main":
+            return
+
+        if self.is_org_channel_id(packet.channel_id):
+            # char_name = self.character_service.get_char_name(packet.char_id)
+            # if packet.extended_message:
+            #     message = packet.extended_message.get_message()
+            # else:
+            #     message = packet.message
+            # # self.logger.log_chat(conn.id, "Org Channel", char_name, message)
+            self.event_service.fire_event(self.ORG_CHANNEL_MESSAGE_EVENT, packet)
+        elif packet.channel_id == self.ORG_MSG_CHANNEL_ID:
+            # char_name = self.character_service.get_char_name(packet.char_id)
+            # if packet.extended_message:
+            #     message = packet.extended_message.get_message()
+            # else:
+            #     message = packet.message
+            # self.logger.log_chat(conn.id, "Org Msg", char_name, message)
+            self.event_service.fire_event(self.ORG_MSG_EVENT, packet)
+
+    def is_org_channel_id(self, channel_id):
+        return channel_id >> 32 == 3
+
+    def get_org_id(self):
+        return self.org_id
+
+    def get_org_name(self):
+        return self.org_name
+
+    def get_all_public_channels(self):
+        return self.id_to_name

core/registry.py

@@ -0,0 +1,117 @@
+import importlib
+import os
+import re
+
+from core.functions import flatmap
+
+
+class Registry:
+    _registry = {}
+    logger = None
+
+    @classmethod
+    def inject_all(cls):
+        # inject registry so instance can get references to other instances
+        for key in cls._registry:
+            try:
+                cls._registry[key].inject
+            except AttributeError:
+                pass
+            else:
+                cls._registry[key].inject(cls)
+
+    @classmethod
+    def pre_start_all(cls):
+        # call pre_start() on instances so they can start any init() processes
+        for key in cls._registry:
+            try:
+                cls._registry[key].pre_start
+            except AttributeError:
+                pass
+            else:
+                cls._registry[key].pre_start()
+
+    @classmethod
+    def start_all(cls):
+        # call start() on instances so they can finish any init() processes
+        for key in cls._registry:
+            try:
+                cls._registry[key].start
+            except AttributeError:
+                pass
+            else:
+                cls._registry[key].start()
+
+    @classmethod
+    def get_instance(cls, name, is_optional=False):
+        instance = cls._registry.get(name)
+        if instance or is_optional:
+            return instance
+        else:
+            raise Exception("Missing required dependency '%s'" % name)
+
+    @classmethod
+    def get_all_instances(cls):
+        return cls._registry
+
+    @classmethod
+    def add_instance(cls, name, inst, override=False):
+        name = cls.format_name(name)
+
+        inst.module_name = Registry.get_module_name(inst)
+        inst.module_dir = Registry.get_module_dir(inst)
+
+        if not override and name in cls._registry:
+            raise Exception("Overriding '%s' with new instance" % name)
+        elif override and name not in cls._registry:
+            raise Exception("No instance '%s' to override" % name)
+        cls._registry[name] = inst
+
+    @classmethod
+    def format_name(cls, name):
+        # camel-case to snake-case
+        s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
+        return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
+
+    @classmethod
+    def load_instances(cls, parent_dirs):
+        # get all subdirectories
+        dirs = flatmap(lambda x: os.walk(x, followlinks=True), parent_dirs)
+        dirs = filter(lambda y: not y[0].endswith("__pycache__"), dirs)
+
+        def get_files(tup):
+            return map(lambda x: os.path.join(tup[0], x), tup[2])
+
+        # get files from subdirectories
+        files = flatmap(get_files, dirs)
+        files = filter(lambda z: z.endswith(".py") and not z.endswith("__init__.py"), files)
+
+        # load files as modules
+        for file in files:
+            cls.load_module(file)
+
+    @classmethod
+    def load_module(cls, file):
+        # strip the extension
+        file = file[:-3]
+        importlib.import_module(file.replace("\\", ".").replace("/", "."))
+
+    @classmethod
+    def get_module_name(cls, inst):
+        parts = inst.__module__.split(".")
+        if parts[0] == "core":
+            return "core"
+        # last name in directory path should be first part, then the next name should be last part
+        elif parts[0] == "modules":
+            return parts[1] + "." + parts[2]
+        else:
+            return ".".join(parts[:-1])
+
+    @classmethod
+    def get_module_dir(cls, inst):
+        parts = inst.__module__.split(".")
+        return "." + os.sep + os.sep.join(parts[:-1])
+
+    @classmethod
+    def clear(cls):
+        cls._registry = {}

core/sender_obj.py

@@ -0,0 +1,17 @@
+class SenderObj:
+    def __init__(self, char_id, name, access_level):
+        self.char_id = char_id
+        self.name = name
+        self.access_level = access_level
+
+    def __str__(self):
+        return self.__dict__.__str__()
+
+    def __repr__(self):
+        return self.__str__()
+
+    def __eq__(self, obj):
+        return isinstance(obj, SenderObj) and \
+               obj.char_id == self.char_id and \
+               obj.name == self.name and \
+               obj.access_level == self.access_level

core/setting_service.py

@@ -0,0 +1,118 @@
+import inspect
+
+from core.decorators import instance
+from core.functions import get_attrs
+from core.logger import Logger
+from core.registry import Registry
+
+
+@instance()
+class SettingService:
+    def __init__(self):
+        self.logger = Logger(__name__)
+        self.settings = {}
+        self.db_cache = {}
+        self.change_listeners = {}
+
+    def inject(self, registry):
+        self.db = registry.get_instance("db")
+        self.bot = registry.get_instance("bot")
+        self.util = registry.get_instance("util")
+
+    def start(self):
+        # process decorators
+        for _, inst in Registry.get_all_instances().items():
+            for name, method in get_attrs(inst).items():
+                if hasattr(method, "setting"):
+                    setting_name, value, description, extended_description, obj = getattr(method, "setting")
+                    self.register(setting_name, value, description, obj, inst.module_name, extended_description)
+
+    def register(self, name, value, description, setting, module, extended_description=None):
+        """Deprecated. Use register_new()"""
+        self.logger.warning(f"Using deprecated register method for setting '{name}' in module {module}")
+        self.register_new(module, name, value, setting, description, extended_description)
+
+    def register_new(self, module, name, value, setting, description, extended_description=None):
+        """Call during start"""
+        name = name.lower()
+        module = module.lower()
+        # do not generate settings for not loaded modules
+        if module.split(".")[0] not in self.bot.modules:
+            return
+        setting.set_name(name)
+        setting.set_description(description)
+        setting.set_extended_description(extended_description)
+
+        if not description:
+            self.logger.warning("No description specified for setting '%s'" % name)
+
+        if " " in name:
+            raise Exception("One or more spaces found in setting name '%s' for module '%s'" % (name, module))
+
+        row = self.db.query_single("SELECT name, value, description FROM setting WHERE name = ?", [name])
+
+        if row is None:
+            self.logger.debug("Adding setting '%s'" % name)
+            self.db.exec("INSERT INTO setting (name, value, description, module, verified) VALUES (?, ?, ?, ?, ?)",
+                         [name, "", description, module, 1])
+            print(2, name)
+
+            # verify default value is a valid value, and is formatted appropriately
+            setting.set_value(value)
+        else:
+            self.logger.debug("Updating setting '%s'" % name)
+            self.db.exec("UPDATE setting SET description = ?, verified = ?, module = ? WHERE name = ?",
+                         [description, 1, module, name])
+        self.settings[name] = setting
+
+    def register_change_listener(self, setting_name, handler):
+        """
+        Call during start
+
+        Args:
+            setting_name: str
+            handler: (name: string, old_value, new_value) -> void
+        """
+
+        if len(inspect.signature(handler).parameters) != 3:
+            raise Exception(f"Incorrect number of arguments for handler '{handler.__module__}.{handler.__name__}()'")
+
+        if setting_name in self.settings:
+            if setting_name not in self.change_listeners:
+                self.change_listeners[setting_name] = []
+            self.change_listeners[setting_name].append(handler)
+        else:
+            raise Exception(f"Could not register change_listener for setting '{setting_name}' since it does not exist")
+
+    def get_value(self, name):
+        # check cache first
+        result = self.db_cache.get(name, None)
+        if result:
+            return result.value
+        else:
+            row = self.db.query_single("SELECT value FROM setting WHERE name = ?", [name])
+
+            # store result in cache
+            self.db_cache[name] = row
+
+            return row.value if row else None
+
+    def set_value(self, name, value):
+        old_value = self.get_value(name)
+
+        # clear cache
+        self.db_cache[name] = None
+
+        self.db.exec("UPDATE setting SET value = ? WHERE name = ?", [value, name])
+
+        if name in self.change_listeners:
+            for change_listener in self.change_listeners[name]:
+                change_listener(name, old_value, value)
+
+    def get(self, name):
+        name = name.lower()
+        setting = self.settings.get(name, None)
+        if setting:
+            return setting
+        else:
+            return None

core/setting_types.py

@@ -0,0 +1,302 @@
+import json
+import re
+
+from core.dict_object import DictObject
+from core.registry import Registry
+
+
+class SettingType:
+    def __init__(self):
+        self.setting_service = Registry.get_instance("setting_service")
+        self.name = None
+
+    def set_name(self, name):
+        self.name = name
+
+    def _get_raw_value(self):
+        """Get the value from the database"""
+        return self.setting_service.get_value(self.name)
+
+    def _set_raw_value(self, value):
+        """Set the value in the database"""
+        self.setting_service.set_value(self.name, value)
+
+    def set_value(self, value):
+        """Set the processed/typed value"""
+        pass
+
+    def get_value(self):
+        """Get the processed/typed value"""
+        return self._get_raw_value()
+
+    def get_display_value(self):
+        """Get the value formatted for display"""
+        v = self.get_value()
+        if v == "":
+            v = "<empty>"
+
+        return "<highlight>%s</highlight>" % v
+
+    def set_description(self, description):
+        self.description = description
+
+    def get_description(self):
+        return self.description
+
+    def set_extended_description(self, extended_description):
+        self.extended_description = extended_description
+
+    def get_extended_description(self):
+        return self.extended_description
+
+
+class TextSettingType(SettingType):
+    def __init__(self, options=None, allow_empty=False):
+        super().__init__()
+        self.options = options
+        self.allow_empty = allow_empty
+
+    def set_value(self, value):
+        if len(str(value)) > 255:
+            raise Exception("Setting value cannot be longer than 255 characters.")
+        elif not self.allow_empty and (value is None or value == ""):
+            raise Exception("Setting value cannot be empty.")
+        else:
+            self._set_raw_value(value)
+
+    def get_display(self):
+        text = Registry.get_instance("text")
+
+        clear_str = ""
+        if self.allow_empty:
+            clear_str = "\n\nTo clear this setting:\n\n" + text.make_tellcmd("Clear this setting",
+                                                                             "config setting %s clear" % self.name)
+
+        options_str = ""
+        if self.options:
+            options_str = "\n\nOr choose an option below:\n\n" + "\n".join(
+                map(lambda opt: text.make_tellcmd(str(opt), f"config setting {self.name} set {opt}"), self.options))
+
+        return """For this setting you can enter any text you want (max. 255 characters).
+
+To change this setting:
+
+<highlight>/tell <myname> config setting """ + self.name + """ set <i>_value_</i></highlight>""" \
+               + clear_str + options_str
+
+
+class DictionarySettingType(SettingType):
+    def __init__(self):
+        super().__init__()
+
+    def set_value(self, value):
+        if not value:
+            self._set_raw_value("")
+        elif isinstance(value, dict):
+            self._set_raw_value(json.dumps(value))
+        else:
+            raise Exception("Value must be a dictionary.")
+
+    def get_value(self):
+        value = self._get_raw_value()
+        if value:
+            return DictObject(json.loads(value))
+        else:
+            return value
+
+    def get_display_value(self):
+        return "<highlight>%s</highlight>" % (self.get_value() or "&lt;empty&gt;")
+
+    def get_display(self):
+        return """This setting is controlled by the bot and cannot be set manually."""
+
+
+class HiddenSettingType(TextSettingType):
+    def __init__(self, options=None, allow_empty=False):
+        super().__init__(options, allow_empty)
+
+    def get_display_value(self):
+        if self.get_value():
+            return "<highlight>&lt;hidden&gt;</highlight>"
+        else:
+            return "<highlight>&lt;empty&gt;</highlight>"
+
+    def get_display(self):
+        text = Registry.get_instance("text")
+
+        clear_str = ""
+        if self.allow_empty:
+            clear_str = "\n\nTo clear this setting:\n\n" + text.make_tellcmd("Clear this setting",
+                                                                             f"config setting {self.name} clear")
+
+        return """For this setting you can enter any text you want (max. 255 characters).
+
+To change this setting:
+
+<highlight>/tell <myname> config setting """ + self.name + """ set <i>_value_</i></highlight>""" + clear_str + """
+
+The saved value is never shown in the config but it may appear in the logs and is stored in plain text in the database.
+"""
+
+
+# noinspection LongLine
+class ColorSettingType(SettingType):
+    def __init__(self):
+        super().__init__()
+
+    def get_display_value(self):
+        return self.format_text(self.get_value())
+
+    def set_value(self, value):
+        if re.match("^#([0-9a-fA-F]{6})$", str(value)):
+            self._set_raw_value(value.upper())
+        else:
+            raise Exception("You must enter a valid HTML color.")
+
+    def get_display(self):
+        return """For this setting you can set any Color in the HTML Hexadecimal Color Format.
+
+You can change it manually with the command:
+
+/tell <myname> config setting """ + self.name + """ set <i>_HTML Color_</i>
+
+Or you can choose one of the following colors
+
+<font color='#FF0000'>Red</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #FF0000'>Save it</a>)
+<font color='#FFFFFF'>White</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #FFFFFF'>Save it</a>)
+<font color='#808080'>Grey</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #808080'>Save it</a>)
+<font color='#DDDDDD'>Light Grey</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #DDDDDD'>Save it</a>)
+<font color='#9CC6E7'>Dark Grey</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #9CC6E7'>Save it</a>)
+<font color='#000000'>Black</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #000000'>Save it</a>)
+<font color='#FFFF00'>Yellow</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #FFFF00'>Save it</a>)
+<font color='#8CB5FF'>Blue</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #8CB5FF'>Save it</a>)
+<font color='#00BFFF'>Deep Sky Blue</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #00BFFF'>Save it</a>)
+<font color='#00DE42'>Green</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #00DE42'>Save it</a>)
+<font color='#FCA712'>Orange</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #FCA712'>Save it</a>)
+<font color='#FFD700'>Gold</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #FFD700'>Save it</a>)
+<font color='#FF1493'>Deep Pink</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #FF1493'>Save it</a>)
+<font color='#EE82EE'>Violet</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #EE82EE'>Save it</a>)
+<font color='#8B7355'>Brown</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #8B7355'>Save it</a>)
+<font color='#00FFFF'>Cyan</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #00FFFF'>Save it</a>)
+<font color='#000080'>Navy Blue</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #000080'>Save it</a>)
+<font color='#FF8C00'>Dark Orange</font> (<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set #FF8C00'>Save it</a>)"""
+
+    def get_font_color(self):
+        return "<font color='%s'>" % self.get_value()
+
+    def get_int_value(self):
+        return int(self.get_value().replace("#", ""), 16)
+
+    def format_text(self, msg):
+        return self.get_font_color() + msg + "</font>"
+
+
+class NumberSettingType(SettingType):
+    def __init__(self, options=None, allow_empty=False):
+        super().__init__()
+        self.options = options
+        self.allow_empty = allow_empty
+
+    def get_value(self):
+        v = self._get_raw_value()
+        if v != "":
+            return int(self._get_raw_value())
+        else:
+            return ""
+
+    def set_value(self, value):
+        if value == "":
+            if self.allow_empty:
+                self._set_raw_value(value)
+            else:
+                raise Exception("This setting does not allow an empty value.")
+        elif re.match(r"^\d+$", str(value)):
+            self._set_raw_value(value)
+        else:
+            raise Exception("You must enter a positive integer for this setting.")
+
+    def get_display(self):
+        text = Registry.get_instance("text")
+
+        clear_str = ""
+        if self.allow_empty:
+            clear_str = "\n\nTo clear this setting:\n\n" + text.make_tellcmd("Clear this setting",
+                                                                             f"config setting {self.name} clear")
+
+        options_str = ""
+        if self.options:
+            options_str = "\n\nOr choose an option below:\n\n" + "\n".join(
+                map(lambda opt: text.make_tellcmd(str(opt), f"config setting {self.name} set {opt}"), self.options))
+
+        return """For this setting you can set any positive integer.
+
+To change this setting:
+
+<highlight>/tell <myname> config setting """ + self.name + """ set <i>_number_</i></highlight>""" \
+               + clear_str + options_str
+
+
+class TimeSettingType(SettingType):
+    def __init__(self, options=None):
+        super().__init__()
+        self.options = options
+
+    def get_value(self):
+        return int(self._get_raw_value())
+
+    def get_display_value(self):
+        util = Registry.get_instance("util")
+        return "<highlight>%s</highlight>" % util.time_to_readable(self.get_value())
+
+    def set_value(self, value):
+        util = Registry.get_instance("util")
+        time = util.parse_time(value)
+        if time > 0:
+            self._set_raw_value(time)
+        else:
+            raise Exception("You must enter time in a valid Budatime format")
+
+    def get_display(self):
+        text = Registry.get_instance("text")
+        if self.options:
+            options_str = "\n".join(
+                map(lambda opt: text.make_tellcmd(str(opt), f"config setting {self.name} set {opt}"), self.options))
+        else:
+            options_str = "No presets defined, please use the command above."
+        return """For this setting you must enter a time value. 
+See <a href='chatcmd:///tell <myname> help budatime'>budatime</a> for info on the format of the 'time' parameter.
+
+To change this setting:
+
+<highlight>/tell <myname> config setting """ + self.name + """ set <i>_time_</i></highlight>
+
+Or choose an option below:\n\n""" + options_str
+
+
+class BooleanSettingType(SettingType):
+    def __init__(self):
+        super().__init__()
+
+    def get_value(self):
+        return int(self._get_raw_value()) == 1
+
+    def get_display_value(self):
+        return "<highlight>%s</highlight>" % ("True" if self.get_value() else "False")
+
+    def set_value(self, value):
+        if value is True:
+            self._set_raw_value(1)
+        elif value is False:
+            self._set_raw_value(0)
+        elif value.lower() == "true":
+            self._set_raw_value(1)
+        elif value.lower() == "false":
+            self._set_raw_value(0)
+        else:
+            raise Exception("You must enter either 'true' or 'false'")
+
+    def get_display(self):
+        return """For this setting you can enter either true or false.
+
+<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set true'>True</a>
+<a href='chatcmd:///tell <myname> config setting """ + self.name + """ set false'>False</a>"""

core/text.py

@@ -0,0 +1,359 @@
+import math
+import re
+from html.parser import HTMLParser
+
+from core.chat_blob import ChatBlob
+from core.decorators import instance
+from core.logger import Logger
+from core.setting_service import SettingService
+
+
+class MLStripper(HTMLParser):
+    def error(self, message):
+        pass
+
+    def __init__(self):
+        super().__init__()
+        self.reset()
+        self.strict = False
+        self.convert_charrefs = True
+        self.fed = []
+        self.chat_commands = []
+
+    def handle_data(self, d):
+        self.fed.append(d)
+
+    def get_data(self):
+        return "".join(self.fed)
+
+
+@instance()
+class Text:
+    separators = [{"symbol": "<pagebreak>", "include": False}, {"symbol": "\n", "include": True},
+                  {"symbol": "<br>", "include": True}, {"symbol": " ", "include": True}]
+
+    # taken from IGN bot
+    pixel_mapping = {'i': 3, 'l': 3, 'K': 10, 'R': 10, "'": 3, 'e': 8, 'U': 10, 'j': 5, 'I': 5, '|': 6, 'N': 10, 'f': 5,
+                     '.': 5, ' ': 5,
+                     ',': 5, 'J': 6, 'r': 6, 't': 6, '!': 6, '(': 6, ')': 6, '[': 6, ']': 6, '/': 6, ':': 6, ';': 6,
+                     '"': 6, 'c': 7,
+                     '-': 7, 's': 8, 'v': 8, 'k': 8, 'a': 8, 'y': 8, 'z': 8, 'F': 8, 'L': 8, 'P': 8, 'n': 9, '3': 9,
+                     'b': 9, 'd': 9,
+                     'g': 9, 'h': 9, 'Y': 9, 'S': 10, 'Q': 11, 'w': 11, '<': 11, '>': 11, '=': 11, 'q': 9, 'u': 9,
+                     'x': 9, '0': 9,
+                     '1': 9, '2': 9, '4': 9, '5': 9, '6': 9, '7': 9, '8': 9, '9': 9, 'E': 9, 'T': 9, '$': 9, '*': 9,
+                     '{': 9, '}': 9,
+                     '_': 9, '`': 9, 'A': 10, 'B': 10, 'C': 10, 'H': 10, 'V': 10, 'X': 10, 'Z': 10, '&': 10, 'D': 11,
+                     'G': 11, 'M': 11,
+                     'O': 11, '+': 11, '~': 11, '%': 15, 'p': 9, 'm': 13, 'o': 9, '@': 14, 'W': 15}
+
+    def __init__(self):
+        self.logger = Logger(__name__)
+        self.items_regex = re.compile(r"<a href=\"itemref://(\d+)/(\d+)/(\d+)\">(.+?)</a>")
+
+    def inject(self, registry):
+        self.setting_service: SettingService = registry.get_instance("setting_service")
+        self.ban = registry.get_instance("ban_service")
+        self.bot = registry.get_instance("bot")
+        self.public_channel_service = registry.get_instance("public_channel_service")
+
+    def make_chatcmd(self, name, msg, style=""):
+        msg = msg.strip()
+        msg = msg.replace("'", "&#39;")
+        return "<a %s href='chatcmd://%s'>%s</a>" % (style, msg, name)
+
+    def make_tellcmd(self, name, msg, style="", char="<myname>"):
+        return self.make_chatcmd(name, f"/tell {char} {msg}", style)
+
+    def make_charlink(self, char, style=""):
+        return "<a %s href='user://%s'>%s</a>" % (style, char, char)
+
+    def make_item(self, low_id, high_id, ql, name):
+        return "<a href='itemref://%d/%d/%d'>%s</a>" % (low_id, high_id, ql, name)
+
+    def make_image(self, image_id, image_db="rdb"):
+        return "<img src='%s://%s'>" % (image_db, image_id)
+
+    def format_item(self, item, ql=None, with_icon=True):
+        if not item:
+            return None
+
+        ql = ql or item["highql"]
+
+        result = self.make_item(item["lowid"], item["highid"], ql, item["name"])
+
+        if with_icon:
+            result = self.make_image(item["icon"]) + "\n" + result
+
+        return result
+
+    def generate_item(self, item, ql, synonym=None):
+        if synonym:
+            return {"icon_%s" % synonym: self.make_item(item.lowid, item.highid, ql, self.make_image(item.icon)),
+                    "text_%s" % synonym: self.make_item(item.lowid, item.highid, ql, item.name)}
+        else:
+            return {"icon": self.make_item(item.lowid, item.highid, ql, self.make_image(item.icon)),
+                    "text": self.make_item(item.lowid, item.highid, ql, item.name)}
+
+    def get_count_digits(self, number: int):
+        """Return number of digits in a number."""
+
+        if number == 0:
+            return 1
+
+        number = abs(number)
+
+        if number <= 999999999999997:
+            return math.floor(math.log10(number)) + 1
+
+        count = 0
+        while number:
+            count += 1
+            number //= 10
+        return count
+
+    def zfill(self, numb, highest_number):
+        return f"<black>{(self.get_count_digits(highest_number) - self.get_count_digits(numb)) * '0'}</black>{numb}"
+
+    def format_pagination(self, data, offset, page, formatter, title, no_data_msg, cmd, page_size=10, headline=""):
+        selected = data[offset:offset + page_size]
+        count = len(selected)
+        pages = ""
+        if page > 1:
+            pages += "Pages:   " + self.make_tellcmd("«« Page %d" % (page - 1), f'{cmd} --page={page - 1}')
+        if offset + page_size < len(data):
+            pages += f"   Page {page}/{math.ceil(len(data) / page_size)}"
+            pages += "   " + self.make_tellcmd("Page %d »»" % (page + 1), f'{cmd} --page={page + 1}')
+        pages += "\n"
+        if count == 0:
+            return no_data_msg
+        else:
+            blob = "<font color=CCInfoText>"
+            blob += "" + pages + "\n"
+            blob += headline
+            index = offset
+            for entry in selected:
+                index += 1
+                blob += formatter(entry, index, data)
+            blob += pages
+            blob += "</font>"
+            return ChatBlob(title, blob)
+
+    def format_char_info(self, char_info, online_status=None, check_ban=False):
+        banned = ""
+
+        if char_info.org_name and char_info.org_rank_name:
+            msg = f"<{char_info.faction.lower()}>{char_info.name}</{char_info.faction.lower()}> :: " \
+                  f"{char_info.level}/<green>{char_info.ai_level}</green> {char_info.profession} :: " \
+                  f"<{char_info.faction.lower()}>{char_info.org_name}</{char_info.faction.lower()}>"
+        elif char_info.get("level", None):
+            msg = f"<{char_info.faction.lower()}>{char_info.name}</{char_info.faction.lower()}> :: " \
+                  f"{char_info.level}/<green>{char_info.ai_level}</green> {char_info.profession}"
+        elif char_info.name:
+            msg = f"<highlight>{char_info.name}</highlight>"
+        else:
+            msg = f"<highlight>CharId({char_info.char_id:d})</highlight>"
+        if check_ban:
+            banned = f" :: <red>Banned!</red>" if self.ban.get_ban(char_info.char_id) else ""
+        msg += banned
+        if online_status is not None:
+            msg += " :: " + ("<green>Online</green>" if online_status else "<red>Offline</red>")
+
+        return msg
+
+    def get_formatted_faction(self, faction, contents=None):
+        if not contents:
+            contents = faction.capitalize()
+        faction = faction.lower()
+        if faction in ["omni", "clan", "neutral"]:
+            return f"<{faction}>{contents}</{faction}>"
+        return f"<unknown>{contents}</unknown>"
+
+    def paginate_single(self, chatblob):
+        return self.paginate(chatblob, 8000)[0]
+
+    def paginate(self, chatblob, max_page_length=None, max_num_pages=None, footer=None):
+        label = chatblob.title
+        msg = chatblob.msg
+
+        msg = msg.strip()
+
+        # chat blobs with empty messages are rendered as simple strings instead of links
+        if not msg:
+            return [label]
+
+        msg = self.items_regex.sub(r"<a href='itemref://\1/\2/\3'>\4</a>", msg)
+
+        color = self.setting_service.get("blob_color").get_font_color()
+        msg = ("<header>" + label + "</header>\n\n" + color + msg).replace("\"", "&quot;")
+        msg = self.format_message(msg)
+
+        if footer:
+            footer = "\n\n" + self.format_message(footer.replace("\"", "&quot;").strip())
+        else:
+            footer = ""
+
+        adjusted_max_page_length = None
+        if max_page_length:
+            adjusted_max_page_length = max_page_length - len(footer)
+        pages = self.split_by_separators(msg, adjusted_max_page_length, max_num_pages)
+        pages = list(map(lambda p: p + footer, pages))
+
+        num_pages = len(pages)
+
+        def mapper(tup):
+            page, index = tup
+            if num_pages == 1:
+                label2 = self.format_message(label)
+            else:
+                label2 = self.format_message(label) + " (Page " + str(index) + " / " + str(num_pages) + ")"
+            return chatblob.page_prefix + self.format_page(label2, page) + chatblob.page_postfix
+
+        return list(map(mapper, zip(pages, range(1, num_pages + 1))))
+
+    def split_by_separators(self, content, max_page_length=None, max_num_pages=None):
+        separators = iter(self.separators)
+
+        separator = next(separators)
+        rest = content
+        current_page = ""
+        pages = []
+
+        while len(rest) > 0:
+            line, rest = self.get_next_line(rest, separator)
+            line_length = len(line)
+
+            # if separator is not sufficient, try the next one
+            if max_page_length and line_length > max_page_length:
+                try:
+                    separator = next(separators)
+                    rest = line + rest
+                    continue
+                except StopIteration:
+                    # this is thrown when there are no more separators in the iterator
+                    raise Exception("Could not paginate: page is too large")
+
+            if max_num_pages == len(pages) + 1:
+                if max_page_length and (len(current_page) + line_length > max_page_length):
+                    break
+            else:
+                if max_page_length and len(current_page) + line_length > max_page_length:
+                    pages.append(current_page.strip())
+                    current_page = ""
+
+            current_page += line
+
+        current_page = current_page.strip()
+        if max_page_length and len(current_page) > max_page_length:
+            pages.append(current_page)
+        else:
+            pages.append(current_page)
+
+        return pages
+
+    def format_page(self, label, msg):
+        return "<a href=\"text://%s\">%s</a>" % (msg, label)
+
+    def get_next_line(self, msg, separator):
+        result = msg.split(separator["symbol"], 1)
+        line = result[0]
+        if len(result) == 1:
+            rest = ""
+        else:
+            rest = result[1:][0]
+
+        if separator["include"]:
+            line += separator["symbol"]
+
+        return line, rest
+
+    def strip_html_tags(self, s):
+        if not s:
+            return None
+
+        stripper = MLStripper()
+        stripper.feed(s)
+        return stripper.get_data()
+
+    def pad_table(self, rows, fill=" "):
+        max_width = {}
+        for columns in rows:
+            for i, column in enumerate(columns[:-1]):
+                w = self.get_pixel_width(column)
+                if i not in max_width or max_width[i] < w:
+                    max_width[i] = w
+
+        for columns in rows:
+            adjustment = 0
+            num_cols = len(columns)
+            for i, column in enumerate(columns):
+                if i == num_cols - 1:
+                    continue
+
+                s, new_adjustment = self.pad_string(column, adjustment + max_width[i], fill)
+                columns[i] = s
+                adjustment += new_adjustment
+
+        return rows
+
+    def pad_string(self, s, length, fill=" "):
+        if s is None:
+            s = ""
+
+        s_pixel_width = self.get_pixel_width(s)
+        spacer_pixel_width = self.get_pixel_width(fill)
+        fill_width = length - s_pixel_width
+        if fill_width > 0:
+            num_spacers = round(fill_width / spacer_pixel_width)
+        else:
+            num_spacers = 0
+        adjustment = fill_width - (spacer_pixel_width * num_spacers)
+        return s + (num_spacers * fill), adjustment
+
+    def get_pixel_width(self, s):
+        if not s:
+            return 0
+
+        s = self.strip_html_tags(s)
+
+        width = 0
+        for c in s:
+            pixel_width = self.pixel_mapping.get(c, None)
+            if not pixel_width:
+                self.logger.warning(f"Unknown pixel width mapping for char '{c}'")
+                pixel_width = 8
+            width += pixel_width or 8
+        return width
+
+    def format_message(self, msg, replace_br=True):
+        for t in ["</header>", "</header2>", "</highlight>", "</notice>", "</black>", "</white>", "</yellow>",
+                  "</blue>", "</green>", "</red>", "</orange>", "</grey>", "</cyan>",
+                  "</violet>", "</neutral>", "</omni>", "</clan>", "</unknown>"]:
+            msg = msg.replace(t, "</font>")
+        if replace_br:
+            msg = msg.replace("<br>", "\n")
+        return msg \
+            .replace("<header>", self.setting_service.get("header_color").get_font_color()) \
+            .replace("<header2>", self.setting_service.get("header2_color").get_font_color()) \
+            .replace("<highlight>", self.setting_service.get("highlight_color").get_font_color()) \
+            .replace("<notice>", self.setting_service.get("notice_color").get_font_color()) \
+            .replace("<black>", "<font color=#000000>") \
+            .replace("<white>", "<font color=#FFFFFF>") \
+            .replace("<yellow>", "<font color=#FFFF00>") \
+            .replace("<blue>", "<font color=#8CB5FF>") \
+            .replace("<green>", "<font color=#00DE42>") \
+            .replace("<red>", "<font color=#FF0000>") \
+            .replace("<orange>", "<font color=#FCA712>") \
+            .replace("<grey>", "<font color=#C3C3C3>") \
+            .replace("<cyan>", "<font color=#00FFFF>") \
+            .replace("<violet>", "<font color=#8F00FF>") \
+            .replace("<neutral>", self.setting_service.get("neutral_color").get_font_color()) \
+            .replace("<omni>", self.setting_service.get("omni_color").get_font_color()) \
+            .replace("<clan>", self.setting_service.get("clan_color").get_font_color()) \
+            .replace("<unknown>", self.setting_service.get("unknown_color").get_font_color()) \
+            .replace("<myname>", self.bot.get_char_name()) \
+            .replace("<myorg>", self.public_channel_service.get_org_name() or "Unknown Org") \
+            .replace("<tab>", "    ") \
+            .replace("<end>", "</font>") \
+            .replace("<symbol>", self.setting_service.get("symbol").get_value()) \
+            .replace("\n", "<br>")

core/translation_service.py

@@ -0,0 +1,102 @@
+import inspect
+
+import hjson
+
+from core.decorators import instance
+from core.event_service import EventService
+from core.logger import Logger
+from core.setting_service import SettingService
+from core.setting_types import TextSettingType
+from core.tyrbot import Tyrbot
+from core.util import Util
+
+
+# noinspection PyDefaultArgument
+@instance()
+class TranslationService:
+    strings = {}
+    translation_callbacks = {}
+    language = None
+    lang_codes = ["en_US", "de_DE"]
+    LANGUAGE_SETTING = "language"
+
+    def __init__(self):
+        self.logger = Logger(__name__)
+
+    def inject(self, registry):
+        self.setting_service: SettingService = registry.get_instance("setting_service")
+        self.event_service: EventService = registry.get_instance("event_service")
+        self.util: Util = registry.get_instance("util")
+        self.bot: Tyrbot = registry.get_instance("bot")
+
+    def pre_start(self):
+        self.event_service.register_event_type("reload_translation")
+
+    def start(self):
+        self.setting_service.register_new("core.system", self.LANGUAGE_SETTING, "en_US",
+                                          TextSettingType(self.lang_codes), "Language of the Bot")
+
+        self.language = self.setting_service.get_value(self.LANGUAGE_SETTING)
+        self.register_translation("global", self.load_global_msg)
+        self.setting_service.register_change_listener(self.LANGUAGE_SETTING, self.language_setting_changed)
+
+    def register_translation(self, category, callback):
+        """
+        Call during start
+
+        Args:
+            category: str
+            callback: () -> {}
+        """
+
+        if len(inspect.signature(callback).parameters) != 0:
+            raise Exception(
+                "Incorrect number of arguments for handler '%s.%s()'" % (callback.__module__, callback.__name__))
+
+        if self.translation_callbacks.get(category) is None:
+            self.translation_callbacks[category] = []
+        self.translation_callbacks[category].append(callback)
+        self.update_msg(category, callback)
+
+    def load_global_msg(self):
+        with open("core/global.msg", mode="r", encoding="UTF-8") as f:
+            return hjson.load(f)
+
+    def language_setting_changed(self, name, old_value, new_value):
+        if name == self.LANGUAGE_SETTING and new_value != old_value:
+            self.reload_translation(new_value)
+
+    # This method will load another language, defined in the param 'lang'
+    def reload_translation(self, lang):
+        self.event_service.fire_event("reload_translation")
+        self.language = lang
+        for k1 in self.strings:
+            for callback in self.translation_callbacks.get(k1):
+                self.update_msg(k1, callback)
+
+    # updates the msgs
+    def update_msg(self, category, callback):
+        data = callback()
+        for k in data:
+            if category not in self.strings:
+                self.strings[category] = {}
+            self.strings[category][k] = data[k].get(self.language) or data[k].get("en_US")
+
+    #
+    # the param 'variables' accepts dictionaries ONLY.
+    #
+    def get_response(self, category, key, variables={}):
+        msg = ""
+        try:
+            val = self.strings[category][key]
+            if isinstance(val, list):
+                for line in val:
+                    msg += line.format(**variables)
+            else:
+                msg = val.format(**variables)
+        except KeyError as e:
+            self.logger.error(f"translating error category '{category}' and key '{key}' with params: {variables}", e)
+            msg = "Error translating category: <highlight>{mod}</highlight> key: <highlight>{key}</highlight>" \
+                  " with params: <highlight>{params}</highlight>".format(mod=category, key=key, params=variables)
+        finally:
+            return msg

core/tyrbot.py

@@ -0,0 +1,421 @@
+import inspect
+import threading
+import time
+
+from conf.config import BotConfig
+from core.aochat import server_packets, client_packets
+from core.aochat.extended_message import ExtendedMessage
+from core.bot_status import BotStatus
+from core.chat_blob import ChatBlob
+from core.conn import Conn
+from core.db import DB
+from core.decorators import instance
+from core.dict_object import DictObject
+from core.fifo_queue import FifoQueue
+from core.job_scheduler import JobScheduler
+from core.logger import Logger
+from core.lookup.character_service import CharacterService
+from core.public_channel_service import PublicChannelService
+from core.setting_service import SettingService
+from core.text import Text
+from modules.core.accounting.services.access_service import AccessService
+
+
+@instance("bot")
+class Tyrbot:
+    CONNECT_EVENT = "connect"
+    PACKET_EVENT = "packet"
+    PRIVATE_MSG_EVENT = "private_msg"
+
+    OUTGOING_ORG_MESSAGE_EVENT = "outgoing_org_message"
+    OUTGOING_PRIVATE_MESSAGE_EVENT = "outgoing_private_message"
+    OUTGOING_PRIVATE_CHANNEL_MESSAGE_EVENT = "outgoing_private_channel_message"
+
+    def __init__(self):
+        super().__init__()
+        self.logger = Logger(__name__)
+        self.ready = False
+        self.packet_handlers = {}
+        self.superadmin = []
+        self.status: BotStatus = BotStatus.SHUTDOWN
+        self.dimension = None
+        self.last_timer_event = 0
+        self.start_time = int(time.time())
+        self.major_version = "IGNCore v2.5"
+        self.minor_version = "1"
+        self.incoming_queue = FifoQueue()
+        self.mass_message_queue = None
+        self.conns = DictObject()
+
+    def inject(self, registry):
+        self.db = registry.get_instance("db")
+        self.character_service: CharacterService = registry.get_instance("character_service")
+        self.public_channel_service: PublicChannelService = registry.get_instance("public_channel_service")
+        self.text: Text = registry.get_instance("text")
+        self.setting_service: SettingService = registry.get_instance("setting_service")
+        self.access_service: AccessService = registry.get_instance("access_service")
+        self.event_service = registry.get_instance("event_service")
+        self.job_scheduler: JobScheduler = registry.get_instance("job_scheduler")
+        self.command_service = registry.get_instance("command_service")
+
+    def init(self, config: BotConfig, registry, mmdb_parser):
+        self.mmdb_parser = mmdb_parser
+        self.dimension = config.server.dimension
+        self.name = config.character
+        self.superadmin = config.superadmin
+        self.modules = [x.split("/")[1] for x in config.module_paths]
+        self.db.exec("CREATE TABLE IF NOT EXISTS command_config ("
+                     "command VARCHAR(50) NOT NULL, "
+                     "sub_command VARCHAR(50) NOT NULL, "
+                     "access_level VARCHAR(50) NOT NULL, "
+                     "channel VARCHAR(50) NOT NULL, "
+                     "module VARCHAR(50) NOT NULL, "
+                     "enabled SMALLINT NOT NULL, "
+                     "verified SMALLINT NOT NULL, "
+                     "PRIMARY KEY (`command`, `sub_command`, `access_level`, `channel`) USING BTREE)")
+        self.db.exec("CREATE TABLE IF NOT EXISTS event_config ("
+                     "event_type VARCHAR(50) NOT NULL, "
+                     "event_sub_type VARCHAR(50) NOT NULL, "
+                     "handler VARCHAR(255) NOT NULL, "
+                     "description VARCHAR(255) NOT NULL, "
+                     "module VARCHAR(50) NOT NULL, "
+                     "enabled SMALLINT NOT NULL, "
+                     "verified SMALLINT NOT NULL, "
+                     "is_hidden SMALLINT NOT NULL, "
+                     "PRIMARY KEY (`handler`) USING BTREE, "
+                     "INDEX `event_type` (`event_type`) USING BTREE, "
+                     "INDEX `event_sub_type` (`event_sub_type`) USING BTREE)")
+        self.db.exec("CREATE TABLE IF NOT EXISTS timer_event ("
+                     "event_type VARCHAR(50) NOT NULL, "
+                     "event_sub_type VARCHAR(50) NOT NULL, "
+                     "handler VARCHAR(255) NOT NULL, "
+                     "next_run INT NOT NULL, "
+                     "INDEX `next_run` (`next_run`) USING BTREE)")
+        self.db.exec("CREATE TABLE IF NOT EXISTS setting ("
+                     "name VARCHAR(50) NOT NULL, "
+                     "value VARCHAR(255) NOT NULL, "
+                     "description VARCHAR(255) NOT NULL, "
+                     "module VARCHAR(50) NOT NULL, "
+                     "verified SMALLINT NOT NULL, "
+                     "PRIMARY KEY (`name`) USING BTREE)")
+        self.db.exec("CREATE TABLE IF NOT EXISTS command_alias ("
+                     "alias VARCHAR(50) NOT NULL, "
+                     "command VARCHAR(1024) NOT NULL, "
+                     "enabled SMALLINT NOT NULL, "
+                     "INDEX `alias` (`alias`) USING BTREE, "
+                     "INDEX `command` (`command`) USING BTREE)")
+        self.db.exec("CREATE TABLE IF NOT EXISTS command_usage ("
+                     "command VARCHAR(255) NOT NULL, "
+                     "handler VARCHAR(255) NOT NULL, "
+                     "char_id INT NOT NULL, "
+                     "channel VARCHAR(20) NOT NULL, "
+                     "created_at INT NOT NULL, "
+                     "INDEX `command` (`command`) USING BTREE, "
+                     "INDEX `char_id` (`char_id`) USING BTREE, "
+                     "INDEX `channel` (`channel`) USING BTREE) ENGINE MEMORY")
+
+        # self.db.exec("UPDATE db_version SET verified = 0")
+        self.db.exec("UPDATE db_version SET verified = 1 WHERE file = 'db_version'")
+
+        # prepare commands, events, and settings
+        self.db.exec("UPDATE command_config SET verified = 0 where 1")
+        self.db.exec("UPDATE event_config SET verified = 0 where 1")
+        self.db.exec("UPDATE setting SET verified = 0 where 1")
+
+        # load modules
+        registry.pre_start_all()
+        registry.start_all()
+        if self.db.shared != self.db:
+            self.db: DB
+            self.db.shared.pool.close()
+            # Creates a Exception for some reason??
+            # self.db.shared = None
+
+        # remove commands, events, and settings that are no longer registered
+        # self.db.exec("DELETE FROM db_version WHERE verified = 0")
+        self.db.exec("DELETE FROM command_config WHERE verified = 0")
+        self.db.exec("DELETE FROM event_config WHERE verified = 0")
+        self.db.exec("DELETE FROM timer_event WHERE handler NOT IN "
+                     "(SELECT handler FROM event_config WHERE event_type = ?)", ["timer"])
+        self.db.exec("DELETE FROM setting WHERE verified = 0")
+
+        self.status = BotStatus.RUN
+
+    def pre_start(self):
+        self.event_service.register_event_type(self.CONNECT_EVENT)
+        self.event_service.register_event_type(self.PACKET_EVENT)
+        self.event_service.register_event_type(self.PRIVATE_MSG_EVENT)
+        self.event_service.register_event_type(self.OUTGOING_ORG_MESSAGE_EVENT)
+        self.event_service.register_event_type(self.OUTGOING_PRIVATE_MESSAGE_EVENT)
+        self.event_service.register_event_type(self.OUTGOING_PRIVATE_CHANNEL_MESSAGE_EVENT)
+
+    def start(self):
+        self.register_packet_handler(server_packets.PrivateMessage.id, self.handle_private_message, priority=40)
+
+    def connect(self, config):
+        conn = self.create_conn("main")
+        conn.connect(config.server.host, config.server.port)
+        packet = conn.login(config.username, config.password, config.character)
+        if not packet:
+            self.status = BotStatus.ERROR
+            return False
+        else:
+            self.incoming_queue.put((conn, packet))
+
+        self.create_conn_thread(conn)
+
+        if hasattr(config, 'slaves'):
+            self.mass_message_queue = FifoQueue()
+            for i, slave in enumerate(config.slaves):
+                conn = self.create_conn("slave" + str(i))
+                conn.connect(config.server.host, config.server.port)
+
+                packet = conn.login(slave.username, slave.password, slave.character)
+                if not packet:
+                    self.status = BotStatus.ERROR
+                    return False
+                else:
+                    self.incoming_queue.put((conn, packet))
+
+                self.create_conn_thread(conn, self.mass_message_queue)
+
+        return True
+
+    def create_conn_thread(self, conn: Conn, mass_message_queue=None):
+        def read_packets():
+            try:
+                while self.status == BotStatus.RUN:
+                    packet = conn.read_packet()
+                    if packet:
+                        self.incoming_queue.put((conn, packet))
+
+                    while mass_message_queue and not mass_message_queue.empty() and conn.packet_queue.is_empty():
+                        packet = mass_message_queue.get_or_default(block=False)
+                        if packet:
+                            conn.add_packet_to_queue(packet)
+
+            except (EOFError, OSError) as e:
+                self.status = BotStatus.ERROR
+                self.logger.error("", e)
+                raise e
+
+        dthread = threading.Thread(name=conn.char_name, target=read_packets, daemon=True)
+        dthread.start()
+
+    def create_conn(self, _id):
+        if _id in self.conns:
+            raise Exception(f"A connection with id {_id} already exists")
+
+        def failure_callback():
+            self.status = BotStatus.ERROR
+
+        conn = Conn(_id, failure_callback)
+        self.conns[_id] = conn
+        return conn
+
+    # passthrough
+    def send_packet(self, packet):
+        self.conns["main"].send_packet(packet)
+
+    def disconnect(self):
+        # wait for all threads to stop reading packets, then disconnect them all
+        time.sleep(2)
+        for _id, conn in self.conns.items():
+            conn.disconnect()
+
+    def run(self):
+        start = time.time()
+
+        # wait for flood of packets from login to stop sending
+        time_waited = 0
+        while time_waited < 5:
+            if not self.iterate(1):
+                time_waited += 1
+
+        self.logger.info("Login complete (%fs)" % (time.time() - start))
+
+        start = time.time()
+        self.event_service.fire_event("connect", None)
+        self.event_service.run_timer_events_at_startup()
+        self.logger.info("Connect events finished (%fs)" % (time.time() - start))
+        self.ready = True
+        self.command_service.ignore = []
+        timestamp = int(time.time())
+
+        while self.status == BotStatus.RUN:
+            try:
+                timestamp = int(time.time())
+                self.check_for_timer_events(timestamp)
+
+                self.iterate()
+            except Exception as e:
+                self.logger.error("", e)
+
+        # run any pending jobs/events
+        self.check_for_timer_events(timestamp + 1)
+
+        return self.status
+
+    def check_for_timer_events(self, timestamp):
+        # timer events will execute no more often than once per second
+        if self.last_timer_event < timestamp:
+            self.last_timer_event = timestamp
+            self.job_scheduler.check_for_scheduled_jobs(timestamp)
+            self.event_service.check_for_timer_events(timestamp)
+
+    def register_packet_handler(self, packet_id: int, handler, priority=50):
+        """
+        Call during pre_start
+
+        Args:
+            packet_id: int
+            handler: (conn, packet) -> void
+            priority: int
+        """
+
+        if len(inspect.signature(handler).parameters) != 2:
+            raise Exception(
+                "Incorrect number of arguments for handler '%s.%s()'" % (handler.__module__, handler.__name__))
+
+        handlers = self.packet_handlers.get(packet_id, [])
+        handlers.append(DictObject({"priority": priority, "handler": handler}))
+        self.packet_handlers[packet_id] = sorted(handlers, key=lambda x: x.priority)
+
+    def remove_packet_handler(self, packet_id, handler):
+        handlers = self.packet_handlers.get(packet_id, [])
+        for h in handlers:
+            if h.handler == handler:
+                handlers.remove(h)
+
+    def iterate(self, timeout=0.1):
+        conn, packet = self.incoming_queue.get_or_default(timeout=timeout, default=(None, None))
+        if packet:
+            if isinstance(packet, server_packets.SystemMessage):
+                packet = self.system_message_ext_msg_handling(packet)
+                self.logger.log_chat(conn.id, "SystemMessage", None, packet.extended_message.get_message())
+            elif isinstance(packet, server_packets.PublicChannelMessage):
+                packet = self.public_channel_message_ext_msg_handling(packet)
+            if isinstance(packet, server_packets.BuddyAdded):
+                if packet.char_id == 0:
+                    return
+
+            for handler in self.packet_handlers.get(packet.id, []):
+                handler.handler(conn, packet)
+
+            self.event_service.fire_event("packet:" + str(packet.id), packet)
+
+        return packet
+
+    def public_channel_message_ext_msg_handling(self, packet: server_packets.PublicChannelMessage):
+        msg = packet.message
+        if msg.startswith("~&") and msg.endswith("~"):
+            try:
+                msg = msg[2:-1].encode("utf-8")
+                category_id = self.mmdb_parser.read_base_85(msg[0:5])
+                instance_id = self.mmdb_parser.read_base_85(msg[5: 10])
+                template = self.mmdb_parser.get_message_string(category_id, instance_id)
+                params = self.mmdb_parser.parse_params(msg[10:])
+                packet.extended_message = ExtendedMessage(category_id, instance_id, template, params)
+            except Exception as e:
+                self.logger.error("Error handling extended message for packet: " + str(packet), e)
+        return packet
+
+    def system_message_ext_msg_handling(self, packet: server_packets.SystemMessage):
+        try:
+            category_id = 20000
+            instance_id = packet.message_id
+            template = self.mmdb_parser.get_message_string(category_id, instance_id)
+            params = self.mmdb_parser.parse_params(packet.message_args)
+            packet.extended_message = ExtendedMessage(category_id, instance_id, template, params)
+        except Exception as e:
+            self.logger.error("Error handling extended message: " + str(packet), e)
+        return packet
+
+    def send_org_message(self, msg, add_color=True, fire_outgoing_event=True, conn_id="main"):
+        org_channel_id = self.public_channel_service.org_channel_id
+        if org_channel_id is None:
+            self.logger.debug("ignoring message to org channel since the org_channel_id is unknown")
+        else:
+            color = self.setting_service.get("org_channel_color").get_font_color() if add_color else ""
+            pages = self.get_text_pages(msg, self.setting_service.get("org_channel_max_page_length").get_value())
+            for page in pages:
+                packet = client_packets.PublicChannelMessage(org_channel_id, color + page, "")
+                self.conns[conn_id].add_packet_to_queue(packet)
+
+            if fire_outgoing_event:
+                self.event_service.fire_event(self.OUTGOING_ORG_MESSAGE_EVENT,
+                                              DictObject({"org_channel_id": org_channel_id, "message": msg}))
+
+    def send_private_message(self, char_id, msg, add_color=True, fire_outgoing_event=True, conn_id="main"):
+        if char_id is None:
+            raise Exception("Cannot send message, char_id is empty")
+        else:
+            color = self.setting_service.get("private_message_color").get_font_color() if add_color else ""
+            pages = self.get_text_pages(msg, self.setting_service.get("private_message_max_page_length").get_value())
+            for page in pages:
+                # self.logger.log_tell(conn_id, "To", self.character_service.get_char_name(char_id), page)
+                packet = client_packets.PrivateMessage(char_id, color + page, "\0")
+                self.conns[conn_id].add_packet_to_queue(packet)
+
+            if fire_outgoing_event:
+                self.event_service.fire_event(self.OUTGOING_PRIVATE_MESSAGE_EVENT, DictObject({"char_id": char_id,
+                                                                                               "message": msg}))
+
+    def send_private_channel_message(self, msg, private_channel_id=None, add_color=True, fire_outgoing_event=True,
+                                     conn_id="main"):
+        if private_channel_id is None:
+            private_channel_id = self.get_char_id()
+        color = self.setting_service.get("private_channel_color").get_font_color() if add_color else ""
+        pages = self.get_text_pages(msg, self.setting_service.get("private_channel_max_page_length").get_value())
+        for page in pages:
+            packet = client_packets.PrivateChannelMessage(private_channel_id, color + page, "\0")
+            self.conns[conn_id].send_packet(packet)
+
+        if fire_outgoing_event and private_channel_id == self.get_char_id():
+            self.event_service.fire_event(self.OUTGOING_PRIVATE_CHANNEL_MESSAGE_EVENT,
+                                          DictObject({"private_channel_id": private_channel_id, "message": msg}))
+
+    def send_mass_message(self, char_id, msg, add_color=True, log_message=False):
+        # self.logger.log_tell('spam', 'To', self.character_service.get_char_name(char_id), msg)
+        if not char_id:
+            self.logger.warning("Could not send message to empty char_id")
+        if len(self.conns.items()) == 1:
+            self.send_private_message(char_id, msg, add_color, log_message)
+        else:
+            color = self.setting_service.get("private_message_color").get_font_color() if add_color else ""
+            pages = self.get_text_pages(msg, self.setting_service.get("private_message_max_page_length").get_value())
+            for page in pages:
+                if log_message:
+                    self.logger.log_tell("spam", "To", self.character_service.get_char_name(char_id), page)
+
+                if self.mass_message_queue:
+                    packet = client_packets.PrivateMessage(char_id, color + page, "\0")
+                    self.mass_message_queue.put(packet)
+                else:
+                    packet = client_packets.PrivateMessage(char_id, color + page, "spam")
+                    self.conns["main"].send_packet(packet)
+
+    def handle_private_message(self, conn: Conn, packet: server_packets.PrivateMessage):
+        # self.logger.log_tell(conn.id, "From", self.character_service.get_char_name(packet.char_id), packet.message)
+        self.event_service.fire_event(self.PRIVATE_MSG_EVENT, packet)
+
+    def get_text_pages(self, msg, max_page_length):
+        if isinstance(msg, ChatBlob):
+            return self.text.paginate(msg, max_page_length=max_page_length)
+        else:
+            return [self.text.format_message(msg)]
+
+    def is_ready(self):
+        return self.ready
+
+    def shutdown(self):
+        self.status = BotStatus.SHUTDOWN
+
+    def restart(self):
+        self.status = BotStatus.RESTART
+
+    def get_char_name(self):
+        return self.conns["main"].char_name
+
+    def get_char_id(self):
+        return self.conns["main"].char_id

core/upgrade.py

@@ -0,0 +1,68 @@
+from core.db import DB
+from core.logger import Logger
+from core.registry import Registry
+
+db = Registry.get_instance("db")
+logger = Logger("core.upgrade")
+
+
+def table_info(table_name):
+    if db.type == DB.MARIADB:
+        data = db.query("DESCRIBE %s" % table_name)
+
+        def normalize_table_info(row):
+            row.name = row.Field
+            row.type = row.Type.upper()
+            return row
+
+        return list(map(normalize_table_info, data))
+    else:
+        raise Exception("Unknown database type '%s'" % db.type)
+
+
+def table_exists(table_name):
+    # noinspection PyBroadException
+    try:
+        db.query(f"SELECT * FROM {table_name} LIMIT 1")
+        return True
+    except Exception:
+        return False
+
+
+def column_exists(table_name, column_name):
+    # noinspection PyBroadException
+    try:
+        db.query(f"SELECT {column_name} FROM {table_name} LIMIT 1")
+        return True
+    except Exception:
+        return False
+
+
+def update_version(v):
+    v += 1
+    logger.info("Upgrading db to version '%d'" % v)
+    db.exec("UPDATE db_version SET version = ? WHERE file = 'db_version' and bot =?", [v, db.name])
+    return v
+
+
+def get_version():
+    row = db.query_single("SELECT version FROM db_version WHERE file = 'db_version' and bot=?", [db.name])
+    if row:
+        return int(row.version)
+    else:
+        return 0
+
+
+def run_upgrades():
+    version = get_version()
+    logger.info("Database at version '%d'" % version)
+
+    if version == 0:
+        db.exec("INSERT INTO db_version (file, version, bot, verified) VALUES ('db_version', ?, ?, 1)", [0, db.name])
+        version = update_version(version)
+    db.create_view("db_version")
+    if version == 1:
+        if table_exists("account"):
+            if not column_exists("account", "auto_invite"):
+                db.exec("ALTER TABLE account ADD COLUMN auto_invite INT(2) default 0")
+        version = update_version(version)

core/usage_service.py

@@ -0,0 +1,18 @@
+import time
+
+from core.decorators import instance
+from core.logger import Logger
+
+
+@instance()
+class UsageService:
+    def __init__(self):
+        self.logger = Logger(__name__)
+
+    def inject(self, registry):
+        self.db = registry.get_instance("db")
+
+    def add_usage(self, command, handler, char_id, channel):
+        self.db.exec("INSERT INTO command_usage (command, handler, char_id, channel, created_at) "
+                     "VALUES (?, ?, ?, ?, ?)",
+                     [command, handler, char_id, channel, int(time.time())])

core/util.py

@@ -0,0 +1,293 @@
+import datetime
+import locale
+import math
+import re
+
+import pytz
+
+from core.decorators import instance
+from core.dict_object import DictObject
+from core.text import Text
+
+
+@instance()
+class Util:
+    budatime_full_regex = re.compile("^([0-9]+[a-z]+)+$", re.IGNORECASE)
+    budatime_unit_regex = re.compile("([0-9]+)([a-z]+)", re.IGNORECASE)
+
+    def __init__(self):
+        # needed for self.format_number() to work properly
+        locale.setlocale(locale.LC_NUMERIC, '')
+
+        self.abilities = [
+            "Agility",
+            "Intelligence",
+            "Psychic",
+            "Stamina",
+            "Strength",
+            "Sense"
+        ]
+
+        self.time_units = [
+            {
+                "units": ["yr", "years", "year", "y"],
+                "conversion_factor": 31536000
+            }, {
+                "units": ["month", "months", "mo"],
+                "conversion_factor": 2592000
+            }, {
+                "units": ["week", "weeks", "w"],
+                "conversion_factor": 604800
+            }, {
+                "units": ["day", "days", "d"],
+                "conversion_factor": 86400
+            }, {
+                "units": ["hr", "hours", "hour", "hrs", "h"],
+                "conversion_factor": 3600
+            }, {
+                "units": ["min", "mins", "m"],
+                "conversion_factor": 60
+            }, {
+                "units": ["sec", "secs", "s"],
+                "conversion_factor": 1
+            }
+        ]
+
+    def inject(self, registry):
+        self.text: Text = registry.get_instance("text")
+
+    def get_handler_name(self, handler):
+        return handler.__module__ + "." + handler.__qualname__
+
+    def get_module_name(self, handler):
+        handler_name = self.get_handler_name(handler)
+        parts = handler_name.split(".")
+        return parts[1] + "." + parts[2]
+
+    def parse_time(self, budatime, default=0):
+        unixtime = 0
+
+        if not self.budatime_full_regex.search(budatime):
+            return default
+
+        matches = self.budatime_unit_regex.finditer(budatime)
+
+        for match in matches:
+            for time_unit in self.time_units:
+                if match.group(2).lower() in time_unit["units"]:
+                    unixtime += int(match.group(1)) * time_unit["conversion_factor"]
+                    continue
+
+        return unixtime
+
+    def time_to_readable(self, unixtime, min_unit="sec", max_unit="day", max_levels=2):
+        if unixtime == 0:
+            return "0 secs"
+
+        # handle negative as positive, and add negative sign at the end
+        is_negative = False
+        if unixtime < 0:
+            is_negative = True
+            unixtime *= -1
+
+        found_max_unit = False
+        time_shift = ""
+        levels = 0
+        for time_unit in self.time_units:
+            unit = time_unit["units"][0]
+
+            if max_unit in time_unit["units"]:
+                found_max_unit = True
+
+            # continue to skip until we have found the max unit
+            if not found_max_unit:
+                continue
+
+            unit_value = math.floor(unixtime / time_unit["conversion_factor"])
+
+            if unit_value == 0:
+                # do not show units where unit_value is 0
+                pass
+            elif unit_value == 1:
+                # show singular where unit_value is 1
+                time_shift += str(unit_value) + " " + unit + " "
+            else:
+                # show plural where unit_value is greater than 1
+                time_shift += str(unit_value) + " " + unit + "s "
+
+            unixtime %= time_unit["conversion_factor"]
+
+            # record level after the first a unit has a length
+            if levels or unit_value >= 1:
+                levels += 1
+
+            if levels == max_levels:
+                break
+
+            # if we have reached the min unit, then break, unless we have no output, in which case we continue
+            if time_shift and min_unit in time_unit["units"]:
+                break
+
+        return ("-" if is_negative else "") + time_shift.strip()
+
+    def get_ability(self, ability_str):
+        ability_str = ability_str.capitalize()
+        for ability in self.abilities:
+            if ability.startswith(ability_str):
+                return ability
+        return None
+
+    def get_all_abilities(self):
+        return self.abilities.copy()
+
+    def get_title_level(self, level):
+        if level < 5:
+            return 0
+        elif level < 15:
+            return 1
+        elif level < 50:
+            return 2
+        elif level < 100:
+            return 3
+        elif level < 150:
+            return 4
+        elif level < 190:
+            return 5
+        elif level < 205:
+            return 6
+        else:
+            return 7
+
+    def get_level_range_tl(self, tl):
+        if tl == 0:
+            return 0, 4
+        elif tl == 1:
+            return 5, 14
+        elif tl == 2:
+            return 15, 49
+        elif tl == 3:
+            return 50, 99
+        elif tl == 4:
+            return 100, 149
+        elif tl == 5:
+            return 150, 189
+        elif tl == 6:
+            return 190, 204
+        elif tl == 7:
+            return 205, 300
+        else:
+            return 0, 0
+
+    def format_number(self, number):
+        return locale.format_string("%.*f", (0, number), grouping=True)
+
+    def get_profession(self, search, long=True):
+        search = search.lower()
+
+        if search in ["adv", "advy", "adventurer"]:
+            return "Adventurer" if long else "Adv"
+        elif search in ["agent"]:
+            return "Agent" if long else "Agent"
+        elif search in ["crat", "bureaucrat"]:
+            return "Bureaucrat" if long else "Crat"
+        elif search in ["doc", "doctor"]:
+            return "Doctor" if long else "Doc"
+        elif search in ["enf", "enfo", "enforcer"]:
+            return "Enforcer" if long else "Enf"
+        elif search in ["eng", "engi", "engy", "engineer"]:
+            return "Engineer" if long else "Eng"
+        elif search in ["fix", "fixer"]:
+            return "Fixer" if long else "Fix"
+        elif search in ["keep", "keeper"]:
+            return "Keeper" if long else "Keep"
+        elif search in ["ma", "martial", "martialartist", "martial artist"]:
+            return "Martial Artist" if long else "MA"
+        elif search in ["mp", "meta", "metaphysicist", "meta-physicist"]:
+            return "Meta-Physicist" if long else "MP"
+        elif search in ["nt", "nano", "nanotechnician", "nano-technician"]:
+            return "Nano-Technician" if long else "NT"
+        elif search in ["sha", "shade"]:
+            return "Shade" if long else "Shade"
+        elif search in ["sol", "sold", "soldier"]:
+            return "Soldier" if long else "Sold"
+        elif search in ["tra", "trad", "trader"]:
+            return "Trader" if long else "Trader"
+        elif search in ["gen", "general"]:
+            return "General" if long else "UKN"
+        else:
+            return None
+
+    def get_prof_icon(self, search, large=False):
+        large = "_LARGE" if large else ""
+
+        search = self.get_profession(search)
+        if search == "Adventurer":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_6", "tdb")
+        elif search == "Agent":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_5", "tdb")
+        elif search == "Bureaucrat":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_8", "tdb")
+        elif search == "Doctor":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_10", "tdb")
+        elif search == "Enforcer":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_9", "tdb")
+        elif search == "Engineer":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_3", "tdb")
+        elif search == "Fixer":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_4", "tdb")
+        elif search == "Keeper":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_14", "tdb")
+        elif search == "Martial Artist":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_2", "tdb")
+        elif search == "Meta-Physicist":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_12", "tdb")
+        elif search == "Nano-Technician":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_11", "tdb")
+        elif search == "Shade":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_15", "tdb")
+        elif search == "Soldier":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_1", "tdb")
+        elif search == "Trader":
+            return self.text.make_image(f"id:GFX_GUI_ICON_PROFESSION{large}_7", "tdb")
+        else:
+            return ""
+
+    def get_all_professions(self):
+        return ["Adventurer", "Agent", "Bureaucrat", "Doctor", "Enforcer", "Engineer", "Fixer", "Keeper",
+                "Martial Artist", "Meta-Physicist", "Nano-Technician", "Shade", "Soldier", "Trader"]
+
+    def format_date(self, timestamp):
+        value = datetime.datetime.fromtimestamp(timestamp, tz=pytz.UTC)
+        return value.strftime('%Y-%m-%d')
+
+    def format_time(self, timestamp):
+        value = datetime.datetime.fromtimestamp(timestamp, tz=pytz.UTC)
+        return value.strftime('%H:%M:%S')
+
+    def format_datetime(self, timestamp):
+        value = datetime.datetime.fromtimestamp(timestamp, tz=pytz.UTC)
+        return value.strftime('%Y-%m-%d %H:%M:%S %Z')
+
+    def interpolate_value(self, interpolated_ql, interpolation_ranges, precision=0):
+        min_val = None
+        max_val = None
+        for ql, v in interpolation_ranges.items():
+            # required to avoid division by zero
+            if ql == interpolated_ql:
+                return v
+
+            if interpolated_ql >= ql and (not min_val or ql > min_val.ql):
+                min_val = DictObject({"ql": ql, "val": v})
+
+            if interpolated_ql <= ql and (not max_val or ql < max_val.ql):
+                max_val = DictObject({"ql": ql, "val": v})
+
+        if not min_val or not max_val:
+            return None
+
+        return round((max_val.val - min_val.val) / (max_val.ql - min_val.ql) *
+                     (interpolated_ql - min_val.ql) + min_val.val, precision)
+
+    # noinspection PyMethodMayBeStatic
+    def get_offset_limit(self, page_size, page_number):
+        return (page_number - 1) * page_size, page_size