From bb445e868dbb89f847fb1ce65beede4a0e839698 Mon Sep 17 00:00:00 2001 From: Minidodo Date: Mon, 18 Oct 2021 19:39:14 +0200 Subject: [PATCH] Reduced & updated dependencies. By default, messages larger than 2.000 characters (sent via tells) get sent via slaves, if these exist. Fixes #3 --- bootstrap.py | 2 +- core/chat_blob.py | 3 + core/command_service.py | 6 +- core/db.py | 98 +++++++++---------- core/igncore.py | 4 +- core/lookup/pork_service.py | 2 - dummy.start.sh | 16 +-- modules/core/accounting/account_controller.py | 14 ++- modules/core/discord/discord_controller.py | 2 + .../private_channel_controller.py | 5 +- modules/core/riadmin/riadmin_controller.py | 5 +- modules/onlinebot/online/online_controller.py | 2 +- modules/relaybot/online/online_controller.py | 2 +- modules/standard/aou/aou_controller.py | 7 +- modules/standard/news/mail_controller.py | 18 ++-- modules/standard/news/news_controller.py | 35 ------- modules/standard/timers/timer_controller.py | 2 +- .../whois/character_info_controller.py | 2 +- requirements.txt | 28 +++--- update.sh | 23 +++++ 20 files changed, 124 insertions(+), 152 deletions(-) create mode 100644 update.sh diff --git a/bootstrap.py b/bootstrap.py index 690994a..2d8e8cc 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -50,7 +50,7 @@ try: mod = __import__(f'conf.{bot}', fromlist=['BotConfig']) config: BotConfig = getattr(mod, 'BotConfig') if sys.version_info < (3, 9): - logger.warning("Versions lower then Python3.9 are not supported.") + logger.warning("Versions lower then Python3.10 are not supported.") exit(-1) # load config values from env vars diff --git a/core/chat_blob.py b/core/chat_blob.py index 5544e23..aeb5431 100644 --- a/core/chat_blob.py +++ b/core/chat_blob.py @@ -17,3 +17,6 @@ class ChatBlob: obj.msg == self.msg and \ obj.page_prefix == self.page_prefix and \ obj.page_postfix == self.page_postfix + + def __len__(self): + return len(self.title + self.msg + self.page_postfix + self.page_prefix) \ No newline at end of file diff --git a/core/command_service.py b/core/command_service.py index 9324800..eafec80 100644 --- a/core/command_service.py +++ b/core/command_service.py @@ -243,7 +243,11 @@ class CommandService: response = handler["callback"](CommandRequest(conn, channel, sender, reply), *self.process_matches(matches, handler["params"])) if response is not None: - reply(response) + if len(response) > 2000 and conn.id == "main" and channel == "msg": + self.bot.send_mass_message(sender.char_id, response) + else: + reply(response) + except Exception as e: self.logger.error(f"error processing command: {message}", e) self.relay_hub_service.send_message("access_denied_logger", sender, diff --git a/core/db.py b/core/db.py index 66673e9..2d61033 100644 --- a/core/db.py +++ b/core/db.py @@ -7,7 +7,6 @@ import time import mariadb # noinspection PyProtectedMember from mariadb._mariadb import ConnectionPool, OperationalError -from mysql.connector.cursor import CursorBase from pkg_resources import parse_version from conf.config import BotConfig @@ -75,8 +74,8 @@ class DB: if string.__contains__("UPDATE ") or string.__contains__("INSERT "): conn.commit() except Exception as e: - raise SqlException( f"SQL Error: '{str(e)}' for '{sql}' " - f"[{', '.join(map(lambda x: str(x), params))}]") from e + raise SqlException(f"SQL Error: '{str(e)}' for '{sql}' " + f"[{', '.join(map(lambda x: str(x), params))}]") from e elapsed = time.time() - start_time result = callback(cur) if elapsed > 5: @@ -217,56 +216,53 @@ class DB: if pre_optimized: self._load_optimized_file(filename) return - start = time.time() + raise SqlException(f"SQL File not optimized: {filename}") + # start = time.time() # Short version... instead of executing 90 000 inserts for the itemDB, just do one, # while providing all values during one query - with open(filename, mode="r", encoding="UTF-8") as f: - insert_batches = [] - inserts = [] - others = [] - stat = "" - for i in f.readlines(): - i = i.strip() - if i == "" or i == " ": - continue - if i.startswith("INSERT INTO"): - match2 = re.match("(INSERT INTO .+? VALUES) (\(.+?\));", i) - if match2: - r2 = match2[2].replace("NULL", "None") - r2 = r2.replace("null", "None") - query = match2[1] + f" ({', ?' * len(eval(r2))})" - query = query.replace("(, ", "(") - if stat != query: - if stat != "" or len(inserts) != 0: - insert_batches.append([stat, inserts]) - inserts = [] - stat = query - inserts.append(eval(r2)) - else: - if i.startswith("--"): - continue - others.append(i) - insert_batches.append([stat, inserts]) - with self.shared.lock: - with self.shared.pool.get_connection() as conn: - with conn.cursor() as cur: - cur: CursorBase - if others: - for statement in others: - try: - cur.execute(statement) - except OperationalError: - pass - for sql, param in insert_batches: - if sql == "INSERT INTO trickle (id, group_name, name, amount_agility, " \ - "amount_intelligence, amount_psychic, amount_stamina, " \ - "amount_strength, amount_sense) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)": - for row in param: - cur.execute(sql, row) - continue - cur.executemany(sql, param) - - print(f"Runtime: {time.time() - start: .2f} for {filename}") + # with open(filename, mode="r", encoding="UTF-8") as f: + # insert_batches = [] + # inserts = [] + # others = [] + # stat = "" + # for i in f.readlines(): + # i = i.strip() + # if i == "" or i == " ": + # continue + # if i.startswith("INSERT INTO"): + # match2 = re.match("(INSERT INTO .+? VALUES) (\(.+?\));", i) + # if match2: + # r2 = match2[2].replace("NULL", "None") + # r2 = r2.replace("null", "None") + # query = match2[1] + f" ({', ?' * len(eval(r2))})" + # query = query.replace("(, ", "(") + # if stat != query: + # if stat != "" or len(inserts) != 0: + # insert_batches.append([stat, inserts]) + # inserts = [] + # stat = query + # inserts.append(eval(r2)) + # else: + # if i.startswith("--"): + # continue + # others.append(i) + # insert_batches.append([stat, inserts]) + # with self.shared.lock: + # with self.shared.pool.get_connection() as conn: + # with conn.cursor() as cur: + # cur: CursorBase + # if others: + # for statement in others: + # cur.execute(statement) + # for sql, param in insert_batches: + # if sql == "INSERT INTO trickle (id, group_name, name, amount_agility, " \ + # "amount_intelligence, amount_psychic, amount_stamina, " \ + # "amount_strength, amount_sense) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)": + # for row in param: + # cur.execute(sql, row) + # continue + # cur.executemany(sql, param) + # print(f"Runtime: {time.time() - start: .2f} for {filename}") def get_type(self) -> str: return self.type diff --git a/core/igncore.py b/core/igncore.py index 45a704b..aa257c1 100644 --- a/core/igncore.py +++ b/core/igncore.py @@ -41,8 +41,8 @@ class IgnCore: self.dimension = None self.last_timer_event = 0 self.start_time = int(time.time()) - self.major_version = "IGNCore v2.6" - self.minor_version = "7" + self.major_version = "IGNCore v2.7" + self.minor_version = "0" self.incoming_queue = FifoQueue() self.mass_message_queue = None self.conns = DictObject() diff --git a/core/lookup/pork_service.py b/core/lookup/pork_service.py index 930cb27..720ab20 100644 --- a/core/lookup/pork_service.py +++ b/core/lookup/pork_service.py @@ -1,7 +1,6 @@ import time import requests -from mysql.connector.cursor import CursorBase from requests import ReadTimeout from core.aochat import server_packets @@ -233,7 +232,6 @@ class PorkService: with self.db.pool.get_connection() as conn: with conn.cursor(dictionary=True) as cur: for packet in self.updates: - cur: CursorBase cur.execute( "SELECT char_id, name, first_name, last_name, level, breed, gender, faction, profession, " "profession_title, ai_rank, ai_level, org_id, org_name, org_rank_name, org_rank_id, " diff --git a/dummy.start.sh b/dummy.start.sh index 93ff14e..1b5dd94 100644 --- a/dummy.start.sh +++ b/dummy.start.sh @@ -6,26 +6,12 @@ if ! [ -x "$(command -v $PYTHON_BINARY)" ]; then fi $PYTHON_BINARY --version - -# Ensure virtualenv is present. This is not always the case -$PYTHON_BINARY -m pip install virtualenv --user - -# Create and activate the virtualenv. This can be done even if it already exists -# and will ensure setuptools, wheel and pip are up to date -$PYTHON_BINARY -m virtualenv venv source venv/bin/activate - -# From there on we use 'pip' and 'python' (refers to versions in the virtualenv) -pip install -r requirements.txt - set -o pipefail -o errexit # The bot uses non-zero exit codes to signal state. # The bot will restart until it returns an exit code of zero. while true; do - # shellcheck disable=SC1073 - # shellcheck disable=SC1072 - # shellcheck disable=SC1009 - python bootstrap.py ##bot_name## && exit + python bootstrap.py "$1" && exit sleep 1 done diff --git a/modules/core/accounting/account_controller.py b/modules/core/accounting/account_controller.py index a02a921..0359ecb 100644 --- a/modules/core/accounting/account_controller.py +++ b/modules/core/accounting/account_controller.py @@ -36,8 +36,7 @@ class AccountController: @command(command="account", params=[], access_level="member", description="View your account") def account(self, request: command_request): - out = self.show_account(request.sender.char_id) - self.bot.send_mass_message(request.sender.char_id, out) + return self.show_account(request.sender.char_id) @command(command="accounts", params=[Const('online', is_optional=True)], access_level="moderator", description="View all accounts") @@ -238,6 +237,10 @@ class AccountController: if not mod: response += f" Options: {prefs}\n" response += f" Points: {alts[0].points}\n" + last_seen = {} + for x in alts: + if x.last_seen > last_seen.get("last_seen", 0): + last_seen = x access_levels = {f"Member ({alts[0].member})": self.account_service.check_member(alts[0].char_id), "Officer": self.account_service.check_officer(alts[0].char_id), "General": self.account_service.check_general(alts[0].char_id), @@ -252,12 +255,13 @@ class AccountController: perms.append(key) response += f" Status: {'Open' if alts[0].disabled == 0 else 'Closed'}\n" response += f" Created at: {self.util.format_datetime(alts[0].created)}\n" + response += f" Last seen on {last_seen.name} {self.util.time_to_readable(time.time()-last_seen.last_seen)} ago\n" response += f" Permissions: {', '.join(perms)}\n" if alts[0].discord_joined == 1: - joined = '(Joined server)' + joined = ' (Joined server)' else: - joined = '(Left server)' if alts[0].discord_invite != '' else 'Never joined' - response += f" Discord: {alts[0].discord_handle} {joined}\n\n" + joined = ' (Left server)' if alts[0].discord_invite != '' else 'Never joined' + response += f" Discord: {alts[0].discord_handle}{joined}\n\n" log_types = ['Points', 'Loot', 'Raid', 'Public', 'Admin', 'System'] response += " Logs: [" for i in log_types: diff --git a/modules/core/discord/discord_controller.py b/modules/core/discord/discord_controller.py index 99bd550..5f05c32 100644 --- a/modules/core/discord/discord_controller.py +++ b/modules/core/discord/discord_controller.py @@ -580,6 +580,8 @@ class DiscordController: async def on_message(self, msg: Message): if msg.author.id == self.client.user.id: return + if not self.setting_service.get_value("dc_relay_public"): + return channel: TextChannel = msg.channel if channel.id == int(self.setting_service.get_value("dc_relay_public")): if self.setting_service.get_value('is_alliance_bot') == "0": diff --git a/modules/core/private_channel/private_channel_controller.py b/modules/core/private_channel/private_channel_controller.py index fe37962..382c313 100644 --- a/modules/core/private_channel/private_channel_controller.py +++ b/modules/core/private_channel/private_channel_controller.py @@ -141,10 +141,7 @@ class PrivateChannelController: if len(banned) > 0: out += f'The Character {", ".join(banned)} is banned, ' \ f'and cannot be invited.' - if request.channel == "priv": - return out.strip("\n") - else: - self.bot.send_mass_message(request.sender.char_id, out.strip("\n")) + return out.strip("\n") @command(command="kick", params=[Character("character")], access_level="moderator", description="Kick a character from the private channel") diff --git a/modules/core/riadmin/riadmin_controller.py b/modules/core/riadmin/riadmin_controller.py index 95c07f3..363fdb9 100644 --- a/modules/core/riadmin/riadmin_controller.py +++ b/modules/core/riadmin/riadmin_controller.py @@ -1,3 +1,4 @@ +import json import math from core.buddy_service import BuddyService @@ -165,7 +166,7 @@ class RIAdminController: description="take the RI from another bot") def ritake(self, request, raiders: str): users = raiders.split(",") - if request.sender.char_id not in eval(self.setting_service.get_value('riadmin_network')): + if request.sender.char_id not in json.loads(self.setting_service.get_value('riadmin_network')): return for user in users: char = self.character_service.resolve_char_to_id(user.strip()) @@ -277,7 +278,7 @@ class RIAdminController: def riadmin_create(self, _, _2, raid_instance_name, bot): if not bot.char_id: return self.getresp("global", "char_not_found", {"char": bot.name}) - if bot.char_id not in eval(self.setting_service.get_value('riadmin_network')): + if bot.char_id not in json.loads(self.setting_service.get_value('riadmin_network')): return "Bot not valid: please ask an Administrator to verify it first." raid_instance = self.get_raid_instance(raid_instance_name) if raid_instance: diff --git a/modules/onlinebot/online/online_controller.py b/modules/onlinebot/online/online_controller.py index 16a05ab..52466ae 100644 --- a/modules/onlinebot/online/online_controller.py +++ b/modules/onlinebot/online/online_controller.py @@ -32,4 +32,4 @@ class OrgOnlineController(OnlineController): blob = self.online_display.format_by_channel_prof(query, params) else: blob = self.online_display.format_by_org(query, params) - self.bot.send_mass_message(request.sender.char_id, self.online_display.format_blob(blob)) + return self.online_display.format_blob(blob) diff --git a/modules/relaybot/online/online_controller.py b/modules/relaybot/online/online_controller.py index 20681eb..11e5c67 100644 --- a/modules/relaybot/online/online_controller.py +++ b/modules/relaybot/online/online_controller.py @@ -30,7 +30,7 @@ class OrgOnlineController(OnlineController): query = "and channel_id IN (1, 2) " params = [self.bot.name, self.bot.get_char_id()] blob = self.online_display.format_by_channel_prof(query, params) - self.bot.send_mass_message(request.sender.char_id, self.online_display.format_blob(blob)) + return self.online_display.format_blob(blob) def pre_start(self): super().pre_start() diff --git a/modules/standard/aou/aou_controller.py b/modules/standard/aou/aou_controller.py index 6b7fbae..826e784 100644 --- a/modules/standard/aou/aou_controller.py +++ b/modules/standard/aou/aou_controller.py @@ -85,8 +85,7 @@ class AOUController: obj.aou = self.text.make_chatcmd("AO-Universe.com", "/start https://www.ao-universe.com") obj.text = self.format_bbcode_code(guide_info.text) - self.bot.send_mass_message(request.sender.char_id, - ChatBlob(guide_info.name, self.getresp("module/aou", "guide", {**obj}))) + return ChatBlob(guide_info.name, self.getresp("module/aou", "guide", {**obj})) @command(command="aou", params=[Const("all", is_optional=True), Any("search")], access_level="member", description="Search for an AO-Universe guides") @@ -117,9 +116,9 @@ class AOUController: if count == 0: return self.getresp("module/aou", "no_guide_search", {"search": search}) else: - self.bot.send_mass_message(request.sender.char_id, ChatBlob( + return ChatBlob( self.getresp("module/aou", "search_guide_title" + ("_all" if include_all_matches else ""), - {"search": search, "count": count}), blob)) + {"search": search, "count": count}), blob) def retrieve_guide(self, guide_id): cache_key = "%d.xml" % guide_id diff --git a/modules/standard/news/mail_controller.py b/modules/standard/news/mail_controller.py index c87d476..29224e9 100644 --- a/modules/standard/news/mail_controller.py +++ b/modules/standard/news/mail_controller.py @@ -74,18 +74,16 @@ class MailController: @command(command="mail", params=[Const('all', is_optional=True)], description="Show your mails", access_level="member") def mail_show(self, sender, const_all): + mails, title = None, "You dont have any unread mails." + if const_all: - mails = self.get_mails(sender.sender.char_id, True) - if mails: - self.bot.send_private_message(sender.sender.char_id, ChatBlob("All your recent Mails", mails)) - else: - return "You dont have any mails." + mails, title = self.get_mails(sender.sender.char_id, True), "All your recent Mails" else: - mails = self.get_mails(sender.sender.char_id) - if mails: - self.bot.send_private_message(sender.sender.char_id, ChatBlob("Your unread mails", mails)) - else: - return "You dont have any unread mails." + mails, title = self.get_mails(sender.sender.char_id), "Your unread mails" + if mails: + self.bot.send_private_message(sender.sender.char_id, ChatBlob(title, mails)) + else: + return title @command(command="mail", params=[Options(["read"]), Int("ID")], description="mark a mail as read", access_level="member") diff --git a/modules/standard/news/news_controller.py b/modules/standard/news/news_controller.py index 1f22ee2..9f354c8 100644 --- a/modules/standard/news/news_controller.py +++ b/modules/standard/news/news_controller.py @@ -85,37 +85,6 @@ hh:mm - DD.MM.YYYY f" [{self.text.make_chatcmd('orgs', '/tell orgs')}] " \ f"[{self.text.make_chatcmd('admins', '/tell admins')}]\n" - @event("connect", "prepare info msg") - def prepare_info(self, _, _1): - self.INFO = ChatBlob("Welcome", f"You're getting this message,\n" - f"because you recently joined the clan alliance\n" - f"The New Alliance or in short " - f"TNA.\n" - f"I'm , a bot with the task to " - f"automate a few workflows around here,\n" - f"while making the life of everyone a bit easier.\n" - f"\n" - f"You can see a list of all my orgs by using " - f"{self.text.make_tellcmd('!orgs', 'orgs')}.\n" - f"Whenever you use {self.text.make_tellcmd('!online', 'online')} " - f"I'll show you who's currently around.\n" - f"We've got regular raids, you can view them by using " - f"{self.text.make_tellcmd('!raids', 'raids')}\n" - f"If you'd like to stay out, thats fine. Just turn all settings off here: " - f"{self.text.make_tellcmd('here', 'prefs')}\n" - f"\n" - f"If you encounter any issues, my administrators will help you. just ask them, " - f"you can find them all here: {self.text.make_tellcmd('!admins', 'admins')}\n" - f"\n" - f"Thats it already. " - f"Thank you for taking your time to read this note. See you ingame!" - f"") - self.INFO = f"\n________________\n\n" \ - f" No need to PANIC!\n" \ - f" [{self.text.paginate_single(self.INFO)}]\n" \ - f" to our alliance!\n" \ - f"________________" - @event(event_type="member_logon", description="Send news to players logging in") def logon_event(self, _, data): if not self.bot.is_ready(): @@ -140,10 +109,6 @@ hh:mm - DD.MM.YYYY discord, self.preferences.get_pref_view_small(account)) - # This one is kinda redudant now that the simple checks get done above, will remove it in the future. - if account.last_seen == 0 and self.setting_service.get_value('is_alliance_bot') == "1": - self.bot.send_mass_message(data.packet.char_id, self.INFO) - @command(command="raids", params=[], description="Show the Raids", access_level="member") def show_raids(self, _): with open("data/latest_raids.txt", "r") as f: diff --git a/modules/standard/timers/timer_controller.py b/modules/standard/timers/timer_controller.py index ee74fac..9d3c8bd 100644 --- a/modules/standard/timers/timer_controller.py +++ b/modules/standard/timers/timer_controller.py @@ -89,7 +89,7 @@ class TimerController: blob += "\n\n
Manual Timers
\n" for timer in data: repeats = "" - if timer.repeating_every >= 0: + if timer.repeating_every > 0: repeats = f" (Repeats every {self.util.time_to_readable(timer.repeating_every)})" blob += f"Name: {timer.name}\n" blob += f"Time left: " \ diff --git a/modules/standard/whois/character_info_controller.py b/modules/standard/whois/character_info_controller.py index dc54ee2..25e0eee 100644 --- a/modules/standard/whois/character_info_controller.py +++ b/modules/standard/whois/character_info_controller.py @@ -120,7 +120,7 @@ class CharacterInfoController: msg = ChatBlob(f"Basic Info for {char.name}", blob) else: msg = f"Could not find character {char.name} on RK{dimension:d}." - if request.channel == "msg": + if request.channel == "msg" and request.conn.id == "main": self.bot.send_mass_message(request.sender.char_id, msg) else: request.reply(msg) diff --git a/requirements.txt b/requirements.txt index b01b756..401bb23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,13 @@ -beautifulsoup4~=4.9.3 -cryptography~=3.3.2 +bbcode==1.1.0 +beautifulsoup4==4.10.0 +cryptography==3.3.2 discord.py @ git+https://github.com/Rapptz/discord.py@45d498c1b76deaf3b394d17ccf56112fa691d160 -emojis~=0.6.0 -hjson~=3.0.2 -mariadb~=1.0.6 -mysql-connector-python~=8.0.25 -psutil~=5.8.0 -pytz~=2021.1 -requests~=2.25.1 -websocket-client~=1.1.0 -bbcode~=1.1.0 -websock~=1.0.4 -pip>=21.2.4 -websockets~=9.1 -torpy~=1.1.6 -urllib3==1.25.11 +emojis==0.6.0 +hjson==3.0.2 +mariadb==1.0.7 +psutil==5.8.0 +pytz==2021.3 +requests==2.26.0 +torpy==1.1.6 +websock==1.0.4 +websocket-client==1.2.1 \ No newline at end of file diff --git a/update.sh b/update.sh new file mode 100644 index 0000000..96b82ec --- /dev/null +++ b/update.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +git pull + +PYTHON_BINARY=python3 +if ! [ -x "$(command -v $PYTHON_BINARY)" ]; then + PYTHON_BINARY=python +fi + +$PYTHON_BINARY --version + +# Ensure virtualenv is present. This is not always the case +$PYTHON_BINARY -m pip install virtualenv --user + +# Create and activate the virtualenv. This can be done even if it already exists +# and will ensure setuptools, wheel and pip are up to date +$PYTHON_BINARY -m virtualenv venv +source venv/bin/activate + +# From there on we use 'pip' and 'python' (refers to versions in the virtualenv) +pip install -r requirements.txt + +echo "Update completed."