Initial Release of IGNCore version 2.5

This commit is contained in:
2021-08-09 13:18:56 +02:00
commit a83d98c47e
910 changed files with 224171 additions and 0 deletions
+67
View File
@@ -0,0 +1,67 @@
import json
import time
import requests
from requests import ReadTimeout
from torpy.http.requests import do_request
from core.db import DB
from core.decorators import instance
from core.dict_object import DictObject
from core.logger import Logger
@instance()
class CharacterHistoryService:
CACHE_GROUP = "history"
CACHE_MAX_AGE = 86400
users = []
def __init__(self):
self.logger = Logger(__name__)
def inject(self, registry):
self.bot = registry.get_instance("bot")
self.db: DB = registry.get_instance("db")
self.cache_service = registry.get_instance("cache_service")
def get_character_history(self, name, server_num):
cache_key = "%s.%d.json" % (name, server_num)
t = int(time.time())
# check cache for fresh value
cache_result = self.cache_service.retrieve(self.CACHE_GROUP, cache_key)
if cache_result and cache_result.last_modified > (t - self.CACHE_MAX_AGE):
result = json.loads(cache_result.data)
else:
url = self.get_pork_url(server_num, name)
try:
r = do_request(url)
# with TorRequests() as tor_request:
# with tor_request.get_session(1) as session:
# r = session.get(url, timeout=5)
r = requests.get(url, timeout=5, headers={"User-Agent": self.bot.major_version})
result = r.json()
except ReadTimeout:
self.logger.warning("Timeout while requesting '%s'" % url)
result = None
except Exception as e:
self.logger.error("Error requesting history for url '%s'" % url, e)
result = None
if result:
# store result in cache
self.cache_service.store(self.CACHE_GROUP, cache_key, json.dumps(result))
elif cache_result:
# check cache for any value, even expired
result = json.loads(cache_result.data)
if result:
return map(lambda x: DictObject(x), result)
else:
return None
def get_pork_url(self, dimension, char_name):
# noinspection HttpUrlsUsage
return f"http://pork.budabot.jkbff.com/pork/history.php?server={dimension:d}&name={char_name}"
+99
View File
@@ -0,0 +1,99 @@
import threading
import time
from core.aochat import server_packets
from core.aochat.client_packets import CharacterLookup
from core.decorators import instance
@instance()
class CharacterService:
def __init__(self):
self.name_to_id = {}
self.id_to_name = {}
self.waiting_for_response = set()
self.notify_on_receive = {}
def inject(self, registry):
self.bot = registry.get_instance("bot")
self.db = registry.get_instance("db")
def pre_start(self):
self.bot.register_packet_handler(server_packets.CharacterLookup.id, self.update)
self.bot.register_packet_handler(server_packets.CharacterName.id, self.update)
def start(self):
self.db.shared.exec("CREATE TABLE IF NOT EXISTS `all_orgs` (`org_id` INT(11) NOT NULL, "
"`org_name` VARCHAR(64) NOT NULL COLLATE 'utf8mb4_general_ci', "
"`member_count` INT(11) NULL DEFAULT NULL, "
"`faction` VARCHAR(16) NULL DEFAULT NULL COLLATE 'utf8mb4_general_ci', "
"`last_seen` INT(11) NOT NULL, "
"PRIMARY KEY (`org_id`) USING BTREE, "
"INDEX `org_name` (`org_name`) USING BTREE)")
self.db.create_view("all_orgs")
def _wait_for_char_id(self, char_name):
# char_name must be .capitalize()'ed
packet = self.bot.iterate(1)
while packet and char_name not in self.name_to_id:
packet = self.bot.iterate(1)
return self.name_to_id.get(char_name, None)
# FeatureFlags.THREADING
def _wait_for_char_id_threading(self, char_name):
# char_name must be .capitalize()'ed
event = self.notify_on_receive.get(char_name, None)
if event is None:
event = threading.Event()
self.notify_on_receive[char_name] = event
if char_name not in self.name_to_id:
event.wait(10)
return self.name_to_id.get(char_name, None)
def resolve_char_to_id(self, char):
if isinstance(char, int):
return char
elif char.isdigit():
return int(char)
else:
char_name = char.capitalize()
if char_name in self.name_to_id:
return self.name_to_id[char_name]
else:
self._send_lookup_if_needed(char_name)
return self._wait_for_char_id(char_name)
def resolve_char_to_name(self, char, default=None):
if isinstance(char, int) or char.isdigit():
char_name = self.get_char_name(char)
return char_name if char_name else default
else:
return char
def get_char_name(self, char_id):
return self.id_to_name.get(char_id, None)
def update(self, conn, packet):
self.waiting_for_response.discard(packet.name)
if packet.char_id == 4294967295:
self.name_to_id[packet.name] = None
else:
self.id_to_name[packet.char_id] = packet.name
self.name_to_id[packet.name] = packet.char_id
# self._update_name_history(packet.name, packet.char_id)
def _update_name_history(self, char_name, char_id):
params = [char_name, char_id, int(time.time())]
self.db.exec("INSERT IGNORE INTO name_history (name, char_id, created_at) VALUES (?, ?, ?)", params)
def _send_lookup_if_needed(self, char_name):
# char_name must be .capitalize()'ed
if char_name not in self.name_to_id and char_name not in self.waiting_for_response:
self.waiting_for_response.add(char_name)
self.bot.send_packet(CharacterLookup(char_name))
+187
View File
@@ -0,0 +1,187 @@
import datetime
import json
import time
import requests
from requests import ReadTimeout
from core.db import DB
from core.decorators import instance
from core.dict_object import DictObject
from core.logger import Logger
@instance()
class OrgPorkService:
CACHE_GROUP = "org_roster"
CACHE_MAX_AGE = 86400
def __init__(self):
self.logger = Logger(__name__)
def inject(self, registry):
self.bot = registry.get_instance("bot")
self.db: DB = registry.get_instance("db")
self.character_service = registry.get_instance("character_service")
self.pork_service = registry.get_instance("pork_service")
self.cache_service = registry.get_instance("cache_service")
def get_org_info(self, org_id):
cache_key = f"{org_id:d}.{self.bot.dimension:d}.json"
t = int(time.time())
# check cache for fresh value
cache_result = self.cache_service.retrieve(self.CACHE_GROUP, cache_key)
is_cache = False
if cache_result and cache_result.last_modified > (t - self.CACHE_MAX_AGE):
result = json.loads(cache_result.data)
is_cache = True
else:
url = self.get_pork_url(self.bot.dimension, org_id)
try:
r = requests.get(url, timeout=5)
result = r.json()
# if data is invalid
if result[0]["ORG_INSTANCE"] != org_id:
result = None
except ReadTimeout:
self.logger.warning("Timeout while requesting '%s'" % url)
result = None
except ValueError as e:
# noinspection PyUnboundLocalVariable
self.logger.warning("Error marshalling value as json for url '%s': %s" % (url, r.text), e)
result = None
if result:
# store result in cache
self.cache_service.store(self.CACHE_GROUP, cache_key, json.dumps(result))
elif cache_result:
# check cache for any value, even expired
result = json.loads(cache_result.data)
is_cache = True
if not result:
return None
org_info = result[0]
org_members = result[1]
last_updated = result[2]
new_org_info = DictObject({
"counts": {
"gender": {
"Female": org_info["FEMALECOUNT"],
"Male": org_info["MALECOUNT"],
"Neuter": org_info["NEUTERCOUNT"],
},
"breed": {
"Atrox": org_info["ATROXCOUNT"],
"Nanomage": org_info["NANORACECOUNT"],
"Opifex": org_info["OPIFEXCOUNT"],
"Solitus": org_info["SOLITUSCOUNT"],
},
"profession": {
"Monster": org_info["MONSTERCOUNT"],
"Adventurer": org_info["ADVENTURERCOUNT"],
"Agent": org_info["AGENTCOUNT"],
"Bureaucrat": org_info["BTCOUNT"],
"Doctor": org_info["DOCTORCOUNT"],
"Enforcer": org_info["ENFCOUNT"],
"Engineer": org_info["ENGINEEERCOUNT"],
"Fixer": org_info["FIXERCOUNT"],
"Keeper": org_info["KEEPERCOUNT"],
"Martial Artist": org_info["MACOUNT"],
"Meta-Physicist": org_info["METACOUNT"],
"Nano-Technician": org_info["NANOCOUNT"],
"Shade": org_info["SHADECOUNT"],
"Soldier": org_info["SOLIDERCOUNT"],
"Trader": org_info["TRADERCOUNT"],
}
},
"min_level": org_info["MINLVL"],
"num_members": org_info["NUMMEMBERS"],
"dimension": org_info["ORG_DIMENSION"],
"governing_type": org_info["GOVERNINGNAME"],
"max_level": org_info["MAXLVL"],
"org_id": org_info["ORG_INSTANCE"],
"objective": org_info["OBJECTIVE"],
"description": org_info["DESCRIPTION"],
"history": org_info["HISTORY"],
"avg_level": org_info["AVGLVL"],
"name": org_info["NAME"],
"faction": org_info["SIDE_NAME"],
"faction_id": org_info["SIDE"],
})
members = {}
data = []
for org_member in org_members:
char_info = DictObject({
"name": org_member["NAME"],
"char_id": org_member["CHAR_INSTANCE"],
"first_name": org_member["FIRSTNAME"],
"last_name": org_member["LASTNAME"],
"level": org_member["LEVELX"],
"breed": org_member["BREED"],
"dimension": org_member["CHAR_DIMENSION"],
"gender": org_member["SEX"],
"faction": org_info["SIDE_NAME"],
"profession": org_member["PROF"],
"profession_title": org_member["PROF_TITLE"],
"ai_rank": org_member["DEFENDER_RANK_TITLE"],
"ai_level": org_member["ALIENLEVEL"],
"pvp_rating": org_member["PVPRATING"],
"pvp_title": org_member["PVPTITLE"] or "",
"head_id": org_member["HEADID"],
"org_id": org_info.get("ORG_INSTANCE", 0),
"org_name": org_info.get("NAME", ""),
"org_rank_name": org_member.get("RANK_TITLE", ""),
"org_rank_id": org_member.get("RANK", 0),
"source": "people.anarchy-online.com"
})
if not is_cache:
data.append(
(char_info.char_id, char_info.name, char_info.first_name, char_info.last_name, char_info.level,
char_info.breed, char_info.gender, char_info.faction, char_info.profession,
char_info.profession_title,
char_info.ai_rank, char_info.ai_level, char_info.org_id, char_info.org_name,
char_info.org_rank_name,
char_info.org_rank_id, char_info.dimension, char_info.head_id, char_info.pvp_rating,
char_info.pvp_title, char_info.source, int(time.time())))
# prefetch char ids from chat server
# noinspection PyProtectedMember
self.character_service._send_lookup_if_needed(char_info.name)
members[char_info.char_id] = char_info
if len(data) > 0:
with self.db.lock:
with self.db.pool.get_connection() as conn:
with conn.cursor() as cur:
cur.executemany(
"INSERT IGNORE INTO player ( char_id, name, first_name, last_name, level, breed, gender, "
"faction, profession, profession_title, "
"ai_rank, ai_level, org_id, org_name, "
"org_rank_name, org_rank_id, dimension, "
"head_id, pvp_rating, pvp_title, source, last_updated) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", data)
conn.commit()
if len(members) == 0:
return None
else:
return DictObject({"last_modified": cache_result.last_modified if is_cache else t,
"org_info": new_org_info,
"org_members": members,
"last_updated": int(
datetime.datetime.strptime(last_updated, "%Y/%m/%d %H:%M:%S").timestamp())})
def get_pork_url(self, dimension, org_id):
# Dont use SSL, as its rather slow compared to normal requests....
# noinspection HttpUrlsUsage
return f"http://people.anarchy-online.com/org/stats/d/{dimension}/name/{org_id}/basicstats.xml?data_type=json"
+267
View File
@@ -0,0 +1,267 @@
import time
import requests
from mysql.connector.cursor import CursorBase
from requests import ReadTimeout
from core.aochat import server_packets
from core.db import DB
from core.decorators import instance, timerevent
from core.dict_object import DictObject
from core.logger import Logger
@instance()
class PorkService:
def __init__(self):
self.logger = Logger(__name__)
self.updates = []
def inject(self, registry):
self.bot = registry.get_instance("bot")
self.db: DB = registry.get_instance("db")
self.character_service = registry.get_instance("character_service")
def pre_start(self):
self.bot.register_packet_handler(server_packets.CharacterLookup.id, self.update)
self.bot.register_packet_handler(server_packets.CharacterName.id, self.update)
def start(self):
self.db.shared.exec(
"CREATE TABLE IF NOT EXISTS player ("
"char_id BIGINT PRIMARY KEY, "
"first_name VARCHAR(30) NOT NULL, "
"name VARCHAR(20) NOT NULL, "
"last_name VARCHAR(30) NOT NULL, "
"level SMALLINT NOT NULL, "
"breed VARCHAR(20) NOT NULL, "
"gender VARCHAR(20) NOT NULL, "
"faction VARCHAR(20) NOT NULL, "
"profession VARCHAR(20) NOT NULL, "
"profession_title VARCHAR(50) NOT NULL, "
"ai_rank VARCHAR(20) NOT NULL, "
"ai_level SMALLINT, "
"org_id INT DEFAULT NULL, "
"org_name VARCHAR(255) NOT NULL, "
"org_rank_name VARCHAR(20) NOT NULL, "
"org_rank_id SMALLINT NOT NULL, "
"dimension SMALLINT NOT NULL, "
"head_id INT NOT NULL, "
"pvp_rating SMALLINT NOT NULL, "
"pvp_title VARCHAR(20) NOT NULL, "
"source VARCHAR(50) NOT NULL, "
"last_updated INT NOT NULL, "
"invalid int DEFAULT 0, "
"INDEX `name` (`name`) USING BTREE, "
"INDEX `org_id` (`org_id`) USING BTREE, "
"INDEX `org_name` (`org_name`) USING BTREE,"
"INDEX `org_rank_name` (`org_rank_name`) USING BTREE, "
"INDEX `org_rank_id` (`org_rank_id`) USING BTREE, "
"INDEX `profession` (`profession`) USING BTREE, "
"INDEX `level` (`level`) USING BTREE, "
"INDEX `ai_level` (`ai_level`) USING BTREE)")
self.db.create_view("player")
# forces a lookup from remote PoRK server
# this should not be called directly unless you are requesting info for a char on a different server
# since cache will not be used and the result will also update the cache
def request_char_info(self, char_name, server_num):
url = self.get_pork_url(server_num, char_name)
try:
r = requests.get(url, timeout=5)
result = r.json()
except ReadTimeout:
self.logger.warning("Timeout while requesting '%s'" % url)
result = None
except ValueError as e:
# noinspection PyUnboundLocalVariable
self.logger.debug("Error marshalling value as json for url '%s': %s" % (url, r.text), e)
result = None
char_info = None
if result:
char_info_json = result[0]
org_info_json = result[1] if result[1] else {}
char_info = DictObject({
"name": char_info_json["NAME"],
"char_id": char_info_json["CHAR_INSTANCE"],
"first_name": char_info_json["FIRSTNAME"],
"last_name": char_info_json["LASTNAME"],
"level": char_info_json["LEVELX"],
"breed": char_info_json["BREED"],
"dimension": char_info_json["CHAR_DIMENSION"],
"gender": char_info_json["SEX"],
"faction": char_info_json["SIDE"],
"profession": char_info_json["PROF"],
"profession_title": char_info_json["PROFNAME"],
"ai_rank": char_info_json["RANK_name"],
"ai_level": char_info_json["ALIENLEVEL"],
"pvp_rating": char_info_json["PVPRATING"],
"pvp_title": char_info_json["PVPTITLE"] or "",
"head_id": char_info_json["HEADID"],
"org_id": org_info_json.get("ORG_INSTANCE", 0),
"org_name": org_info_json.get("NAME", ""),
"org_rank_name": org_info_json.get("RANK_TITLE", ""),
"org_rank_id": org_info_json.get("RANK", 0),
"source": "people.anarchy-online.com",
"cache_age": 0
})
return char_info
# standard method to get character pork data when character is on the same server
def get_character_info(self, char, max_cache_age=86400):
char_id = self.character_service.resolve_char_to_id(char)
char_name = self.character_service.resolve_char_to_name(char)
t = int(time.time())
# if there is an entry in database and it is within the cache time, use that
db_char_info = self.get_from_database(char_id=char_id, char_name=char_name)
if db_char_info:
db_char_info.cache_age = t - db_char_info.last_updated
if db_char_info.cache_age < max_cache_age and db_char_info.source != "chat_server":
return db_char_info
# if we can't resolve to a char_name, we can't make a call to pork
if not char_name:
return db_char_info
char_info = self.request_char_info(char_name, self.bot.dimension)
if char_info and char_info.char_id == char_id:
self.save_character_info(char_info)
return char_info
else:
# return cached info from database, even tho it's old, and set cache_age (if it exists)
if db_char_info:
db_char_info.cache_age = t - db_char_info.last_updated
return db_char_info
# forces a skeleton object into the player table in the case that PoRK does not return any data
# call this method if you don't need the data now but want to ensure there is a record in the database
def load_character_info(self, char_id, char_name=None, skeleton_only=False):
char_info = self.get_character_info(char_id)
if not skeleton_only and (not char_info and char_name):
char_info = self.get_character_info(char_name)
if not char_info:
char_info = DictObject({
"name": "Unknown:" + str(char_id),
"char_id": char_id,
"first_name": "",
"last_name": "",
"level": 0,
"breed": "",
"dimension": self.bot.dimension,
"gender": "",
"faction": "",
"profession": "",
"profession_title": "",
"ai_rank": "",
"ai_level": 0,
"pvp_rating": 0,
"pvp_title": "",
"head_id": 0,
"org_id": 0,
"org_name": "",
"org_rank_name": "",
"org_rank_id": 6,
"source": "stub"
})
self.save_character_info(char_info)
def save_character_info(self, char_info):
if char_info["dimension"] != self.bot.dimension:
return
with self.db.pool.get_connection() as conn:
with conn.cursor() as cur:
# cur.execute("DELETE FROM player WHERE char_id = ?", [char_info["char_id"]])
insert_sql = """
REPLACE INTO player (char_id, name, first_name, last_name, level,
breed, gender, faction, profession, profession_title, ai_rank, ai_level,
org_id, org_name, org_rank_name, org_rank_id, dimension, head_id,
pvp_rating, pvp_title, source, last_updated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"""
cur.execute(insert_sql,
[char_info["char_id"], char_info["name"], char_info["first_name"], char_info["last_name"],
char_info["level"], char_info["breed"],
char_info["gender"], char_info["faction"], char_info["profession"],
char_info["profession_title"], char_info["ai_rank"], char_info["ai_level"],
char_info["org_id"], char_info["org_name"], char_info["org_rank_name"],
char_info["org_rank_id"], char_info["dimension"], char_info["head_id"],
char_info["pvp_rating"], char_info["pvp_title"], char_info["source"], int(time.time())])
def get_from_database(self, char_id=None, char_name=None):
if char_id:
return self.db.query_single(
"SELECT char_id, name, first_name, last_name, level, breed, gender, faction, profession, "
"profession_title, ai_rank, ai_level, org_id, org_name, org_rank_name, org_rank_id, "
"dimension, head_id, pvp_rating, pvp_title, source, last_updated "
"FROM player WHERE char_id = ?", [char_id])
elif char_name:
return self.db.query_single(
"SELECT char_id, name, first_name, last_name, level, breed, gender, faction, profession, "
"profession_title, ai_rank, ai_level, org_id, org_name, org_rank_name, org_rank_id, "
"dimension, head_id, pvp_rating, pvp_title, source, last_updated "
"FROM player WHERE name = ?", [char_name])
else:
return None
def update(self, conn, packet):
# don't update if we didn't get a valid response
if packet.char_id == 4294967295:
return
self.updates.append(packet)
@timerevent(budatime="1min", description="Save player changes", is_hidden=True)
def batch_update(self, event_type, event_data):
update = "UPDATE player SET name = ? WHERE char_id = ?"
insert = "INSERT IGNORE INTO player ( char_id, name, first_name, last_name, level, breed, gender, faction, " \
"profession, profession_title, ai_rank, ai_level, org_id, org_name, org_rank_name, org_rank_id, " \
"dimension, head_id, pvp_rating, pvp_title, source, last_updated) " \
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
updates, inserts = [], []
with self.db.pool.get_connection() as conn:
with conn.cursor(dictionary=True) as cur:
for packet in self.updates:
cur: CursorBase
cur.execute(
"SELECT char_id, name, first_name, last_name, level, breed, gender, faction, profession, "
"profession_title, ai_rank, ai_level, org_id, org_name, org_rank_name, org_rank_id, "
"dimension, head_id, pvp_rating, pvp_title, source, last_updated "
"FROM player WHERE char_id = ?",
[packet.char_id])
data = cur.fetchone()
character = None
if data:
character = DictObject(data)
if character:
if character.name != packet.name:
updates.append((packet.name, packet.char_id))
else:
inserts.append((packet.char_id, packet.name, "", "", 0, "", "", "", "", "", "", 0, 0, "", "", 6,
self.bot.dimension, 0, 0, "", "chat_server", int(time.time())))
if inserts:
cur.executemany(insert, inserts)
if updates:
cur.executemany(update, updates)
self.updates = []
# noinspection SqlResolve
def find_orgs(self, search):
return self.db.query("SELECT DISTINCT org_name, org_id FROM all_orgs WHERE org_name <EXTENDED_LIKE=0> ?",
[search], extended_like=True)
def get_pork_url(self, dimension, char_name):
# Dont use SSL, as its rather slow compared to normal requests....
# noinspection HttpUrlsUsage
return f"http://people.anarchy-online.com/character/bio/d/{dimension}/name/{char_name}/bio.xml?data_type=json"