From b609d7646e2964f7e3faf789228426226dfb16bc Mon Sep 17 00:00:00 2001 From: legonzaur Date: Thu, 20 Apr 2023 10:05:29 +0200 Subject: [PATCH] alternate data processing --- package.json | 1 - src/cache/cacheUtils.ts | 97 ------- src/cache/redis.ts | 21 -- src/client/client.ts | 133 +++++----- src/clientMain.ts | 18 +- .../2023-04-09-02 server auth patch.ts | 18 +- .../2023-04-19-01 last aggregate.ts | 46 ++++ src/process/onKill.ts | 148 ++++------- src/process/process.ts | 241 +++++------------- src/processMain.ts | 14 - src/server/server.ts | 5 +- tests/client.test.ts | 26 +- tests/realtime.test.ts | 10 +- 13 files changed, 264 insertions(+), 514 deletions(-) delete mode 100644 src/cache/cacheUtils.ts delete mode 100644 src/cache/redis.ts create mode 100644 src/db/migrations/2023-04-19-01 last aggregate.ts delete mode 100644 src/processMain.ts diff --git a/package.json b/package.json index 803e0a9..1b1a924 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,6 @@ "build": "npx tsc", "startServer": "node out/serverMain.js", "startClient": "node out/clientMain.js", - "startProcess": "node out/processMain.js", "documentation": "redocly build-docs .\\docs\\v1.yml --output=docs\\index.html", "test": "jest --runInBand" } diff --git a/src/cache/cacheUtils.ts b/src/cache/cacheUtils.ts deleted file mode 100644 index 0dd17d4..0000000 --- a/src/cache/cacheUtils.ts +++ /dev/null @@ -1,97 +0,0 @@ -import cache from './redis' -/** - * Shows the data for one weapon - * @param weapon - * @param server - * @param player - * @returns - */ -export async function getWeaponReport( - weapon: string, - server?: number, - player?: string -) { - const serverPrefix = !isNaN(Number(server?.toString())) - ? `servers:${server}:` - : '' - const playerPrefix = player ? `players:${player}:` : '' - const cacheLocation = serverPrefix + playerPrefix + `weapons:` + weapon - const data = await cache.HGETALL(cacheLocation) - delete data.last_entry - return data -} - -export function getWeaponSet(server?: number, player?: string) { - const serverPrefix = !isNaN(Number(server?.toString())) - ? `servers:${server}:` - : '' - const playerPrefix = player ? `players:${player}:` : '' - const cacheLocation = serverPrefix + playerPrefix + `weapons` - return cache.SMEMBERS(cacheLocation) -} - -/** - * Show the processed list of weapons - * @param server - * @param player - * @returns string - */ -export async function getWeaponList(server?: string, player?: string) { - const serverPrefix = !isNaN(Number(server?.toString())) - ? `servers:${server}:` - : '' - const playerPrefix = player ? `players:${player}:` : '' - return JSON.parse( - (await cache.GET(serverPrefix + playerPrefix + `weapons:processedList`)) || - '{}' - ) -} - -/** - * Show processed data for a player - * @param player - * @param server - * @param weapon - * @returns - */ -export async function getPlayerReport( - player: string, - server?: number, - weapon?: string -) { - const serverPrefix = !isNaN(Number(server?.toString())) - ? `servers:${server}:` - : '' - const weaponPrefix = weapon ? `weapons:${weapon}:` : '' - const cacheLocation = serverPrefix + weaponPrefix + `players:` + player - const data = await cache.HGETALL(cacheLocation) - delete data.last_entry - return data -} - -/** - * Shows processed list of players - * @param server - * @param weapon - * @returns string - */ -export async function getPlayerList(server?: number, weapon?: string) { - const serverPrefix = !isNaN(Number(server?.toString())) - ? `servers:${server}:` - : '' - const weaponPrefix = weapon ? `weapons:${weapon}:` : '' - - return JSON.parse( - (await cache.GET(serverPrefix + weaponPrefix + `players:processedList`)) || - '{}' - ) -} - -export function getPlayerSet(server?: number, weapon?: string) { - const serverPrefix = !isNaN(Number(server?.toString())) - ? `servers:${server}:` - : '' - const weaponPrefix = weapon ? `weapons:${weapon}:` : '' - const cacheLocation = serverPrefix + weaponPrefix + `players` - return cache.SMEMBERS(cacheLocation) -} diff --git a/src/cache/redis.ts b/src/cache/redis.ts deleted file mode 100644 index 30e91df..0000000 --- a/src/cache/redis.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { createClient } from 'redis' - -import * as dotenv from 'dotenv' -dotenv.config() - -const client = createClient({ - url: process.env.REDIS_URL, - username: process.env.REDIS_USER, - password: process.env.REDIS_PASSWORD -}) - -client.on('error', (err) => console.log('Redis Client Error', err)) - -export function cacheReady() { - return client.connect() -} -export default client - -//await client.set('key', 'value'); -//const value = await client.get('key'); -//await client.disconnect(); diff --git a/src/client/client.ts b/src/client/client.ts index 90b722f..e4819cd 100644 --- a/src/client/client.ts +++ b/src/client/client.ts @@ -1,86 +1,81 @@ import { Router } from 'express' import { param, query } from 'express-validator' import { validateErrors } from '../common' -import db from '../db/db' -import client from '../cache/redis' +import { allData } from '../process/process' + const router = Router() //timeout middleware ? router.get('/*', (req, res, next) => { next() }) -router.get( - '/weapons/:weaponId', - param(['weaponId']).exists().isString(), - query(['player', 'server']).optional().toInt().isInt(), - validateErrors, - async (req, res) => { - let path = '' - if (req.query.server) path = path + `servers.${req.query.server}.` - if (req.query.player) path = path + `players.${req.query.player}.` - path = path + `weapons.${req.params.weaponId}` - if (await client.json.type('kills', path)) return res.status(200).send(await client.json.get('kills', { path })) - res.status(404).send() - } -) +function filters(e: any, query: any) { + return ( + (query.player ? query.player == e.attacker_id : true) && + (query.server ? query.server == e.servername : true) && + (query.host ? Number(query.host) == e.host : true) && + (query.map ? query.map == e.map : true) && + (query.weapon ? query.weapon == e.cause_of_death : true) && + (query.gamemode ? query.gamemode == e.game_mode : true) + ) +} router.get( - '/weapons/', - query(['player', 'server']).optional().toInt().isInt(), + '/:dataType', + param('dataType') + .custom( + (e) => e == 'weapons' || e == 'players' || e == 'maps' || e == 'servers' + ) + .withMessage('Only weapons, players, maps or servers are valid paths'), + query(['player', 'host']).optional().toInt().isInt(), + query(['server', 'map', 'weapon', 'gamemode']).optional().isString(), validateErrors, - async (req, res) => { - let path = '' - if (req.query.server) path = path + `servers.${req.query.server}.` - if (req.query.player) path = path + `players.${req.query.player}.` - path = path + `weapons` - if (await client.json.type('kills', path)) { - const data = await client.json.get('kills', { path: `weapons` }) as { [key: number]: any } - return res.status(200).send(Object.fromEntries(Object.entries(data).map(([key, value]) => { return [key, { total_distance: value.total_distance, max_distance: value.max_distance, kills: value.kills, deaths: value.deaths }] }))) + (req, res) => { + const data: { + [key: string]: { + deaths: number + kills: number + max_distance: number + total_distance: number + } + } = {} + let index: 'cause_of_death' | 'attacker_id' | 'map' | 'servername' + switch (req.params.dataType) { + case 'weapons': + index = 'cause_of_death' + break + case 'players': + index = 'attacker_id' + break + case 'maps': + index = 'map' + break + case 'servers': + index = 'servername' + break + default: + return res.status(400).send() } - res.status(404).send() + allData + .filter((e) => filters(e, req.query)) + .forEach((e) => { + if (!data[e[index]]) + data[e[index]] = { + deaths: 0, + kills: 0, + max_distance: 0, + total_distance: 0 + } + data[e[index]].deaths += Number(e.deaths) + data[e[index]].kills += Number(e.kills) + data[e[index]].total_distance += Number(e.total_distance) + data[e[index]].max_distance = Math.max( + data[e[index]].max_distance, + Number(e.max_distance) + ) + }) + res.status(200).send(data) } ) -router.get( - '/players/:playerId', - param(['playerId']).exists().toInt().isInt(), - query(['server']).optional().toInt().isInt(), - query('weapon').optional().isString(), - validateErrors, - async (req, res) => { - let path = '' - if (req.query.server) path = path + `servers.${req.query.server}.` - if (req.query.weapon) path = path + `weapons.${req.query.weapon}.` - path = path + `players.${req.params.playerId}` - if (await client.json.type('kills', path)) return res.status(200).send(await client.json.get('kills', { path })) - res.status(404).send() - } -) - -router.get( - '/players/', - query(['server']).optional().toInt().isInt(), - query('weapon').optional().isString(), - validateErrors, - async (req, res) => { - let path = '' - if (req.query.server) path = path + `servers.${req.query.server}.` - if (req.query.weapon) path = path + `weapons.${req.query.weapon}.` - path = path + `players` - if (await client.json.type('kills', path)) { - const data = await client.json.get('kills', { path: `players` }) as { [key: number]: any } - return res.status(200).send(Object.fromEntries(Object.entries(data).map(([key, value]) => { return [key, { total_distance: value.total_distance, max_distance: value.max_distance, kills: value.kills, deaths: value.deaths }] }))) - } - res.status(404).send() - } -) - -//router.get('/maps/', (req, res, next) => {}) - -router.get('/servers/', async (req, res) => { - const data = await client.json.get('kills', { path: `servers` }) as { [key: number]: any } - return res.status(200).send(Object.fromEntries(Object.entries(data).map(([key, value]) => { return [key, value.data] }))) -}) -//router.get('/servers/:serverId/', (req, res, next) => {}) - export default router diff --git a/src/clientMain.ts b/src/clientMain.ts index f51f21f..59fb734 100644 --- a/src/clientMain.ts +++ b/src/clientMain.ts @@ -3,8 +3,8 @@ dotenv.config() import express from 'express' import cors from 'cors' import client from './client/client' -import db, { dbReady } from './db/db' -import cache, { cacheReady } from './cache/redis' +import { dbReady } from './db/db' +import processAll from './process/process' const app = express() const port = 3000 @@ -19,13 +19,11 @@ app.get('/', (req, res) => { app.use('/', client) export default - new Promise((resolve, reject) => { - dbReady().then((e) => { - cacheReady().then((e) => { - const listenServer = app.listen(port, '0.0.0.0', () => { - console.log(`Tone client api listening on port ${port}`) - resolve(listenServer) - }) - }) + new Promise(async (resolve, reject) => { + await dbReady() + await processAll() + const listenServer = app.listen(port, '0.0.0.0', () => { + console.log(`Tone client api listening on port ${port}`) + resolve(listenServer) }) }) \ No newline at end of file diff --git a/src/db/migrations/2023-04-09-02 server auth patch.ts b/src/db/migrations/2023-04-09-02 server auth patch.ts index 41038cd..734bce5 100644 --- a/src/db/migrations/2023-04-09-02 server auth patch.ts +++ b/src/db/migrations/2023-04-09-02 server auth patch.ts @@ -11,20 +11,26 @@ const pgClient = new Client({ export async function up(db: Kysely): Promise { await pgClient.connect() await pgClient.query('ALTER TABLE kill ADD servername character varying NULL;') + await pgClient.query('ALTER TABLE kill ADD host integer NULL;') await pgClient.query('UPDATE kill SET servername = server.name FROM server WHERE server.id = kill.server;') await pgClient.query('ALTER TABLE kill ALTER COLUMN servername SET NOT NULL;') await pgClient.query('ALTER TABLE kill DROP COLUMN server;') - await pgClient.query('ALTER TABLE server RENAME TO hoster') - await pgClient.query('ALTER TABLE hoster DROP COLUMN description') - await pgClient.query('ALTER TABLE kill ADD COLUMN host integer NULL') + //await pgClient.query('ALTER TABLE server RENAME TO hoster') + //await pgClient.query('ALTER TABLE hoster DROP COLUMN description') + //await pgClient.query('ALTER TABLE kill ADD COLUMN host integer NULL') + await pgClient.query('CREATE TABLE host (name character varying, token character varying)') + //Update the hosts column once the hoster table is manually updated + await pgClient.end() } export async function down(db: Kysely): Promise { - await pgClient.query('ALTER TABLE kill DROP COLUMN host') - await pgClient.query('ALTER TABLE hoster add description character varying NULL') - await pgClient.query('ALTER TABLE hoster RENAME TO server') + await pgClient.connect() + //await pgClient.query('ALTER TABLE kill DROP COLUMN host') + //await pgClient.query('ALTER TABLE hoster add description character varying NULL') + //await pgClient.query('ALTER TABLE hoster RENAME TO server') + await pgClient.query('DROP TABLE host') await pgClient.query(`ALTER TABLE kill ADD server integer NULL;`) await pgClient.query('UPDATE kill SET server = server.id FROM server where server.name = kill.servername;') diff --git a/src/db/migrations/2023-04-19-01 last aggregate.ts b/src/db/migrations/2023-04-19-01 last aggregate.ts new file mode 100644 index 0000000..d410198 --- /dev/null +++ b/src/db/migrations/2023-04-19-01 last aggregate.ts @@ -0,0 +1,46 @@ +import { Kysely, sql } from 'kysely' +import { Client } from 'pg' + +const pgClient = new Client({ + host: process.env.POSTGRES_HOST, + database: process.env.POSTGRES_DATABASE, + user: process.env.POSTGRES_USER, + password: process.env.POSTGRES_PASSWORD +}) + +export async function up(db: Kysely): Promise { + await pgClient.connect() + await pgClient.query(`CREATE OR REPLACE FUNCTION public.first_agg (anyelement, anyelement) + RETURNS anyelement + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS +'SELECT $1';`) + + await pgClient.query(`CREATE AGGREGATE public.first (anyelement) ( + SFUNC = public.first_agg + , STYPE = anyelement + , PARALLEL = safe + );`) + + await pgClient.query(`CREATE OR REPLACE FUNCTION public.last_agg (anyelement, anyelement) + RETURNS anyelement + LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS +'SELECT $2';`) + + + await pgClient.query(`CREATE AGGREGATE public.last (anyelement) ( + SFUNC = public.last_agg + , STYPE = anyelement + , PARALLEL = safe + );`) + //Update the hosts column once the hoster table is manually updated + await pgClient.end() +} + +export async function down(db: Kysely): Promise { + await pgClient.connect() + pgClient.query('DROP AGGREGATE public.last') + pgClient.query('DROP FUNCTION public.last_agg') + pgClient.query('DROP AGGREGATE public.first') + pgClient.query('DROP AGGREGATE public.first_agg') + await pgClient.end() +} diff --git a/src/process/onKill.ts b/src/process/onKill.ts index 8d51722..5ef0387 100644 --- a/src/process/onKill.ts +++ b/src/process/onKill.ts @@ -1,6 +1,5 @@ import { Client } from 'pg' -import client from '../cache/redis' -import { genPrefix } from './process' +import { allData } from './process' const pgClient = new Client({ host: process.env.POSTGRES_HOST, @@ -13,98 +12,63 @@ export default async function listenKills() { await pgClient.connect() await pgClient.query('LISTEN new_kill') - pgClient.on('notification', async (data) => { + pgClient.on('notification', (data) => { if (!data.payload) return const payload = JSON.parse(data.payload) - await updateGlobal(payload) + let killEntry = allData.find( + (e) => + e.attacker_id == payload.attacker_id && + e.cause_of_death == payload.cause_of_death && + e.game_mode == payload.game_mode && + e.host == payload.host && + e.map == payload.map && + e.servername == payload.servername + ) + let deathEntry = allData.find( + (e) => + e.attacker_id == payload.victim_id && + e.cause_of_death == payload.cause_of_death && + e.game_mode == payload.game_mode && + e.host == payload.host && + e.map == payload.map && + e.servername == payload.servername + ) + if (!killEntry) { + killEntry = { + kills: 0, + deaths: 0, + max_distance: 0, + total_distance: 0, + attacker_id: payload.attacker_id, + cause_of_death: payload.cause_of_death, + game_mode: payload.game_mode, + host: payload.host, + map: payload.map, + servername: payload.servername + } + allData.push(killEntry) + } + + if (!deathEntry) { + deathEntry = { + kills: 0, + deaths: 0, + max_distance: 0, + total_distance: 0, + attacker_id: payload.victim_id, + cause_of_death: payload.victim_current_weapon, + game_mode: payload.game_mode, + host: payload.host, + map: payload.map, + servername: payload.servername + } + allData.push(deathEntry) + } + + killEntry.kills++ + killEntry.total_distance += payload.distance + killEntry.max_distance = Math.max(killEntry.max_distance, payload.distance) + deathEntry.deaths++ }) return pgClient } - - -async function updateGlobal({ cause_of_death, distance, attacker_id, victim_id, servername, host }: { cause_of_death: string, distance: number, attacker_id: string, victim_id: string, servername: string, host: number }) { - servername = servername.replace(/[^a-z0-9]/gi, '') - const promises: Promise[] = [] - - if (!await client.json.type('kills')) await client.json.set('kills', '', { data: {}, weapons: {}, servers: {}, players: {} }) - - if (!await client.json.type('kills', genPrefix.weapon({ cause_of_death }))) await client.json.set('kills', genPrefix.weapon({ cause_of_death }), { players: {}, max_distance: 0, kills: 0, total_distance: 0 }) - if (!await client.json.type('kills', genPrefix.player({ attacker_id }))) await client.json.set('kills', genPrefix.player({ attacker_id }), { weapons: {}, max_distance: 0, kills: 0, deaths: 0, total_distance: 0 }) - if (!await client.json.type('kills', `servers.${host}`)) await client.json.set('kills', `servers.${host}`, {}) - if (!await client.json.type('kills', `servers.${host}.${servername}`)) await client.json.set('kills', `servers.${host}.${servername}`, { data: {}, weapons: {}, players: {}, max_distance: 0, kills: 0, total_distance: 0 }) - if (!await client.json.type('kills', genPrefix.weapon({ cause_of_death, server: `${host}.${servername}` }))) await client.json.set('kills', genPrefix.weapon({ cause_of_death, server: `${host}.${servername}` }), { players: {}, max_distance: 0, kills: 0, total_distance: 0 }) - if (!await client.json.type('kills', genPrefix.player({ attacker_id, server: `${host}.${servername}` }))) await client.json.set('kills', genPrefix.player({ attacker_id, server: `${host}.${servername}` }), { weapons: {}, max_distance: 0, kills: 0, deaths: 0, total_distance: 0 }) - - if (!await client.json.type('kills', genPrefix.player({ attacker_id: victim_id, server: `${host}.${servername}` }))) await client.json.set('kills', genPrefix.player({ attacker_id: victim_id, server: `${host}.${servername}` }), { weapons: {}, max_distance: 0, kills: 0, deaths: 0, total_distance: 0 }) - if (!await client.json.type('kills', genPrefix.player({ attacker_id: victim_id }))) await client.json.set('kills', genPrefix.player({ attacker_id: victim_id }), { weapons: {}, max_distance: 0, kills: 0, deaths: 0, total_distance: 0 }) - - if (!await client.json.type('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death }))) await client.json.set('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death }), { max_distance: 0, kills: 0, deaths: 0, total_distance: 0 }) - if (!await client.json.type('kills', genPrefix.playerWeapons({ attacker_id: victim_id, cause_of_death }))) await client.json.set('kills', genPrefix.playerWeapons({ attacker_id: victim_id, cause_of_death }), { max_distance: 0, kills: 0, total_distance: 0 }) - - if (!await client.json.type('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death, server: `${host}.${servername}` }))) await client.json.set('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death, server: `${host}.${servername}` }), { max_distance: 0, kills: 0, deaths: 0, total_distance: 0 }) - if (!await client.json.type('kills', genPrefix.playerWeapons({ attacker_id: victim_id, cause_of_death, server: `${host}.${servername}` }))) await client.json.set('kills', genPrefix.playerWeapons({ attacker_id: victim_id, cause_of_death, server: `${host}.${servername}` }), { max_distance: 0, kills: 0, total_distance: 0 }) - - promises.push(updatePath(`data`, { distance })) - //servers.1 - promises.push(updatePath(`servers.${host}.${servername}`, { distance }).then(() => Promise.all([ - //servers.1.players.123456789 - updatePath(`servers.${host}.${servername}.players.${attacker_id}`, { distance }).then(async () => { - if (!await client.json.type('kills', `servers.${host}.${servername}.players.${victim_id}.deaths`)) await client.json.set('kills', `servers.${host}.${servername}.players.${victim_id}.deaths`, 0) - await client.json.numIncrBy('kills', `servers.${host}.${servername}.players.${victim_id}.deaths`, 1) - //servers.1.players.123456789.weapons.epg - await updatePath(`servers.${host}.${servername}.players.${attacker_id}.weapons.${cause_of_death}`, { distance }) - } - - ), - //servers.1.weapons.epg - updatePath(`servers.${host}.${servername}.weapons.${cause_of_death}`, { distance }).then(async () => { - //servers.1.weapons.epg.players.123456789 - await updatePath(`servers.${host}.${servername}.weapons.${cause_of_death}.players.${attacker_id}`, { distance }) - if (!await client.json.type('kills', `servers.${host}.${servername}.players.${victim_id}.deaths`)) await client.json.set('kills', `servers.${host}.${servername}.weapons.${cause_of_death}.players.${victim_id}`, 0) - await client.json.numIncrBy('kills', `servers.${host}.${servername}.weapons.${cause_of_death}.players.${victim_id}.deaths`, 1) - } - - )]) - )) - - - //players.123456789 - promises.push(updatePath(`players.${attacker_id}`, { distance }).then(async () => { - if (!await client.json.type('kills', `players.${victim_id}.deaths`)) await client.json.set('kills', `players.${victim_id}.deaths`, 0) - await client.json.numIncrBy('kills', `players.${victim_id}.deaths`, 1) - //players.123456789.weapons.epg - await updatePath(`players.${attacker_id}.weapons.${cause_of_death}`, { distance }) - })) - - - //weapons.epg - promises.push(updatePath(`weapons.${cause_of_death}`, { distance }).then(async () => { - //weapons.epg.players.123456789 - await updatePath(`weapons.${cause_of_death}.players.${attacker_id}`, { distance }) - if (!await client.json.type('kills', `weapons.${cause_of_death}.players.${victim_id}.deaths`)) await client.json.set('kills', `weapons.${cause_of_death}.players.${victim_id}.deaths`, 0) - await client.json.numIncrBy('kills', `weapons.${cause_of_death}.players.${victim_id}.deaths`, 1) - } - )) - - - await Promise.all(promises) -} - -async function updatePath(path: string, { distance }: { distance: number }) { - await setupPath(path) - console.log(path + '.max_distance', await client.json.type('kills', path + '.max_distance')) - console.log(path, await client.json.type('kills', path)) - if (!await client.json.type('kills', path)) await client.json.set('kills', path, {}) - let transaction = client.multi() - transaction = transaction.json.numIncrBy('kills', `${path}.total_distance`, distance) - let max_distance = Number(await client.json.get('kills', { path: `${path}.max_distance` })) - if (isNaN(max_distance) || max_distance < distance) transaction = transaction.json.set('kills', `${path}.max_distance`, distance) - transaction = transaction.json.numIncrBy('kills', `${path}.kills`, 1) - await transaction.exec() -} - -async function setupPath(path: string) { - if (!await client.json.type('kills', path + '.kills')) await client.json.set('kills', path + '.kills', 0) - if (!await client.json.type('kills', path + '.total_distance')) await client.json.set('kills', path + '.total_distance', 0) - if (!await client.json.type('kills', path + '.max_distance')) await client.json.set('kills', path + '.max_distance', 0) -} \ No newline at end of file diff --git a/src/process/process.ts b/src/process/process.ts index 4b4440c..ea5bbd8 100644 --- a/src/process/process.ts +++ b/src/process/process.ts @@ -1,35 +1,19 @@ import db from '../db/db' -const { count, max, sum } = db.fn -import client from '../cache/redis' +const { count, max, sum, coalesce } = db.fn +import { sql } from 'kysely' -export const genPrefix = { - global: ({ server }: { server?: string }) => { - return (server ? `servers.${server}.` : '') + 'data' - }, - weapon: ({ cause_of_death, server }: { cause_of_death: string, server?: string }) => { - return (server ? `servers.${server}.` : '') + `weapons.${cause_of_death}` - }, - player: ({ attacker_id, server }: { attacker_id: string, server?: string }) => { - return (server ? `servers.${server}.` : '') + `players.${attacker_id}` - }, - playerWeapons: ({ cause_of_death, attacker_id, server }: { attacker_id: string, cause_of_death: string, server?: string }) => { - return (server ? `servers.${server}.` : '') + `players.${attacker_id}.weapons.${cause_of_death}` - }, - weaponPlayers: ({ cause_of_death, attacker_id, server }: { attacker_id: string, cause_of_death: string, server?: string }) => { - return (server ? `servers.${server}.` : '') + `weapons.${cause_of_death}.players.${attacker_id}` - }, -} +export let allData: Awaited> /** - * Starts the process of populating the REDIS database globally + * populates allData * @returns */ async function processAll() { console.log('Starting data calculation...') const timeStart = new Date() - if (!await client.json.type('kills')) await client.json.set('kills', '$', { data: {}, weapons: {}, players: {}, servers: {} }) - await Promise.all([processGlobalStats(), processServerStats()]) + allData = await processGlobalStats() + console.log( 'Data calculation finished. Took + ' + Math.abs(new Date().getTime() - timeStart.getTime()) / 1000 + @@ -38,158 +22,71 @@ async function processAll() { if (process.env.ENVIRONMENT == 'production') { return } - setTimeout(processAll, 3600000) } async function processGlobalStats() { - let promises: Promise[] = [] - //Global kills - await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance')]).execute() - .then(async (data) => { - let transaction = client.multi() - await Promise.all(data.map(async ({ kills, max_distance, total_distance }) => { - await processData(genPrefix.global({}), { total_distance, max_distance, kills }, transaction) - })) - return transaction.exec() - }) - - //Global weapon kill - await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death']).groupBy('cause_of_death').execute() - .then(async (data) => { - let transaction = client.multi() - await Promise.all(data.map(async ({ kills, max_distance, total_distance, cause_of_death }) => { - const prefix = genPrefix.weapon({ cause_of_death }) - if (!await client.json.type('kills', prefix)) await client.json.set('kills', prefix, { players: {} }) - await processData(prefix, { kills, max_distance, total_distance }, transaction) - })) - return transaction.exec() - }) - - //Global player kill - await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id']).groupBy('attacker_id').whereRef("attacker_id", '!=', 'victim_id').execute() - .then(async (data) => { - let transaction = client.multi() - await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id }) => { - const prefix = genPrefix.player({ attacker_id }) - if (!await client.json.type('kills', prefix)) await client.json.set('kills', prefix, { weapons: {} }) - await processData(prefix, { kills, max_distance, total_distance }, transaction) - })) - return transaction.exec() - }) - //Global player deaths - await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id']).groupBy('victim_id').execute() - .then(async (data) => { - let transaction = client.multi() - await Promise.all(data.map(async ({ kills, victim_id }) => { - const prefix = genPrefix.player({ attacker_id: victim_id }) - transaction.json.set('kills', prefix + ".deaths", Number(kills)) - })) - return transaction.exec() - }) - - //Player weapon kills - await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id', 'cause_of_death']).groupBy(['attacker_id', 'cause_of_death']).whereRef("attacker_id", '!=', 'victim_id').execute() - .then(async (data) => { - let transaction = client.multi() - await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id, cause_of_death }) => { - await processData(genPrefix.playerWeapons({ attacker_id, cause_of_death }), { kills, max_distance, total_distance }, transaction) - await processData(genPrefix.weaponPlayers({ attacker_id, cause_of_death }), { kills, max_distance, total_distance }, transaction) - })) - return transaction.exec() - }) - //player weapon deaths - await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id', 'cause_of_death']).groupBy(['victim_id', 'cause_of_death']).execute() - .then(async (data) => { - let transaction = client.multi() - await Promise.all(data.map(async ({ kills, victim_id, cause_of_death }) => { - transaction.json.set('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death }) + ".deaths", Number(kills)) - })) - return transaction.exec() - }) + return await db + .with('kills', (db) => + db + .selectFrom('kill') + .select([ + count('id').as('kills'), + coalesce( + max('distance'), + sql`0` + ).as('max_distance'), + coalesce(sum('distance'), sql`0`).as( + 'total_distance' + ), + 'attacker_id', + 'cause_of_death', + 'map', + 'game_mode', + 'servername', + 'host' + ]) + .whereRef('attacker_id', '!=', 'victim_id') + .groupBy([ + 'attacker_id', + 'cause_of_death', + 'map', + 'servername', + 'host', + 'game_mode' + ]) + ) + .with('deaths', (db) => + db + .selectFrom('kill') + .select([ + count('id').as('deaths'), + 'victim_id', + 'victim_current_weapon', + 'map', + 'game_mode', + 'servername', + 'host' + ]) + .groupBy([ + 'victim_id', + 'victim_current_weapon', + 'map', + 'servername', + 'host', + 'game_mode' + ]) + ) + .selectFrom('kills') + .leftJoin('deaths', (join) => + join + .onRef('deaths.victim_id', '=', 'kills.attacker_id') + .onRef('deaths.victim_current_weapon', '=', 'kills.cause_of_death') + .onRef('deaths.servername', '=', 'kills.servername') + .onRef('deaths.host', '=', 'kills.host') + .onRef('deaths.game_mode', '=', 'kills.game_mode') + ) + .selectAll('kills') + .select(sql`COALESCE(deaths.deaths, 0)`.as('deaths')) + .execute() } - -async function processServerStats() { - let promises: Promise[] = [] - //Server global kills - await db.selectFrom('kill') - .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'servername', 'host']) - .groupBy(['servername', 'host']).execute().then(async (data) => { - let transaction = client.multi() - const p = data.map(async ({ kills, max_distance, total_distance, servername, host }) => { - servername = servername.replace(/[^a-z0-9]/gi, '') - if (!await client.json.type('kills', `servers.${host}`)) await client.json.set('kills', `servers.${host}`, {}) - if (!await client.json.type('kills', `servers.${host}.${servername}`)) await client.json.set('kills', `servers.${host}.${servername}`, { data: {}, weapons: {}, players: {} }) - await processData(genPrefix.global({ server: `${host}.${servername}` }), { total_distance, max_distance, kills }, transaction) - }) - await Promise.all(p) - return transaction.exec() - }) - - //Server weapon kills - await db.selectFrom('kill') - .select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'cause_of_death', 'servername', 'host']).groupBy(['cause_of_death', 'servername', 'host']).execute().then(async (data) => { - let transaction = client.multi() - await Promise.all(data.map(async ({ kills, max_distance, total_distance, cause_of_death, servername, host }) => { - servername = servername.replace(/[^a-z0-9]/gi, '') - const prefix = genPrefix.weapon({ cause_of_death, server: `${host}.${servername}` }) - if (!await client.json.type('kills', prefix)) await client.json.set('kills', prefix, { players: {} }) - await processData(prefix, { kills, max_distance, total_distance }, transaction) - })) - return transaction.exec() - }) - //server player kill - await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id', 'servername', 'host']).groupBy(['attacker_id', 'servername', 'host']).whereRef("attacker_id", '!=', 'victim_id').execute() - .then(async (data) => { - let transaction = client.multi() - await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id, servername, host }) => { - servername = servername.replace(/[^a-z0-9]/gi, '') - const prefix = genPrefix.player({ attacker_id, server: `${host}.${servername}` }) - if (!await client.json.type('kills', prefix)) await client.json.set('kills', prefix, { weapons: {} }) - await processData(prefix, { kills, max_distance, total_distance }, transaction) - })) - return transaction.exec() - }) - //server player deaths - await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id', 'servername', 'host']).groupBy(['victim_id', 'servername', 'host']).execute() - .then(async (data) => { - let transaction = client.multi() - await Promise.all(data.map(async ({ kills, victim_id, servername, host }) => { - servername = servername.replace(/[^a-z0-9]/gi, '') - const prefix = genPrefix.player({ attacker_id: victim_id, server: `${host}.${servername}` }) - transaction.json.set('kills', prefix + ".deaths", Number(kills)) - })) - return transaction.exec() - }) - - //player weapon kills - await db.selectFrom('kill').select([count('id').as('kills'), max('distance').as('max_distance'), sum('distance').as('total_distance'), 'attacker_id', 'cause_of_death', 'servername', 'host']).groupBy(['attacker_id', 'cause_of_death', 'servername', 'host']).whereRef("attacker_id", '!=', 'victim_id').execute() - .then(async (data) => { - let transaction = client.multi() - await Promise.all(data.map(async ({ kills, max_distance, total_distance, attacker_id, cause_of_death, servername, host }) => { - servername = servername.replace(/[^a-z0-9]/gi, '') - await processData(genPrefix.playerWeapons({ attacker_id, cause_of_death, server: `${host}.${servername}` }), { kills, max_distance, total_distance }, transaction) - await processData(genPrefix.weaponPlayers({ attacker_id, cause_of_death, server: `${host}.${servername}` }), { kills, max_distance, total_distance }, transaction) - })) - return transaction.exec() - }) - //player weapon deaths - await db.selectFrom('kill').select([count('id').as('kills'), 'victim_id', 'cause_of_death', 'servername', 'host']).groupBy(['victim_id', 'cause_of_death', 'servername', 'host']).execute() - .then(async (data) => { - let transaction = client.multi() - await Promise.all(data.map(async ({ kills, victim_id, cause_of_death, servername, host }) => { - servername = servername.replace(/[^a-z0-9]/gi, '') - transaction.json.set('kills', genPrefix.weaponPlayers({ attacker_id: victim_id, cause_of_death, server: `${host}.${servername}` }) + ".deaths", Number(kills)) - })) - return transaction.exec() - }) -} - -async function processData(prefix: string, { total_distance, max_distance, kills }: { total_distance: string | number | bigint, max_distance: number, kills: string | number | bigint }, transaction: ReturnType) { - if (!await client.json.type('kills', prefix)) await client.json.set('kills', prefix, {}) - transaction = transaction.json.set('kills', prefix + ".total_distance", Number(total_distance)) - transaction = transaction.json.set('kills', prefix + ".max_distance", Number(max_distance)) - transaction = transaction.json.set('kills', prefix + ".kills", Number(kills)) - return -} - export default processAll diff --git a/src/processMain.ts b/src/processMain.ts deleted file mode 100644 index 5be088d..0000000 --- a/src/processMain.ts +++ /dev/null @@ -1,14 +0,0 @@ -import * as dotenv from 'dotenv' -dotenv.config() -import { dbReady } from './db/db' -import { cacheReady } from './cache/redis' -import processAll from './process/process' -import listenKills from "./process/onKill" - -async function startup() { - await dbReady() - await cacheReady() - await listenKills() - await processAll() -} -export default startup() \ No newline at end of file diff --git a/src/server/server.ts b/src/server/server.ts index afe4886..6bbaa71 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -1,6 +1,5 @@ -import { NextFunction, Router } from 'express' -import expressBasicAuth from 'express-basic-auth' -import { body, header, param } from 'express-validator' +import { Router } from 'express' +import { body, header } from 'express-validator' import { CreateKillRecord, CheckServerToken } from '../db/db' import { validateErrors } from '../common' diff --git a/tests/client.test.ts b/tests/client.test.ts index 6f04e57..7a057d0 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -2,7 +2,6 @@ import { afterAll, beforeAll, describe, expect, test } from '@jest/globals' import clientMain from '../src/clientMain' import * as dotenv from 'dotenv' import db from '../src/db/db' -import cache from '../src/cache/redis' dotenv.config() let listenServer @@ -16,9 +15,9 @@ describe('client', () => { const request = await fetch("http://127.0.0.1:3000/servers") const data = await request.json() const first = Object.entries(data)[0] - expect(first[1]).toHaveProperty('name') - expect(first[1]).toHaveProperty('id') - expect(first[1]).toHaveProperty('description') + expect(first[1]).toHaveProperty('max_distance') + expect(first[1]).toHaveProperty('total_distance') + expect(first[1]).toHaveProperty('kills') }) test('player list', async () => { @@ -39,15 +38,6 @@ describe('client', () => { expect(first[1]).toHaveProperty('kills') }) - test('player list with weapon and server filter', async () => { - const request = await fetch("http://127.0.0.1:3000/players?weapons=sniper&server=1") - const data = await request.json() - const first = Object.entries(data)[0] - expect(first[1]).toHaveProperty('max_distance') - expect(first[1]).toHaveProperty('total_distance') - expect(first[1]).toHaveProperty('kills') - }) - test('weapon list', async () => { const request = await fetch("http://127.0.0.1:3000/weapons") const data = await request.json() @@ -65,21 +55,11 @@ describe('client', () => { expect(first[1]).toHaveProperty('total_distance') expect(first[1]).toHaveProperty('kills') }) - - test('weapon list with player and server filter', async () => { - const request = await fetch("http://127.0.0.1:3000/weapons?players=1005930844007&server=1") - const data = await request.json() - const first = Object.entries(data)[0] - expect(first[1]).toHaveProperty('max_distance') - expect(first[1]).toHaveProperty('total_distance') - expect(first[1]).toHaveProperty('kills') - }) }) afterAll((done) => { listenServer.close(async () => { await db.destroy() - await cache.quit() done() }) }) \ No newline at end of file diff --git a/tests/realtime.test.ts b/tests/realtime.test.ts index cc8e98b..dacce00 100644 --- a/tests/realtime.test.ts +++ b/tests/realtime.test.ts @@ -4,7 +4,6 @@ import serverMain from '../src/serverMain' import listenKills from "../src/process/onKill" import * as dotenv from 'dotenv' import db from '../src/db/db' -import cache from '../src/cache/redis' dotenv.config() let listenClient @@ -62,8 +61,8 @@ beforeAll(async () => { listenClient = await clientMain; listenServer = await serverMain; pgClient = await listenKills() - const yea = waitFor(10000) - const response = await fetch(`http://127.0.0.1:3001/${process.env.SERVERAUTH_ID}/kill`, { + const yea = waitFor(1000) + const response = await fetch(`http://127.0.0.1:3001/kill`, { method: "POST", // *GET, POST, PUT, DELETE, etc. credentials: "same-origin", // include, *same-origin, omit headers: { @@ -103,8 +102,8 @@ describe('realtime', () => { test('update player', async () => { jest.setTimeout(15000) - const yea = waitFor(10000) - const response = await fetch(`http://127.0.0.1:3001/${process.env.SERVERAUTH_ID}/kill`, { + const yea = waitFor(1000) + const response = await fetch(`http://127.0.0.1:3001/kill`, { method: "POST", // *GET, POST, PUT, DELETE, etc. credentials: "same-origin", // include, *same-origin, omit headers: { @@ -138,7 +137,6 @@ afterAll((done) => { listenServer.close(async () => { await pgClient.end() await db.destroy() - await cache.quit() done() }) })