From d6f735044fd2301aa34fea6a17c7a219c58603c3 Mon Sep 17 00:00:00 2001 From: legonzaur Date: Wed, 31 May 2023 19:35:51 +0200 Subject: [PATCH] Cache /player on backend for fastest response --- src/client/client.ts | 35 ++++++----- src/clientMain.ts | 4 ++ src/process/onKill.ts | 88 +++++---------------------- src/process/process.ts | 132 +++++++---------------------------------- tests/realtime.test.ts | 3 +- 5 files changed, 64 insertions(+), 198 deletions(-) diff --git a/src/client/client.ts b/src/client/client.ts index 5c517ec..c8cbbc3 100644 --- a/src/client/client.ts +++ b/src/client/client.ts @@ -3,6 +3,7 @@ import { param } from 'express-validator' import { validateErrors } from '../common' import db, { getHostList } from '../db/db' import { sql } from 'kysely' +import { allData } from '../process/process' const { max, sum } = db.fn const router = Router() @@ -31,8 +32,8 @@ const path = { interface KillRecord { kills: number - deaths?: number - deaths_while_equipped?: number + deaths: number + deaths_while_equipped: number username?: string max_distance: number total_distance: number @@ -72,21 +73,25 @@ router.get('/players', (req, res) => { void (async () => { let result = db.selectFrom('kill_view') - if (req.query) { + let data + if (Object.keys(req.query).length > 0) { result = processQueryArgs(result, req.query as unknown as string | string[]) + const selection = result.select([sum('kills').as('kills'), sum('deaths').as('deaths'), sum('deaths_with_weapon').as('deaths_while_equipped'), 'attacker_id', sql`last(attacker_name)`.as('username'), sum('total_distance').as('total_distance'), max('max_distance').as('max_distance')]).groupBy('attacker_id') + data = (await selection.execute()).reduce>((acc, curr) => { + acc[curr.attacker_id] = { + kills: Number(curr.kills), + deaths: Number(curr.deaths), + max_distance: Number(curr.max_distance), + total_distance: Number(curr.total_distance), + deaths_while_equipped: Number(curr.deaths_while_equipped), + username: curr.username + } + return acc + }, {}) + } else { + data = allData } - const selection = result.select([sum('kills').as('kills'), sum('deaths').as('deaths'), sum('deaths_with_weapon').as('deaths_while_equipped'), 'attacker_id', sql`last(attacker_name)`.as('username'), sum('total_distance').as('total_distance'), max('max_distance').as('max_distance')]).groupBy('attacker_id') - const data = (await selection.execute()).reduce>((acc, curr) => { - acc[curr.attacker_id] = { - kills: Number(curr.kills), - deaths: Number(curr.deaths), - max_distance: Number(curr.max_distance), - total_distance: Number(curr.total_distance), - deaths_while_equipped: Number(curr.deaths_while_equipped), - username: curr.username - } - return acc - }, {}) + const dataString = JSON.stringify(data) const buffer = Buffer.from(dataString) const size = buffer.length diff --git a/src/clientMain.ts b/src/clientMain.ts index 9fe8d28..4b62501 100644 --- a/src/clientMain.ts +++ b/src/clientMain.ts @@ -5,6 +5,8 @@ import express from 'express' import cors from 'cors' import client from './client/client' import { dbReady } from './db/db' +import processAll from './process/process' +import listenKills from './process/onKill' dotenv.config() const cCPUs = os.cpus().length @@ -38,6 +40,8 @@ new Promise((resolve, reject) => { app.use('/', client) await dbReady() + await processAll() + await listenKills() const listenServer = app.listen(port, '0.0.0.0', () => { console.log(`Tone client api listening on port ${port}`) resolve(listenServer) diff --git a/src/process/onKill.ts b/src/process/onKill.ts index 825f711..0f018ed 100644 --- a/src/process/onKill.ts +++ b/src/process/onKill.ts @@ -1,5 +1,5 @@ import { Client } from 'pg' -// import { allData } from './process' +import { allData } from './process' export const pgClient = new Client({ host: process.env.POSTGRES_HOST, @@ -12,53 +12,23 @@ export default async function listenKills () { await pgClient.connect() await pgClient.query('LISTEN new_kill') - /* pgClient.on('notification', (data) => { + pgClient.on('notification', (data) => { if (!data.payload) return const payload = JSON.parse(data.payload) + console.log(`recieved kill for ${payload.attacker_name as string}`) + let killEntry = allData[payload.attacker_id] + let deathEntry = allData[payload.victim_id] - let killEntry = allData.find( - (e) => - e.attacker_id == payload.attacker_id && - e.cause_of_death == payload.cause_of_death && - e.game_mode == payload.game_mode && - e.host == payload.host && - e.map == payload.map && - e.servername == payload.servername - ) - let deathEntry = allData.find( - (e) => - e.attacker_id == payload.victim_id && - e.cause_of_death == payload.cause_of_death && - e.game_mode == payload.game_mode && - e.host == payload.host && - e.map == payload.map && - e.servername == payload.servername - ) - let deathWithWeaponEntry = allData.find( - (e) => - e.attacker_id == payload.victim_id && - e.cause_of_death == payload.victim_current_weapon && - e.game_mode == payload.game_mode && - e.host == payload.host && - e.map == payload.map && - e.servername == payload.servername - ) if (!killEntry) { killEntry = { kills: 0, deaths: 0, max_distance: 0, total_distance: 0, - deaths_with_weapon: 0, - attacker_name: payload.attacker_name, - attacker_id: payload.attacker_id, - cause_of_death: payload.cause_of_death, - game_mode: payload.game_mode, - host: payload.host, - map: payload.map, - servername: payload.servername + deaths_while_equipped: 0, + username: payload.attacker_name } - allData.push(killEntry) + allData[payload.attacker_id] = killEntry } if (!deathEntry) { @@ -67,47 +37,21 @@ export default async function listenKills () { deaths: 0, max_distance: 0, total_distance: 0, - deaths_with_weapon: 0, - attacker_name: payload.victim_name, - attacker_id: payload.victim_id, - cause_of_death: payload.cause_of_death, - game_mode: payload.game_mode, - host: payload.host, - map: payload.map, - servername: payload.servername + deaths_while_equipped: 0, + username: payload.attacker_name } - allData.push(deathEntry) + allData[payload.victim_id] = deathEntry } - if (!deathWithWeaponEntry) { - deathWithWeaponEntry = { - kills: 0, - deaths: 0, - max_distance: 0, - total_distance: 0, - deaths_with_weapon: 0, - attacker_name: payload.victim_name, - attacker_id: payload.victim_id, - cause_of_death: payload.victim_current_weapon, - game_mode: payload.game_mode, - host: payload.host, - map: payload.map, - servername: payload.servername - } - allData.push(deathWithWeaponEntry) - } - - if (payload.victim_id != payload.attacker_id) { + if (payload.victim_id !== payload.attacker_id) { killEntry.kills++ killEntry.total_distance = Number(killEntry.total_distance) + Number(payload.distance) killEntry.max_distance = Math.max(killEntry.max_distance || 0, payload.distance) } deathEntry.deaths++ - deathEntry.attacker_name = payload.victim_name - deathWithWeaponEntry.deaths_with_weapon++ - deathWithWeaponEntry.attacker_name = payload.victim_name - killEntry.attacker_name = payload.attacker_name - - }) */ + deathEntry.deaths_while_equipped++ + deathEntry.username = payload.victim_name + killEntry.username = payload.attacker_name + }) return pgClient } diff --git a/src/process/process.ts b/src/process/process.ts index 5513ca0..ac9f9c7 100644 --- a/src/process/process.ts +++ b/src/process/process.ts @@ -1,6 +1,6 @@ import db from '../db/db' import { sql } from 'kysely' -const { count, max, sum, coalesce } = db.fn +const { max, sum } = db.fn export let allData: Awaited> @@ -12,119 +12,31 @@ async function processAll (): Promise { console.log('Starting data calculation...') const timeStart = new Date() - // allData = await processGlobalStats() + allData = await processGlobalStats() console.log(`Data calculation finished. Took + ${Math.abs(new Date().getTime() - timeStart.getTime()) / 1000} seconds`) } +interface KillRecord { + kills: number + deaths: number + deaths_while_equipped: number + username?: string + max_distance: number + total_distance: number +} + async function processGlobalStats () { - return await db - .with('kills', (db) => - db - .selectFrom('kill') - .select([ - count('id').as('kills'), - coalesce( - max('distance'), - sql`0` - ).as('max_distance'), - coalesce(sum('distance'), sql`0`).as( - 'total_distance' - ), - 'attacker_id', - sql`last(attacker_name)`.as('attacker_name'), - 'cause_of_death', - 'map', - 'game_mode', - 'servername', - 'host' - ]) - .whereRef('attacker_id', '!=', 'victim_id') - .groupBy([ - 'attacker_id', - 'cause_of_death', - 'map', - 'servername', - 'host', - 'game_mode' - ]) - ) - .with('deaths', (db) => - db - .selectFrom('kill') - .select([ - count('id').as('deaths'), - 'victim_id', - sql`last(victim_name)`.as('victim_name'), - 'cause_of_death', - 'map', - 'game_mode', - 'servername', - 'host' - ]) - .groupBy([ - 'victim_id', - 'cause_of_death', - 'map', - 'servername', - 'host', - 'game_mode' - ]) - ).with('deathswithweapon', (db) => - db - .selectFrom('kill') - .select([ - count('id').as('deaths'), - 'victim_id', - 'victim_current_weapon', - 'map', - 'game_mode', - 'servername', - 'host' - ]) - .groupBy([ - 'victim_id', - 'victim_current_weapon', - 'map', - 'servername', - 'host', - 'game_mode' - ]) - ) - .selectFrom('kills') - .fullJoin('deaths', (join) => - join - .onRef('deaths.victim_id', '=', 'kills.attacker_id') - .onRef('deaths.cause_of_death', '=', 'kills.cause_of_death') - .onRef('deaths.servername', '=', 'kills.servername') - .onRef('deaths.host', '=', 'kills.host') - .onRef('deaths.game_mode', '=', 'kills.game_mode') - .onRef('deaths.map', '=', 'kills.map') - ) - .leftJoin('deathswithweapon', (join) => - join - .onRef('deathswithweapon.victim_id', '=', 'kills.attacker_id') - .onRef('deathswithweapon.victim_current_weapon', '=', 'kills.cause_of_death') - .onRef('deathswithweapon.servername', '=', 'kills.servername') - .onRef('deathswithweapon.host', '=', 'kills.host') - .onRef('deathswithweapon.game_mode', '=', 'kills.game_mode') - .onRef('deathswithweapon.map', '=', 'kills.map') - ) - // .selectAll('kills') - .select([ - sql`COALESCE(kills.attacker_name, deaths.victim_name)`.as('attacker_name'), - sql`COALESCE(kills.attacker_id, deaths.victim_id)`.as('attacker_id'), - sql`COALESCE(kills.kills, 0)`.as('kills'), - sql`COALESCE(kills.total_distance, 0)`.as('total_distance'), - sql`COALESCE(deathswithweapon.deaths, 0)`.as('deaths_with_weapon'), - sql`COALESCE(kills.max_distance, 0)`.as('max_distance'), - sql`COALESCE(deaths.deaths, 0)`.as('deaths'), - sql`COALESCE(kills.servername, deaths.servername)`.as('servername'), - sql`COALESCE(kills.host, deaths.host)`.as('host'), - sql`COALESCE(kills.cause_of_death, deaths.cause_of_death)`.as('cause_of_death'), - sql`COALESCE(kills.map, deaths.map)`.as('map'), - sql`COALESCE(kills.game_mode, deaths.game_mode)`.as('game_mode') - ]) - .execute() + const data = await db.selectFrom('kill_view').select(['attacker_id', sql`last(attacker_name)`.as('username'), sum('kills').as('kills'), sum('deaths').as('deaths'), sum('deaths_with_weapon').as('deaths_while_equipped'), sum('total_distance').as('total_distance'), max('max_distance').as('max_distance')]).groupBy('attacker_id').execute() + return data.reduce>((acc, curr) => { + acc[curr.attacker_id] = { + kills: Number(curr.kills), + deaths: Number(curr.deaths), + max_distance: Number(curr.max_distance), + total_distance: Number(curr.total_distance), + deaths_while_equipped: Number(curr.deaths_while_equipped) + } + return acc + }, {}) } export default processAll diff --git a/tests/realtime.test.ts b/tests/realtime.test.ts index 5715837..8a79e45 100644 --- a/tests/realtime.test.ts +++ b/tests/realtime.test.ts @@ -112,7 +112,6 @@ describe('realtime', () => { test('update player', async () => { jest.setTimeout(15000) - const yea = waitFor(1000) const response = await fetch(`http://127.0.0.1:3001/kill`, { method: "POST", // *GET, POST, PUT, DELETE, etc. credentials: "same-origin", // include, *same-origin, omit @@ -123,10 +122,12 @@ describe('realtime', () => { body: JSON.stringify(data), // body data type must match "Content-Type" header }); expect(response.status).toBe(201) + const yea = waitFor(1000) await yea }) test('check player update', async () => { + const request = await fetch("http://127.0.0.1:3000/players") const data = await request.json() const player = data["1"]